From 9b8d688657f7044ad27ff81f7eb7d50f7f593ed6 Mon Sep 17 00:00:00 2001 From: Akarys42 Date: Fri, 25 Oct 2019 12:10:26 +0200 Subject: Autodelete offensive messages after one week. If the filter cog filter a message that's considered as offensive (filter["offensive_msg"] is True), the cog create a new offensive message object in the bot db with a delete_date of one week after it was sent. A background task run every day, pull up a list of message to delete, find them back, and delete them. --- bot/cogs/filtering.py | 89 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 79 insertions(+), 10 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 1d1d74e74..d1d28ac10 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -1,12 +1,15 @@ +import asyncio +import datetime import logging import re from typing import Optional, Union import discord.errors from dateutil.relativedelta import relativedelta -from discord import Colour, DMChannel, Member, Message, TextChannel +from discord import Colour, DMChannel, Member, Message, NotFound, TextChannel from discord.ext.commands import Bot, Cog +from bot.api import ResponseCodeError from bot.cogs.moderation import ModLog from bot.constants import ( Channels, Colours, DEBUG_MODE, @@ -16,13 +19,13 @@ from bot.constants import ( log = logging.getLogger(__name__) INVITE_RE = re.compile( - r"(?:discord(?:[\.,]|dot)gg|" # Could be discord.gg/ - r"discord(?:[\.,]|dot)com(?:\/|slash)invite|" # or discord.com/invite/ + r"(?:discord(?:[\.,]|dot)gg|" # Could be discord.gg/ + r"discord(?:[\.,]|dot)com(?:\/|slash)invite|" # or discord.com/invite/ r"discordapp(?:[\.,]|dot)com(?:\/|slash)invite|" # or discordapp.com/invite/ - r"discord(?:[\.,]|dot)me|" # or discord.me - r"discord(?:[\.,]|dot)io" # or discord.io. - r")(?:[\/]|slash)" # / or 'slash' - r"([a-zA-Z0-9]+)", # the invite code itself + r"discord(?:[\.,]|dot)me|" # or discord.me + r"discord(?:[\.,]|dot)io" # or discord.io. + r")(?:[\/]|slash)" # / or 'slash' + r"([a-zA-Z0-9]+)", # the invite code itself flags=re.IGNORECASE ) @@ -36,6 +39,8 @@ TOKEN_WATCHLIST_PATTERNS = [ re.compile(fr'{expression}', flags=re.IGNORECASE) for expression in Filter.token_watchlist ] +OFFENSIVE_MSG_DELETE_TIME = datetime.timedelta(days=7) # Time before an offensive msg is deleted. + class Filtering(Cog): """Filtering out invites, blacklisting domains, and warning us of certain regular expressions.""" @@ -54,7 +59,8 @@ class Filtering(Cog): "notification_msg": ( "Your post has been removed for abusing Unicode character rendering (aka Zalgo text). " f"{_staff_mistake_str}" - ) + ), + "offensive_msg": False }, "filter_invites": { "enabled": Filter.filter_invites, @@ -65,7 +71,8 @@ class Filtering(Cog): "notification_msg": ( f"Per Rule 10, your invite link has been removed. {_staff_mistake_str}\n\n" r"Our server rules can be found here: " - ) + ), + "offensive_msg": False }, "filter_domains": { "enabled": Filter.filter_domains, @@ -75,28 +82,47 @@ class Filtering(Cog): "user_notification": Filter.notify_user_domains, "notification_msg": ( f"Your URL has been removed because it matched a blacklisted domain. {_staff_mistake_str}" - ) + ), + "offensive_msg": False }, "watch_rich_embeds": { "enabled": Filter.watch_rich_embeds, "function": self._has_rich_embed, "type": "watchlist", "content_only": False, + "offensive_msg": False }, "watch_words": { "enabled": Filter.watch_words, "function": self._has_watchlist_words, "type": "watchlist", "content_only": True, + "offensive_msg": True }, "watch_tokens": { "enabled": Filter.watch_tokens, "function": self._has_watchlist_tokens, "type": "watchlist", "content_only": True, + "offensive_msg": True }, } + self.deletion_task = None + self.bot.loop.create_task(self.init_deletion_task()) + + def cog_unload(self) -> None: + """Cancel any running updater tasks on cog unload.""" + if self.deletion_task is not None: + self.deletion_task.cancel() + + async def init_deletion_task(self) -> None: + """Start offensive messages deletion event loop if it hasn't already started.""" + await self.bot.wait_until_ready() + if self.deletion_task is None: + coro = delete_offensive_msg(self.bot) + self.deletion_task = self.bot.loop.create_task(coro) + @property def mod_log(self) -> ModLog: """Get currently loaded ModLog cog instance.""" @@ -159,6 +185,21 @@ class Filtering(Cog): triggered = await _filter["function"](msg) if triggered: + # If the message is classed as offensive, we store it in the site db and + # it will be deleted it after one week. + if _filter["offensive_msg"]: + delete_date = msg.created_at.date() + OFFENSIVE_MSG_DELETE_TIME + await self.bot.api_client.post( + 'bot/offensive-message', + json={ + 'id': msg.id, + 'channel_id': msg.channel.id, + 'delete_date': delete_date.isoformat() + } + ) + log.trace(f"Offensive message will be deleted on " + f"{delete_date.isoformat()}") + # If this is a filter (not a watchlist), we should delete the message. if _filter["type"] == "filter": try: @@ -360,6 +401,34 @@ class Filtering(Cog): await channel.send(f"{filtered_member.mention} {reason}") +async def delete_offensive_msg(bot: Bot) -> None: + """Background task that pull up a list of offensive messages every day and delete them.""" + while True: + tomorrow = datetime.date.today() + datetime.timedelta(days=1) + time_until_next = datetime.datetime(tomorrow.year, tomorrow.month, tomorrow.day) - datetime.datetime.now() + try: + msg_list = await bot.api_client.get( + 'bot/offensive-message', + params={'delete_date': datetime.date.today().isoformat()} + ) + except ResponseCodeError as e: + log.error(f"Failed to get offending messages to delete (got code {e.response.status}), " + f"retrying in 30 minutes.") + time_until_next = datetime.timedelta(minutes=30) + msg_list = [] + for msg in msg_list: + try: + channel = bot.get_channel(msg['channel_id']) + if channel: + msg_obj = await channel.fetch_message(msg['id']) + await msg_obj.delete() + except NotFound: + log.info(f"Tried to delete message {msg['id']}, but the message can't be found " + f"(it has been probably already deleted).") + log.info(f"Deleted {len(msg_list)} offensive message(s).") + await asyncio.sleep(time_until_next.seconds) + + def setup(bot: Bot) -> None: """Filtering cog load.""" bot.add_cog(Filtering(bot)) -- cgit v1.2.3 From 59914bf3e741d654d999ff5eb7f9c12621628c6b Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Fri, 25 Oct 2019 15:00:20 +0200 Subject: Revert whitespace changes --- bot/cogs/filtering.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index d1d28ac10..4c99be0af 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -19,13 +19,13 @@ from bot.constants import ( log = logging.getLogger(__name__) INVITE_RE = re.compile( - r"(?:discord(?:[\.,]|dot)gg|" # Could be discord.gg/ - r"discord(?:[\.,]|dot)com(?:\/|slash)invite|" # or discord.com/invite/ + r"(?:discord(?:[\.,]|dot)gg|" # Could be discord.gg/ + r"discord(?:[\.,]|dot)com(?:\/|slash)invite|" # or discord.com/invite/ r"discordapp(?:[\.,]|dot)com(?:\/|slash)invite|" # or discordapp.com/invite/ - r"discord(?:[\.,]|dot)me|" # or discord.me - r"discord(?:[\.,]|dot)io" # or discord.io. - r")(?:[\/]|slash)" # / or 'slash' - r"([a-zA-Z0-9]+)", # the invite code itself + r"discord(?:[\.,]|dot)me|" # or discord.me + r"discord(?:[\.,]|dot)io" # or discord.io. + r")(?:[\/]|slash)" # / or 'slash' + r"([a-zA-Z0-9]+)", # the invite code itself flags=re.IGNORECASE ) -- cgit v1.2.3 From 9c78146c79dd7b4c3a633f3eabeef2036eb8ab7f Mon Sep 17 00:00:00 2001 From: Akarys42 Date: Fri, 25 Oct 2019 18:39:58 +0200 Subject: Move offensive message delete time to config file. --- bot/cogs/filtering.py | 2 +- bot/constants.py | 1 + config-default.yml | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 4c99be0af..f9aee5a9a 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -39,7 +39,7 @@ TOKEN_WATCHLIST_PATTERNS = [ re.compile(fr'{expression}', flags=re.IGNORECASE) for expression in Filter.token_watchlist ] -OFFENSIVE_MSG_DELETE_TIME = datetime.timedelta(days=7) # Time before an offensive msg is deleted. +OFFENSIVE_MSG_DELETE_TIME = datetime.timedelta(days=Filter.offensive_msg_delete_time) class Filtering(Cog): diff --git a/bot/constants.py b/bot/constants.py index 4beae84e9..6106d911c 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -211,6 +211,7 @@ class Filter(metaclass=YAMLGetter): notify_user_domains: bool ping_everyone: bool + offensive_msg_delete_time: int guild_invite_whitelist: List[int] domain_blacklist: List[str] word_watchlist: List[str] diff --git a/config-default.yml b/config-default.yml index 197743296..fc702e991 100644 --- a/config-default.yml +++ b/config-default.yml @@ -161,7 +161,8 @@ filter: notify_user_domains: false # Filter configuration - ping_everyone: true # Ping @everyone when we send a mod-alert? + ping_everyone: true # Ping @everyone when we send a mod-alert? + offensive_msg_delete_time: 7 # How many days before deleting an offensive message? guild_invite_whitelist: - 280033776820813825 # Functional Programming -- cgit v1.2.3 From f67378c77a45c581c57d1dfdd5a704319e83ba3a Mon Sep 17 00:00:00 2001 From: Akarys42 Date: Fri, 25 Oct 2019 18:43:27 +0200 Subject: Remove the possibility that we send a message to the API that the filter has already deleted. --- bot/cogs/filtering.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index f9aee5a9a..ea6919707 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -185,21 +185,6 @@ class Filtering(Cog): triggered = await _filter["function"](msg) if triggered: - # If the message is classed as offensive, we store it in the site db and - # it will be deleted it after one week. - if _filter["offensive_msg"]: - delete_date = msg.created_at.date() + OFFENSIVE_MSG_DELETE_TIME - await self.bot.api_client.post( - 'bot/offensive-message', - json={ - 'id': msg.id, - 'channel_id': msg.channel.id, - 'delete_date': delete_date.isoformat() - } - ) - log.trace(f"Offensive message will be deleted on " - f"{delete_date.isoformat()}") - # If this is a filter (not a watchlist), we should delete the message. if _filter["type"] == "filter": try: @@ -216,6 +201,21 @@ class Filtering(Cog): except discord.errors.NotFound: return + # If the message is classed as offensive, we store it in the site db and + # it will be deleted it after one week. + if _filter["offensive_msg"]: + delete_date = msg.created_at.date() + OFFENSIVE_MSG_DELETE_TIME + await self.bot.api_client.post( + 'bot/offensive-message', + json={ + 'id': msg.id, + 'channel_id': msg.channel.id, + 'delete_date': delete_date.isoformat() + } + ) + log.trace(f"Offensive message will be deleted on " + f"{delete_date.isoformat()}") + # Notify the user if the filter specifies if _filter["user_notification"]: await self.notify_member(msg.author, _filter["notification_msg"], msg.channel) -- cgit v1.2.3 From 1eb057b229c9dcae15834c55faf7188360b675a2 Mon Sep 17 00:00:00 2001 From: Akarys42 Date: Fri, 25 Oct 2019 18:44:50 +0200 Subject: Rename offensive_msg flag to schedule_deletion. --- bot/cogs/filtering.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index ea6919707..1342dade8 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -60,7 +60,7 @@ class Filtering(Cog): "Your post has been removed for abusing Unicode character rendering (aka Zalgo text). " f"{_staff_mistake_str}" ), - "offensive_msg": False + "schedule_deletion": False }, "filter_invites": { "enabled": Filter.filter_invites, @@ -72,7 +72,7 @@ class Filtering(Cog): f"Per Rule 10, your invite link has been removed. {_staff_mistake_str}\n\n" r"Our server rules can be found here: " ), - "offensive_msg": False + "schedule_deletion": False }, "filter_domains": { "enabled": Filter.filter_domains, @@ -83,28 +83,28 @@ class Filtering(Cog): "notification_msg": ( f"Your URL has been removed because it matched a blacklisted domain. {_staff_mistake_str}" ), - "offensive_msg": False + "schedule_deletion": False }, "watch_rich_embeds": { "enabled": Filter.watch_rich_embeds, "function": self._has_rich_embed, "type": "watchlist", "content_only": False, - "offensive_msg": False + "schedule_deletion": False }, "watch_words": { "enabled": Filter.watch_words, "function": self._has_watchlist_words, "type": "watchlist", "content_only": True, - "offensive_msg": True + "schedule_deletion": True }, "watch_tokens": { "enabled": Filter.watch_tokens, "function": self._has_watchlist_tokens, "type": "watchlist", "content_only": True, - "offensive_msg": True + "schedule_deletion": True }, } @@ -203,7 +203,7 @@ class Filtering(Cog): # If the message is classed as offensive, we store it in the site db and # it will be deleted it after one week. - if _filter["offensive_msg"]: + if _filter["schedule_deletion"]: delete_date = msg.created_at.date() + OFFENSIVE_MSG_DELETE_TIME await self.bot.api_client.post( 'bot/offensive-message', -- cgit v1.2.3 From f306c4153a5d4fb969856a2282e7ff3f7a111885 Mon Sep 17 00:00:00 2001 From: Akarys42 Date: Fri, 25 Oct 2019 21:16:04 +0200 Subject: Use Scheduler instead of a custom async loop --- bot/cogs/filtering.py | 84 ++++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 1342dade8..35c14f101 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -2,19 +2,20 @@ import asyncio import datetime import logging import re -from typing import Optional, Union +from typing import Mapping, Optional, Union import discord.errors from dateutil.relativedelta import relativedelta from discord import Colour, DMChannel, Member, Message, NotFound, TextChannel from discord.ext.commands import Bot, Cog -from bot.api import ResponseCodeError from bot.cogs.moderation import ModLog from bot.constants import ( Channels, Colours, DEBUG_MODE, Filter, Icons, URLs ) +from bot.utils.scheduling import Scheduler +from bot.utils.time import wait_until log = logging.getLogger(__name__) @@ -42,11 +43,12 @@ TOKEN_WATCHLIST_PATTERNS = [ OFFENSIVE_MSG_DELETE_TIME = datetime.timedelta(days=Filter.offensive_msg_delete_time) -class Filtering(Cog): +class Filtering(Cog, Scheduler): """Filtering out invites, blacklisting domains, and warning us of certain regular expressions.""" def __init__(self, bot: Bot): self.bot = bot + super().__init__() _staff_mistake_str = "If you believe this was a mistake, please let staff know!" self.filters = { @@ -109,19 +111,7 @@ class Filtering(Cog): } self.deletion_task = None - self.bot.loop.create_task(self.init_deletion_task()) - - def cog_unload(self) -> None: - """Cancel any running updater tasks on cog unload.""" - if self.deletion_task is not None: - self.deletion_task.cancel() - - async def init_deletion_task(self) -> None: - """Start offensive messages deletion event loop if it hasn't already started.""" - await self.bot.wait_until_ready() - if self.deletion_task is None: - coro = delete_offensive_msg(self.bot) - self.deletion_task = self.bot.loop.create_task(coro) + self.bot.loop.create_task(self.reschedule_offensive_msg_deletion()) @property def mod_log(self) -> ModLog: @@ -400,33 +390,45 @@ class Filtering(Cog): except discord.errors.Forbidden: await channel.send(f"{filtered_member.mention} {reason}") + async def _scheduled_task(self, msg: dict) -> None: + """A coroutine which delete the offensive message once the delete date is reached.""" + delete_at = datetime.datetime.fromisoformat(msg['delete_date'][:-1]) + + await wait_until(delete_at) + await self.delete_offensive_msg(msg) + + self.cancel_task(msg['id']) + + async def reschedule_offensive_msg_deletion(self) -> None: + """Get all the pending message deletion from the API and reschedule them.""" + await self.bot.wait_until_ready() + response = await self.bot.api_client.get( + 'bot/offensive-message', + ) + + now = datetime.datetime.utcnow() + loop = asyncio.get_event_loop() -async def delete_offensive_msg(bot: Bot) -> None: - """Background task that pull up a list of offensive messages every day and delete them.""" - while True: - tomorrow = datetime.date.today() + datetime.timedelta(days=1) - time_until_next = datetime.datetime(tomorrow.year, tomorrow.month, tomorrow.day) - datetime.datetime.now() + for msg in response: + delete_at = datetime.datetime.fromisoformat(msg['delete_date'][:-1]) + + if delete_at < now: + await self.delete_offensive_msg(msg) + else: + self.schedule_task(loop, msg['id'], msg) + + async def delete_offensive_msg(self, msg: Mapping[str, str]) -> None: + """Delete an offensive message, and then delete it from the db.""" try: - msg_list = await bot.api_client.get( - 'bot/offensive-message', - params={'delete_date': datetime.date.today().isoformat()} - ) - except ResponseCodeError as e: - log.error(f"Failed to get offending messages to delete (got code {e.response.status}), " - f"retrying in 30 minutes.") - time_until_next = datetime.timedelta(minutes=30) - msg_list = [] - for msg in msg_list: - try: - channel = bot.get_channel(msg['channel_id']) - if channel: - msg_obj = await channel.fetch_message(msg['id']) - await msg_obj.delete() - except NotFound: - log.info(f"Tried to delete message {msg['id']}, but the message can't be found " - f"(it has been probably already deleted).") - log.info(f"Deleted {len(msg_list)} offensive message(s).") - await asyncio.sleep(time_until_next.seconds) + channel = self.bot.get_channel(msg['channel_id']) + if channel: + msg_obj = await channel.fetch_message(msg['id']) + await msg_obj.delete() + except NotFound: + log.info(f"Tried to delete message {msg['id']}, but the message can't be found " + f"(it has been probably already deleted).") + await self.bot.api_client.delete(f'bot/offensive-message/{msg["id"]}') + log.info(f"Deleted the offensive message with id {msg['id']}.") def setup(bot: Bot) -> None: -- cgit v1.2.3 From 95c6e56891a21ebb2d1555cf850daad375d57afe Mon Sep 17 00:00:00 2001 From: Akarys42 Date: Sat, 26 Oct 2019 11:11:15 +0200 Subject: Switch to datetime.datetime --- bot/cogs/filtering.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 5bd72a584..8962a85c1 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -187,25 +187,25 @@ class Filtering(Cog, Scheduler): except discord.errors.NotFound: return + # Notify the user if the filter specifies + if _filter["user_notification"]: + await self.notify_member(msg.author, _filter["notification_msg"], msg.channel) + # If the message is classed as offensive, we store it in the site db and # it will be deleted it after one week. if _filter["schedule_deletion"]: - delete_date = msg.created_at.date() + OFFENSIVE_MSG_DELETE_TIME + delete_date = msg.created_at + OFFENSIVE_MSG_DELETE_TIME await self.bot.api_client.post( 'bot/offensive-message', json={ 'id': msg.id, 'channel_id': msg.channel.id, - 'delete_date': delete_date.isoformat() + 'delete_date': delete_date.isoformat()[:-1] } ) log.trace(f"Offensive message will be deleted on " f"{delete_date.isoformat()}") - # Notify the user if the filter specifies - if _filter["user_notification"]: - await self.notify_member(msg.author, _filter["notification_msg"], msg.channel) - if isinstance(msg.channel, DMChannel): channel_str = "via DM" else: -- cgit v1.2.3 From b57811ea12aa41285e4e4585b951dd105be4b275 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Thu, 12 Dec 2019 09:25:53 +0100 Subject: Add space for readability Co-Authored-By: Mark --- bot/cogs/filtering.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 61c8f389b..39ba22354 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -436,6 +436,7 @@ class Filtering(Cog, Scheduler): except NotFound: log.info(f"Tried to delete message {msg['id']}, but the message can't be found " f"(it has been probably already deleted).") + await self.bot.api_client.delete(f'bot/offensive-message/{msg["id"]}') log.info(f"Deleted the offensive message with id {msg['id']}.") -- cgit v1.2.3 From bc9b335454d0183013b4f89f2923b638dcc127ba Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Thu, 12 Dec 2019 09:38:26 +0100 Subject: Make use of the Bot subclass --- bot/cogs/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index ca84e6240..9d88d9153 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -7,7 +7,7 @@ from typing import Mapping, Optional, Union import discord.errors from dateutil.relativedelta import relativedelta from discord import Colour, DMChannel, Member, Message, NotFound, TextChannel -from discord.ext.commands import Bot, Cog +from discord.ext.commands import Cog from bot.bot import Bot from bot.cogs.moderation import ModLog -- cgit v1.2.3 From 7eea7ad353239a0be1dfd744486e3f46a99cd661 Mon Sep 17 00:00:00 2001 From: Akarys42 Date: Sat, 14 Dec 2019 11:21:44 +0100 Subject: Filtering cog clean up --- bot/cogs/filtering.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 9d88d9153..172c5fa7e 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -111,7 +111,6 @@ class Filtering(Cog, Scheduler): }, } - self.deletion_task = None self.bot.loop.create_task(self.reschedule_offensive_msg_deletion()) @property @@ -201,11 +200,13 @@ class Filtering(Cog, Scheduler): json={ 'id': msg.id, 'channel_id': msg.channel.id, - 'delete_date': delete_date.isoformat()[:-1] + 'delete_date': delete_date.isoformat() } ) - log.trace(f"Offensive message will be deleted on " - f"{delete_date.isoformat()}") + log.trace( + f"Offensive message will be deleted on " + f"{delete_date.isoformat()}" + ) if isinstance(msg.channel, DMChannel): channel_str = "via DM" @@ -412,9 +413,7 @@ class Filtering(Cog, Scheduler): async def reschedule_offensive_msg_deletion(self) -> None: """Get all the pending message deletion from the API and reschedule them.""" await self.bot.wait_until_ready() - response = await self.bot.api_client.get( - 'bot/offensive-message', - ) + response = await self.bot.api_client.get('bot/offensive-message',) now = datetime.datetime.utcnow() loop = asyncio.get_event_loop() @@ -435,8 +434,10 @@ class Filtering(Cog, Scheduler): msg_obj = await channel.fetch_message(msg['id']) await msg_obj.delete() except NotFound: - log.info(f"Tried to delete message {msg['id']}, but the message can't be found " - f"(it has been probably already deleted).") + log.info( + f"Tried to delete message {msg['id']}, but the message can't be found " + f"(it has been probably already deleted)." + ) await self.bot.api_client.delete(f'bot/offensive-message/{msg["id"]}') log.info(f"Deleted the offensive message with id {msg['id']}.") -- cgit v1.2.3 From ba5af375a3acfc160de1ffefa063a915318e6bdd Mon Sep 17 00:00:00 2001 From: Akarys42 Date: Sat, 14 Dec 2019 11:25:59 +0100 Subject: Make use of dateutil.parser.isoparse --- bot/cogs/filtering.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 172c5fa7e..4388b29ad 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -4,6 +4,7 @@ import logging import re from typing import Mapping, Optional, Union +import dateutil import discord.errors from dateutil.relativedelta import relativedelta from discord import Colour, DMChannel, Member, Message, NotFound, TextChannel @@ -403,7 +404,7 @@ class Filtering(Cog, Scheduler): async def _scheduled_task(self, msg: dict) -> None: """A coroutine which delete the offensive message once the delete date is reached.""" - delete_at = datetime.datetime.fromisoformat(msg['delete_date'][:-1]) + delete_at = dateutil.parser.isoparse(msg['delete_date']) await wait_until(delete_at) await self.delete_offensive_msg(msg) @@ -419,7 +420,7 @@ class Filtering(Cog, Scheduler): loop = asyncio.get_event_loop() for msg in response: - delete_at = datetime.datetime.fromisoformat(msg['delete_date'][:-1]) + delete_at = dateutil.parser.isoparse(msg['delete_date']) if delete_at < now: await self.delete_offensive_msg(msg) -- cgit v1.2.3 From e6940b938882aaeda18baa9fcc23cc297c6cfcd2 Mon Sep 17 00:00:00 2001 From: Akarys42 Date: Sat, 14 Dec 2019 11:31:32 +0100 Subject: Rename config entry to offensive_msg_delete_days --- bot/cogs/filtering.py | 2 +- bot/constants.py | 2 +- config-default.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 4388b29ad..c0e115a8f 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -42,7 +42,7 @@ TOKEN_WATCHLIST_PATTERNS = [ re.compile(fr'{expression}', flags=re.IGNORECASE) for expression in Filter.token_watchlist ] -OFFENSIVE_MSG_DELETE_TIME = datetime.timedelta(days=Filter.offensive_msg_delete_time) +OFFENSIVE_MSG_DELETE_TIME = datetime.timedelta(days=Filter.offensive_msg_delete_day) class Filtering(Cog, Scheduler): diff --git a/bot/constants.py b/bot/constants.py index 075722a01..e6f23ff61 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -211,7 +211,7 @@ class Filter(metaclass=YAMLGetter): notify_user_domains: bool ping_everyone: bool - offensive_msg_delete_time: int + offensive_msg_delete_day: int guild_invite_whitelist: List[int] domain_blacklist: List[str] word_watchlist: List[str] diff --git a/config-default.yml b/config-default.yml index 0765407af..33072790b 100644 --- a/config-default.yml +++ b/config-default.yml @@ -180,7 +180,7 @@ filter: # Filter configuration ping_everyone: true # Ping @everyone when we send a mod-alert? - offensive_msg_delete_time: 7 # How many days before deleting an offensive message? + offensive_msg_delete_days: 7 # How many days before deleting an offensive message? guild_invite_whitelist: - 280033776820813825 # Functional Programming -- cgit v1.2.3 From b2a5ab90622e986ace37be6204d6a59823390cce Mon Sep 17 00:00:00 2001 From: Akarys42 Date: Sat, 14 Dec 2019 11:36:37 +0100 Subject: Catch all HTTPExecption --- bot/cogs/filtering.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index c0e115a8f..c0a2c7d3b 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -7,7 +7,7 @@ from typing import Mapping, Optional, Union import dateutil import discord.errors from dateutil.relativedelta import relativedelta -from discord import Colour, DMChannel, Member, Message, NotFound, TextChannel +from discord import Colour, DMChannel, HTTPException, Member, Message, NotFound, TextChannel from discord.ext.commands import Cog from bot.bot import Bot @@ -439,6 +439,10 @@ class Filtering(Cog, Scheduler): f"Tried to delete message {msg['id']}, but the message can't be found " f"(it has been probably already deleted)." ) + except HTTPException: + log.warning( + f"Failed to delete message {msg['id']}." + ) await self.bot.api_client.delete(f'bot/offensive-message/{msg["id"]}') log.info(f"Deleted the offensive message with id {msg['id']}.") -- cgit v1.2.3 From ab6b0032ceb6ad638cee6f174778bc38254ab038 Mon Sep 17 00:00:00 2001 From: Akarys42 Date: Sat, 14 Dec 2019 11:41:30 +0100 Subject: Actually schedule message for deletion --- bot/cogs/filtering.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index c0a2c7d3b..0879bfee6 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -204,6 +204,8 @@ class Filtering(Cog, Scheduler): 'delete_date': delete_date.isoformat() } ) + loop = asyncio.get_event_loop() + self.schedule_task(loop, msg.id, {'id': msg.id, 'channel_id': msg.channel.id}) log.trace( f"Offensive message will be deleted on " f"{delete_date.isoformat()}" -- cgit v1.2.3 From a81ad8f1f7d548f20d1f2428a808e14dbbfe22bc Mon Sep 17 00:00:00 2001 From: Akarys42 Date: Sat, 14 Dec 2019 11:42:10 +0100 Subject: Fix docstring typo --- bot/cogs/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 0879bfee6..2c3f41c05 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -405,7 +405,7 @@ class Filtering(Cog, Scheduler): await channel.send(f"{filtered_member.mention} {reason}") async def _scheduled_task(self, msg: dict) -> None: - """A coroutine which delete the offensive message once the delete date is reached.""" + """A coroutine that delete the offensive message once the delete date is reached.""" delete_at = dateutil.parser.isoparse(msg['delete_date']) await wait_until(delete_at) -- cgit v1.2.3 From 15958986ae6bcb508c4e2dd23c51624d4ee26cb5 Mon Sep 17 00:00:00 2001 From: Akarys42 Date: Sat, 14 Dec 2019 11:47:08 +0100 Subject: Rename route /bot/offensive-message to /bot/offensive-messages --- bot/cogs/filtering.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 2c3f41c05..63f8685c9 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -197,7 +197,7 @@ class Filtering(Cog, Scheduler): if _filter["schedule_deletion"]: delete_date = msg.created_at + OFFENSIVE_MSG_DELETE_TIME await self.bot.api_client.post( - 'bot/offensive-message', + 'bot/offensive-messages', json={ 'id': msg.id, 'channel_id': msg.channel.id, @@ -416,7 +416,7 @@ class Filtering(Cog, Scheduler): async def reschedule_offensive_msg_deletion(self) -> None: """Get all the pending message deletion from the API and reschedule them.""" await self.bot.wait_until_ready() - response = await self.bot.api_client.get('bot/offensive-message',) + response = await self.bot.api_client.get('bot/offensive-messages',) now = datetime.datetime.utcnow() loop = asyncio.get_event_loop() @@ -446,7 +446,7 @@ class Filtering(Cog, Scheduler): f"Failed to delete message {msg['id']}." ) - await self.bot.api_client.delete(f'bot/offensive-message/{msg["id"]}') + await self.bot.api_client.delete(f'bot/offensive-messages/{msg["id"]}') log.info(f"Deleted the offensive message with id {msg['id']}.") -- cgit v1.2.3 From 2f5b9fbd7bc2a11452a982f61c7603d589ded95a Mon Sep 17 00:00:00 2001 From: Akarys42 Date: Sat, 14 Dec 2019 11:52:54 +0100 Subject: Make setting filter.offensive_msg_delete_days plural --- bot/cogs/filtering.py | 2 +- bot/constants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 63f8685c9..a709fe7cd 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -42,7 +42,7 @@ TOKEN_WATCHLIST_PATTERNS = [ re.compile(fr'{expression}', flags=re.IGNORECASE) for expression in Filter.token_watchlist ] -OFFENSIVE_MSG_DELETE_TIME = datetime.timedelta(days=Filter.offensive_msg_delete_day) +OFFENSIVE_MSG_DELETE_TIME = datetime.timedelta(days=Filter.offensive_msg_delete_days) class Filtering(Cog, Scheduler): diff --git a/bot/constants.py b/bot/constants.py index e6f23ff61..f47688185 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -211,7 +211,7 @@ class Filter(metaclass=YAMLGetter): notify_user_domains: bool ping_everyone: bool - offensive_msg_delete_day: int + offensive_msg_delete_days: int guild_invite_whitelist: List[int] domain_blacklist: List[str] word_watchlist: List[str] -- cgit v1.2.3 From 4cf2166d6dee6a04a055db7e14cf4ece0656213a Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 15 Dec 2019 09:39:58 -0800 Subject: Filtering: log the status code of caught HTTPException --- bot/cogs/filtering.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index a709fe7cd..4d91432e7 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -441,10 +441,8 @@ class Filtering(Cog, Scheduler): f"Tried to delete message {msg['id']}, but the message can't be found " f"(it has been probably already deleted)." ) - except HTTPException: - log.warning( - f"Failed to delete message {msg['id']}." - ) + except HTTPException as e: + log.warning(f"Failed to delete message {msg['id']}: status {e.status}") await self.bot.api_client.delete(f'bot/offensive-messages/{msg["id"]}') log.info(f"Deleted the offensive message with id {msg['id']}.") -- cgit v1.2.3 From 7e4a43546490d90c619d08837da69466768fae80 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 15 Dec 2019 09:39:00 -0800 Subject: Filtering: refactor scheduling of deletion task --- bot/cogs/filtering.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 4d91432e7..4c94b73a5 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -1,4 +1,3 @@ -import asyncio import datetime import logging import re @@ -195,21 +194,19 @@ class Filtering(Cog, Scheduler): # If the message is classed as offensive, we store it in the site db and # it will be deleted it after one week. if _filter["schedule_deletion"]: - delete_date = msg.created_at + OFFENSIVE_MSG_DELETE_TIME + delete_date = (msg.created_at + OFFENSIVE_MSG_DELETE_TIME).isoformat() await self.bot.api_client.post( 'bot/offensive-messages', json={ 'id': msg.id, 'channel_id': msg.channel.id, - 'delete_date': delete_date.isoformat() + 'delete_date': delete_date } ) - loop = asyncio.get_event_loop() - self.schedule_task(loop, msg.id, {'id': msg.id, 'channel_id': msg.channel.id}) - log.trace( - f"Offensive message will be deleted on " - f"{delete_date.isoformat()}" - ) + + task_data = {'id': msg.id, 'channel_id': msg.channel.id} + self.schedule_task(self.bot.loop, msg.id, task_data) + log.trace(f"Offensive message {msg.id} will be deleted on {delete_date}") if isinstance(msg.channel, DMChannel): channel_str = "via DM" @@ -335,7 +332,7 @@ class Filtering(Cog, Scheduler): Attempts to catch some of common ways to try to cheat the system. """ - # Remove backslashes to prevent escape character aroundfuckery like + # Remove backslashes to prevent escape character around fuckery like # discord\.gg/gdudes-pony-farm text = text.replace("\\", "") @@ -405,7 +402,7 @@ class Filtering(Cog, Scheduler): await channel.send(f"{filtered_member.mention} {reason}") async def _scheduled_task(self, msg: dict) -> None: - """A coroutine that delete the offensive message once the delete date is reached.""" + """Delete an offensive message once its deletion date is reached.""" delete_at = dateutil.parser.isoparse(msg['delete_date']) await wait_until(delete_at) @@ -419,7 +416,6 @@ class Filtering(Cog, Scheduler): response = await self.bot.api_client.get('bot/offensive-messages',) now = datetime.datetime.utcnow() - loop = asyncio.get_event_loop() for msg in response: delete_at = dateutil.parser.isoparse(msg['delete_date']) @@ -427,7 +423,7 @@ class Filtering(Cog, Scheduler): if delete_at < now: await self.delete_offensive_msg(msg) else: - self.schedule_task(loop, msg['id'], msg) + self.schedule_task(self.bot.loop, msg['id'], msg) async def delete_offensive_msg(self, msg: Mapping[str, str]) -> None: """Delete an offensive message, and then delete it from the db.""" -- cgit v1.2.3 From 832762add664186665904817e9ffc25f79ffb20a Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 15 Dec 2019 10:03:19 -0800 Subject: Filtering: fix comparison between tz naïve and aware datetimes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bot/cogs/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 4c94b73a5..02e3011ab 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -418,7 +418,7 @@ class Filtering(Cog, Scheduler): now = datetime.datetime.utcnow() for msg in response: - delete_at = dateutil.parser.isoparse(msg['delete_date']) + delete_at = dateutil.parser.isoparse(msg['delete_date']).replace(tzinfo=None) if delete_at < now: await self.delete_offensive_msg(msg) -- cgit v1.2.3 From b36206e8c62459f22685c285fb1a7e299f08b1bd Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 15 Dec 2019 11:12:56 -0800 Subject: Filtering: fix missing deletion date in scheduled task data --- bot/cogs/filtering.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 02e3011ab..83e706a26 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -195,17 +195,14 @@ class Filtering(Cog, Scheduler): # it will be deleted it after one week. if _filter["schedule_deletion"]: delete_date = (msg.created_at + OFFENSIVE_MSG_DELETE_TIME).isoformat() - await self.bot.api_client.post( - 'bot/offensive-messages', - json={ - 'id': msg.id, - 'channel_id': msg.channel.id, - 'delete_date': delete_date - } - ) - - task_data = {'id': msg.id, 'channel_id': msg.channel.id} - self.schedule_task(self.bot.loop, msg.id, task_data) + data = { + 'id': msg.id, + 'channel_id': msg.channel.id, + 'delete_date': delete_date + } + + await self.bot.api_client.post('bot/offensive-messages', json=data) + self.schedule_task(self.bot.loop, msg.id, data) log.trace(f"Offensive message {msg.id} will be deleted on {delete_date}") if isinstance(msg.channel, DMChannel): -- cgit v1.2.3 From 0323919b08342fd650ff32e1f3a2fc2d9eee9c59 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Sat, 29 Feb 2020 08:30:14 +0100 Subject: Make sure that the offensive message deletion date returned by the API is naive It could have caused an issue later with a mix of naive and aware datetime Co-Authored-By: Sebastiaan Zeeff <33516116+SebastiaanZ@users.noreply.github.com> --- bot/cogs/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 83e706a26..2d91695f3 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -400,7 +400,7 @@ class Filtering(Cog, Scheduler): async def _scheduled_task(self, msg: dict) -> None: """Delete an offensive message once its deletion date is reached.""" - delete_at = dateutil.parser.isoparse(msg['delete_date']) + delete_at = dateutil.parser.isoparse(msg['delete_date']).replace(tzinfo=None) await wait_until(delete_at) await self.delete_offensive_msg(msg) -- cgit v1.2.3 From ae14de8745c40ebc9e09d785c55f0fd472dc50ae Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Mon, 2 Mar 2020 17:42:34 +0100 Subject: Remove task self cancel. Recent changes to the scheduler requires this line to be removed. --- bot/cogs/filtering.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 2d91695f3..6c9e9a4b7 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -405,8 +405,6 @@ class Filtering(Cog, Scheduler): await wait_until(delete_at) await self.delete_offensive_msg(msg) - self.cancel_task(msg['id']) - async def reschedule_offensive_msg_deletion(self) -> None: """Get all the pending message deletion from the API and reschedule them.""" await self.bot.wait_until_ready() -- cgit v1.2.3 From 0fc2ebbfeb6f6296f8eb74a7191785bfaa551f90 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Mon, 2 Mar 2020 17:48:40 +0100 Subject: Delete the loop argument from schedule_task calls The function doesn't take the loop as an argument anymore --- bot/cogs/filtering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 6c9e9a4b7..347554ae2 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -202,7 +202,7 @@ class Filtering(Cog, Scheduler): } await self.bot.api_client.post('bot/offensive-messages', json=data) - self.schedule_task(self.bot.loop, msg.id, data) + self.schedule_task(msg.id, data) log.trace(f"Offensive message {msg.id} will be deleted on {delete_date}") if isinstance(msg.channel, DMChannel): @@ -418,7 +418,7 @@ class Filtering(Cog, Scheduler): if delete_at < now: await self.delete_offensive_msg(msg) else: - self.schedule_task(self.bot.loop, msg['id'], msg) + self.schedule_task(msg['id'], msg) async def delete_offensive_msg(self, msg: Mapping[str, str]) -> None: """Delete an offensive message, and then delete it from the db.""" -- cgit v1.2.3 From ce5fcdab852600342fe69211b038426ce2821107 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 4 Apr 2020 14:51:13 +0300 Subject: (Banning): Added logging and truncating to correct length for Discord Audit Log when ban reason length is more than 512 characters. --- bot/cogs/moderation/infractions.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index efa19f59e..f41484711 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -244,7 +244,14 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - action = ctx.guild.ban(user, reason=reason, delete_message_days=0) + if len(reason) > 512: + log.info("Ban reason is longer than 512 characters. Reason will be truncated for Audit Log.") + + action = ctx.guild.ban( + user, + reason=f"{reason[:509]}..." if len(reason) > 512 else reason, + delete_message_days=0 + ) await self.apply_infraction(ctx, infraction, user, action) if infraction.get('expires_at') is not None: -- cgit v1.2.3 From 6ad86ace9f34245180aefeed9553c407602602e8 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 4 Apr 2020 14:53:11 +0300 Subject: (Kick Command): Added logging and truncating to correct length for Discord Audit Log when kick reason length is more than 512 characters. --- bot/cogs/moderation/infractions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index f41484711..f8c3e8da3 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -225,7 +225,10 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - action = user.kick(reason=reason) + if len(reason) > 512: + log.info("Kick reason is longer than 512 characters. Reason will be truncated for Audit Log.") + + action = user.kick(reason=f"{reason[:509]}..." if len(reason) > 512 else reason) await self.apply_infraction(ctx, infraction, user, action) @respect_role_hierarchy() -- cgit v1.2.3 From 80e483a7ebaf57e6544429343c94cf0eed8821ef Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 4 Apr 2020 16:03:58 +0300 Subject: (Ban and Kick): Replaced force reason truncating with `textwrap.shorten`. --- bot/cogs/moderation/infractions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index f8c3e8da3..a0bdf0d97 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -1,4 +1,5 @@ import logging +import textwrap import typing as t import discord @@ -228,7 +229,7 @@ class Infractions(InfractionScheduler, commands.Cog): if len(reason) > 512: log.info("Kick reason is longer than 512 characters. Reason will be truncated for Audit Log.") - action = user.kick(reason=f"{reason[:509]}..." if len(reason) > 512 else reason) + action = user.kick(textwrap.shorten(reason, width=509, placeholder="...") if len(reason) > 512 else reason) await self.apply_infraction(ctx, infraction, user, action) @respect_role_hierarchy() @@ -252,7 +253,7 @@ class Infractions(InfractionScheduler, commands.Cog): action = ctx.guild.ban( user, - reason=f"{reason[:509]}..." if len(reason) > 512 else reason, + reason=textwrap.shorten(reason, width=509, placeholder="...") if len(reason) > 512 else reason, delete_message_days=0 ) await self.apply_infraction(ctx, infraction, user, action) -- cgit v1.2.3 From ec8cc8b02b0823deaa4ea2c97801d16d6aef5244 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sun, 5 Apr 2020 18:13:38 +0300 Subject: (Ban and Kick): Applied simplification to reason truncating. --- bot/cogs/moderation/infractions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index a0bdf0d97..5bdea5755 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -229,7 +229,7 @@ class Infractions(InfractionScheduler, commands.Cog): if len(reason) > 512: log.info("Kick reason is longer than 512 characters. Reason will be truncated for Audit Log.") - action = user.kick(textwrap.shorten(reason, width=509, placeholder="...") if len(reason) > 512 else reason) + action = user.kick(reason=textwrap.shorten(reason, width=509, placeholder="...")) await self.apply_infraction(ctx, infraction, user, action) @respect_role_hierarchy() @@ -253,7 +253,7 @@ class Infractions(InfractionScheduler, commands.Cog): action = ctx.guild.ban( user, - reason=textwrap.shorten(reason, width=509, placeholder="...") if len(reason) > 512 else reason, + reason=textwrap.shorten(reason, width=509, placeholder="..."), delete_message_days=0 ) await self.apply_infraction(ctx, infraction, user, action) -- cgit v1.2.3 From aad67373d3ac6d3d3a236d6734a6c22019dce120 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sun, 5 Apr 2020 18:17:16 +0300 Subject: (Mod Scheduler): Added reason truncations to Scheduler's `apply_infraction` --- bot/cogs/moderation/scheduler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 917697be9..45e9d58ad 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -84,7 +84,8 @@ class InfractionScheduler(Scheduler): """Apply an infraction to the user, log the infraction, and optionally notify the user.""" infr_type = infraction["type"] icon = utils.INFRACTION_ICONS[infr_type][0] - reason = infraction["reason"] + # Truncate reason when it's too long to avoid raising error on sending ModLog entry + reason = textwrap.shorten(infraction["reason"], width=1900, placeholder="...") expiry = time.format_infraction_with_duration(infraction["expires_at"]) id_ = infraction['id'] -- cgit v1.2.3 From 72768b432b07acd3b1bfd5533c55241126329886 Mon Sep 17 00:00:00 2001 From: rohanjnr Date: Sun, 5 Apr 2020 21:22:28 +0530 Subject: Add feature to restrict tags to specific role(s) --- bot/cogs/tags.py | 52 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index 539105017..9c897ad36 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -4,7 +4,7 @@ import time from pathlib import Path from typing import Callable, Dict, Iterable, List, Optional -from discord import Colour, Embed +from discord import Colour, Embed, Member from discord.ext.commands import Cog, Context, group from bot import constants @@ -36,18 +36,32 @@ class Tags(Cog): """Get all tags.""" # Save all tags in memory. cache = {} - tag_files = Path("bot", "resources", "tags").iterdir() + tag_files = Path("bot", "resources", "tags").glob("**/*") for file in tag_files: - tag_title = file.stem - tag = { - "title": tag_title, - "embed": { - "description": file.read_text() + file_path = str(file).split("/") + if file.is_file(): + tag_title = file.stem + tag = { + "title": tag_title, + "embed": { + "description": file.read_text() + }, + "restricted_to": "developers" } - } - cache[tag_title] = tag + if len(file_path) == 5: + restricted_to = file_path[3] + tag["restricted_to"] = restricted_to + + cache[tag_title] = tag return cache + @staticmethod + def check_accessibility(user: Member, tag: dict) -> bool: + """Check if user can access a tag.""" + if tag["restricted_to"].lower() in [role.name.lower() for role in user.roles]: + return True + return False + @staticmethod def _fuzzy_search(search: str, target: str) -> float: """A simple scoring algorithm based on how many letters are found / total, with order in mind.""" @@ -92,7 +106,7 @@ class Tags(Cog): return self._get_suggestions(tag_name) return found - def _get_tags_via_content(self, check: Callable[[Iterable], bool], keywords: str) -> list: + def _get_tags_via_content(self, check: Callable[[Iterable], bool], keywords: str, user: Member) -> list: """ Search for tags via contents. @@ -113,8 +127,9 @@ class Tags(Cog): matching_tags = [] for tag in self._cache.values(): - if check(query in tag['embed']['description'].casefold() for query in keywords_processed): - matching_tags.append(tag) + if self.check_accessibility(user, tag): + if check(query in tag['embed']['description'].casefold() for query in keywords_processed): + matching_tags.append(tag) return matching_tags @@ -151,7 +166,7 @@ class Tags(Cog): Only search for tags that has ALL the keywords. """ - matching_tags = self._get_tags_via_content(all, keywords) + matching_tags = self._get_tags_via_content(all, keywords, ctx.author) await self._send_matching_tags(ctx, keywords, matching_tags) @search_tag_content.command(name='any') @@ -161,7 +176,7 @@ class Tags(Cog): Search for tags that has ANY of the keywords. """ - matching_tags = self._get_tags_via_content(any, keywords or 'any') + matching_tags = self._get_tags_via_content(any, keywords or 'any', ctx.author) await self._send_matching_tags(ctx, keywords, matching_tags) @tags_group.command(name='get', aliases=('show', 'g')) @@ -198,6 +213,10 @@ class Tags(Cog): if tag_name is not None: founds = self._get_tag(tag_name) + for found_tag in founds: + if not self.check_accessibility(ctx.author, found_tag): + founds.remove(found_tag) + if len(founds) == 1: tag = founds[0] if ctx.channel.id not in TEST_CHANNELS: @@ -222,7 +241,10 @@ class Tags(Cog): else: embed: Embed = Embed(title="**Current tags**") await LinePaginator.paginate( - sorted(f"**»** {tag['title']}" for tag in tags), + sorted( + f"**»** {tag['title']}" for tag in tags + if self.check_accessibility(ctx.author, tag) + ), ctx, embed, footer_text=FOOTER_TEXT, -- cgit v1.2.3 From 00d22a316041a8670903eb5fd4b4a7d143993330 Mon Sep 17 00:00:00 2001 From: rohanjnr Date: Sun, 5 Apr 2020 22:04:51 +0530 Subject: Remove unnecessary variable creation and join two if statements --- bot/cogs/tags.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index 9c897ad36..bb74ab1ca 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -49,8 +49,7 @@ class Tags(Cog): "restricted_to": "developers" } if len(file_path) == 5: - restricted_to = file_path[3] - tag["restricted_to"] = restricted_to + tag["restricted_to"] = file_path[3] cache[tag_title] = tag return cache @@ -127,9 +126,11 @@ class Tags(Cog): matching_tags = [] for tag in self._cache.values(): - if self.check_accessibility(user, tag): - if check(query in tag['embed']['description'].casefold() for query in keywords_processed): - matching_tags.append(tag) + if ( + self.check_accessibility(user, tag) + and check(query in tag['embed']['description'].casefold() for query in keywords_processed) + ): + matching_tags.append(tag) return matching_tags -- cgit v1.2.3 From 025857541b7a0cbb77adf2a0282873c9e116169a Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 6 Apr 2020 08:48:32 +0300 Subject: (Ban and Kick): Changed length in `textwrap.shorten` from 309 to 312 because shorten already include `placeholder` to length. --- bot/cogs/moderation/infractions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 5bdea5755..7a044fc1c 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -229,7 +229,7 @@ class Infractions(InfractionScheduler, commands.Cog): if len(reason) > 512: log.info("Kick reason is longer than 512 characters. Reason will be truncated for Audit Log.") - action = user.kick(reason=textwrap.shorten(reason, width=509, placeholder="...")) + action = user.kick(reason=textwrap.shorten(reason, width=512, placeholder="...")) await self.apply_infraction(ctx, infraction, user, action) @respect_role_hierarchy() @@ -253,7 +253,7 @@ class Infractions(InfractionScheduler, commands.Cog): action = ctx.guild.ban( user, - reason=textwrap.shorten(reason, width=509, placeholder="..."), + reason=textwrap.shorten(reason, width=512, placeholder="..."), delete_message_days=0 ) await self.apply_infraction(ctx, infraction, user, action) -- cgit v1.2.3 From c4f7359d2e301e6ab19666a6867c9cb69892da0b Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 6 Apr 2020 08:50:46 +0300 Subject: (Ban and Kick): Added space to `textwrap.shorten` `placeholder`. --- bot/cogs/moderation/infractions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 7a044fc1c..2c809535b 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -253,7 +253,7 @@ class Infractions(InfractionScheduler, commands.Cog): action = ctx.guild.ban( user, - reason=textwrap.shorten(reason, width=512, placeholder="..."), + reason=textwrap.shorten(reason, width=512, placeholder=" ..."), delete_message_days=0 ) await self.apply_infraction(ctx, infraction, user, action) -- cgit v1.2.3 From 482c3f4b475cdbe16b377dd5bb85910be0387166 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 09:04:15 +0300 Subject: (Mod Utils): Added shortening reason on embed creation in `notify_infraction`. --- bot/cogs/moderation/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py index 3598f3b1f..9811d059f 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/cogs/moderation/utils.py @@ -135,7 +135,7 @@ async def notify_infraction( description=textwrap.dedent(f""" **Type:** {infr_type.capitalize()} **Expires:** {expires_at or "N/A"} - **Reason:** {reason or "No reason provided."} + **Reason:** {textwrap.shorten(reason, width=1500, placeholder="...") or "No reason provided."} """), colour=Colours.soft_red ) -- cgit v1.2.3 From a59092659271832a46c8ab0166031bffdc68c0a6 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 09:06:17 +0300 Subject: (Infractions): Removed unnecessary logging that notify when reason will be truncated for Audit Log. --- bot/cogs/moderation/infractions.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 2c809535b..d1e77311c 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -226,9 +226,6 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - if len(reason) > 512: - log.info("Kick reason is longer than 512 characters. Reason will be truncated for Audit Log.") - action = user.kick(reason=textwrap.shorten(reason, width=512, placeholder="...")) await self.apply_infraction(ctx, infraction, user, action) @@ -248,9 +245,6 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - if len(reason) > 512: - log.info("Ban reason is longer than 512 characters. Reason will be truncated for Audit Log.") - action = ctx.guild.ban( user, reason=textwrap.shorten(reason, width=512, placeholder=" ..."), -- cgit v1.2.3 From 42e18061a21e0ec1b8a4a692bc8d96f8ef1fd45b Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 09:09:10 +0300 Subject: (Infractions): Moved truncated reason to variable instead on ban coroutine creating. --- bot/cogs/moderation/infractions.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index d1e77311c..3340744b0 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -245,11 +245,9 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - action = ctx.guild.ban( - user, - reason=textwrap.shorten(reason, width=512, placeholder=" ..."), - delete_message_days=0 - ) + truncated_reason = textwrap.shorten(reason, width=512, placeholder=" ...") + + action = ctx.guild.ban(user, reason=truncated_reason, delete_message_days=0) await self.apply_infraction(ctx, infraction, user, action) if infraction.get('expires_at') is not None: -- cgit v1.2.3 From 10ea74bc5ed390c36d64a0f7413b8422f158708a Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 09:16:36 +0300 Subject: (Superstarify, Scheduler): Added reason shortening for ModLog. --- bot/cogs/moderation/scheduler.py | 2 +- bot/cogs/moderation/superstarify.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 45e9d58ad..7404ec8ac 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -326,7 +326,7 @@ class InfractionScheduler(Scheduler): log_text = { "Member": f"<@{user_id}>", "Actor": str(self.bot.get_user(actor) or actor), - "Reason": infraction["reason"], + "Reason": textwrap.shorten(infraction["reason"], width=1500, placeholder="..."), "Created": created, } diff --git a/bot/cogs/moderation/superstarify.py b/bot/cogs/moderation/superstarify.py index ca3dc4202..d77e61e6b 100644 --- a/bot/cogs/moderation/superstarify.py +++ b/bot/cogs/moderation/superstarify.py @@ -183,7 +183,7 @@ class Superstarify(InfractionScheduler, Cog): text=textwrap.dedent(f""" Member: {member.mention} (`{member.id}`) Actor: {ctx.message.author} - Reason: {reason} + Reason: {textwrap.shorten(reason, width=1500, placeholder="...")} Expires: {expiry_str} Old nickname: `{old_nick}` New nickname: `{forced_nick}` -- cgit v1.2.3 From 27e15c42e71f3d2df828d13a5e92c53c664b4431 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 09:20:04 +0300 Subject: (Scheduler): Changed reason truncating in `apply_infraction` from 1900 chars to 1500, added shortening to end message too. --- bot/cogs/moderation/scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 7404ec8ac..345f08f19 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -85,7 +85,7 @@ class InfractionScheduler(Scheduler): infr_type = infraction["type"] icon = utils.INFRACTION_ICONS[infr_type][0] # Truncate reason when it's too long to avoid raising error on sending ModLog entry - reason = textwrap.shorten(infraction["reason"], width=1900, placeholder="...") + reason = textwrap.shorten(infraction["reason"], width=1500, placeholder="...") expiry = time.format_infraction_with_duration(infraction["expires_at"]) id_ = infraction['id'] @@ -128,7 +128,7 @@ class InfractionScheduler(Scheduler): f"Infraction #{id_} actor is bot; including the reason in the confirmation message." ) - end_msg = f" (reason: {infraction['reason']})" + end_msg = f" (reason: {textwrap.shorten(infraction['reason'], width=1500, placeholder='...')})" elif ctx.channel.id not in STAFF_CHANNELS: log.trace( f"Infraction #{id_} context is not in a staff channel; omitting infraction count." -- cgit v1.2.3 From 1100dba71b789bdc35c6c42d1b4003c7d28dcbb0 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 15:10:40 +0300 Subject: (ModLog): Added mod log item embed description truncating when it's too long. --- bot/cogs/moderation/modlog.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index c63b4bab9..e15a80c6d 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -2,6 +2,7 @@ import asyncio import difflib import itertools import logging +import textwrap import typing as t from datetime import datetime from itertools import zip_longest @@ -98,7 +99,7 @@ class ModLog(Cog, name="ModLog"): footer: t.Optional[str] = None, ) -> Context: """Generate log embed and send to logging channel.""" - embed = discord.Embed(description=text) + embed = discord.Embed(description=textwrap.shorten(text, width=2048, placeholder="...")) if title and icon_url: embed.set_author(name=title, icon_url=icon_url) -- cgit v1.2.3 From b765ecb280a3ba0ca017350a4c69dc9c07f97a67 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 15:12:56 +0300 Subject: (Scheduler): Removed reason truncation from `apply_infraction`, changed order of ModLog embed description item in same function. --- bot/cogs/moderation/scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 345f08f19..fbb2d457b 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -85,7 +85,7 @@ class InfractionScheduler(Scheduler): infr_type = infraction["type"] icon = utils.INFRACTION_ICONS[infr_type][0] # Truncate reason when it's too long to avoid raising error on sending ModLog entry - reason = textwrap.shorten(infraction["reason"], width=1500, placeholder="...") + reason = infraction["reason"] expiry = time.format_infraction_with_duration(infraction["expires_at"]) id_ = infraction['id'] @@ -182,8 +182,8 @@ class InfractionScheduler(Scheduler): text=textwrap.dedent(f""" Member: {user.mention} (`{user.id}`) Actor: {ctx.message.author}{dm_log_text} - Reason: {reason} {expiry_log_text} + Reason: {reason} """), content=log_content, footer=f"ID {infraction['id']}" -- cgit v1.2.3 From 03028eab84a09c047c0ef879fc06fccacbe30420 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 15:18:38 +0300 Subject: (Mod Utils): Removed truncation of reason itself and added truncation to whole embed in `notify_infraction`. --- bot/cogs/moderation/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py index 9811d059f..0423e5373 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/cogs/moderation/utils.py @@ -132,11 +132,11 @@ async def notify_infraction( log.trace(f"Sending {user} a DM about their {infr_type} infraction.") embed = discord.Embed( - description=textwrap.dedent(f""" + description=textwrap.shorten(textwrap.dedent(f""" **Type:** {infr_type.capitalize()} **Expires:** {expires_at or "N/A"} - **Reason:** {textwrap.shorten(reason, width=1500, placeholder="...") or "No reason provided."} - """), + **Reason:** {reason or "No reason provided."} + """), width=2048, placeholder="..."), colour=Colours.soft_red ) -- cgit v1.2.3 From 36f947c1cb52faedc1d2e00869e109f2cde12c11 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 15:19:44 +0300 Subject: (Superstarify): Removed unnecessary truncation on `superstarify` command, reordered ModLog text. --- bot/cogs/moderation/superstarify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/superstarify.py b/bot/cogs/moderation/superstarify.py index d77e61e6b..e221ad909 100644 --- a/bot/cogs/moderation/superstarify.py +++ b/bot/cogs/moderation/superstarify.py @@ -183,10 +183,10 @@ class Superstarify(InfractionScheduler, Cog): text=textwrap.dedent(f""" Member: {member.mention} (`{member.id}`) Actor: {ctx.message.author} - Reason: {textwrap.shorten(reason, width=1500, placeholder="...")} Expires: {expiry_str} Old nickname: `{old_nick}` New nickname: `{forced_nick}` + Reason: {reason} """), footer=f"ID {id_}" ) -- cgit v1.2.3 From 8d283fb8eefd7be288702f88653aaebcdcda37c3 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 08:12:09 +0300 Subject: (Mod Utils): Moved embed description to variable. --- bot/cogs/moderation/utils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py index 0423e5373..fc8c26031 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/cogs/moderation/utils.py @@ -131,12 +131,14 @@ async def notify_infraction( """DM a user about their new infraction and return True if the DM is successful.""" log.trace(f"Sending {user} a DM about their {infr_type} infraction.") + text = textwrap.dedent(f""" + **Type:** {infr_type.capitalize()} + **Expires:** {expires_at or "N/A"} + **Reason:** {reason or "No reason provided."} + """) + embed = discord.Embed( - description=textwrap.shorten(textwrap.dedent(f""" - **Type:** {infr_type.capitalize()} - **Expires:** {expires_at or "N/A"} - **Reason:** {reason or "No reason provided."} - """), width=2048, placeholder="..."), + description=textwrap.shorten(text, width=2048, placeholder="..."), colour=Colours.soft_red ) -- cgit v1.2.3 From b148beeec8c897d91fa100d0bbd1cb4965f58e6e Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 08:27:10 +0300 Subject: (Scheduler): Move reason to end of log text to avoid truncating keys. --- bot/cogs/moderation/scheduler.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index fbb2d457b..3352806e7 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -283,6 +283,9 @@ class InfractionScheduler(Scheduler): f"{log_text.get('Failure', '')}" ) + # Move reason to end of entry to avoid cutting out some keys + log_text["Reason"] = log_text.pop("Reason") + # Send a log message to the mod log. await self.mod_log.send_log_message( icon_url=utils.INFRACTION_ICONS[infr_type][1], @@ -326,7 +329,7 @@ class InfractionScheduler(Scheduler): log_text = { "Member": f"<@{user_id}>", "Actor": str(self.bot.get_user(actor) or actor), - "Reason": textwrap.shorten(infraction["reason"], width=1500, placeholder="..."), + "Reason": infraction["reason"], "Created": created, } @@ -396,6 +399,9 @@ class InfractionScheduler(Scheduler): user = self.bot.get_user(user_id) avatar = user.avatar_url_as(static_format="png") if user else None + # Move reason to end so when reason is too long, this is not gonna cut out required items. + log_text["Reason"] = log_text.pop("Reason") + log.trace(f"Sending deactivation mod log for infraction #{id_}.") await self.mod_log.send_log_message( icon_url=utils.INFRACTION_ICONS[type_][1], @@ -405,7 +411,6 @@ class InfractionScheduler(Scheduler): text="\n".join(f"{k}: {v}" for k, v in log_text.items()), footer=f"ID: {id_}", content=log_content, - ) return log_text -- cgit v1.2.3 From 816e76fe8c5e5231cdc85ab974294c1d2fb4a87c Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 08:28:44 +0300 Subject: (Scheduler): Replaced `infraction['reason']` with `reason` variable using in `end_msg`. --- bot/cogs/moderation/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 3352806e7..b238cf4e2 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -128,7 +128,7 @@ class InfractionScheduler(Scheduler): f"Infraction #{id_} actor is bot; including the reason in the confirmation message." ) - end_msg = f" (reason: {textwrap.shorten(infraction['reason'], width=1500, placeholder='...')})" + end_msg = f" (reason: {textwrap.shorten(reason, width=1500, placeholder='...')})" elif ctx.channel.id not in STAFF_CHANNELS: log.trace( f"Infraction #{id_} context is not in a staff channel; omitting infraction count." -- cgit v1.2.3 From 3bbd10ac91e9677e24588734d256a0558c0b46a2 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 09:52:19 +0300 Subject: (Talent Pool): Applied reason shortening. --- bot/cogs/watchchannels/talentpool.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/cogs/watchchannels/talentpool.py b/bot/cogs/watchchannels/talentpool.py index ad0c51fa6..15af7e34d 100644 --- a/bot/cogs/watchchannels/talentpool.py +++ b/bot/cogs/watchchannels/talentpool.py @@ -106,8 +106,8 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): if history: total = f"({len(history)} previous nominations in total)" - start_reason = f"Watched: {history[0]['reason']}" - end_reason = f"Unwatched: {history[0]['end_reason']}" + start_reason = f"Watched: {textwrap.shorten(history[0]['reason'], width=500, placeholder='...')}" + end_reason = f"Unwatched: {textwrap.shorten(history[0]['end_reason'], width=500, placeholder='...')}" msg += f"\n\nUser's previous watch reasons {total}:```{start_reason}\n\n{end_reason}```" await ctx.send(msg) @@ -224,7 +224,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): Status: **Active** Date: {start_date} Actor: {actor.mention if actor else actor_id} - Reason: {nomination_object["reason"]} + Reason: {textwrap.shorten(nomination_object["reason"], width=200, placeholder="...")} Nomination ID: `{nomination_object["id"]}` =============== """ @@ -237,10 +237,10 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): Status: Inactive Date: {start_date} Actor: {actor.mention if actor else actor_id} - Reason: {nomination_object["reason"]} + Reason: {textwrap.shorten(nomination_object["reason"], width=200, placeholder="...")} End date: {end_date} - Unwatch reason: {nomination_object["end_reason"]} + Unwatch reason: {textwrap.shorten(nomination_object["end_reason"], width=200, placeholder="...")} Nomination ID: `{nomination_object["id"]}` =============== """ -- cgit v1.2.3 From 2c9bc9f6fe5174096a6177560acd91f869c296ef Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 11:30:51 +0300 Subject: (Watchchannel): Added footer shortening. --- bot/cogs/watchchannels/watchchannel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/watchchannels/watchchannel.py b/bot/cogs/watchchannels/watchchannel.py index 479820444..ac1aa38ee 100644 --- a/bot/cogs/watchchannels/watchchannel.py +++ b/bot/cogs/watchchannels/watchchannel.py @@ -280,8 +280,9 @@ class WatchChannel(metaclass=CogABCMeta): else: message_jump = f"in [#{msg.channel.name}]({msg.jump_url})" + footer = f"Added {time_delta} by {actor} | Reason: {reason}" embed = Embed(description=f"{msg.author.mention} {message_jump}") - embed.set_footer(text=f"Added {time_delta} by {actor} | Reason: {reason}") + embed.set_footer(text=textwrap.shorten(footer, width=128, placeholder="...")) await self.webhook_send(embed=embed, username=msg.author.display_name, avatar_url=msg.author.avatar_url) -- cgit v1.2.3 From 472b58b3fb1b3d8695d9de1a13ce92f129e6bcc4 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 11:34:54 +0300 Subject: (Big Brother): Added truncating reason. --- bot/cogs/watchchannels/bigbrother.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bot/cogs/watchchannels/bigbrother.py b/bot/cogs/watchchannels/bigbrother.py index 903c87f85..69df849f0 100644 --- a/bot/cogs/watchchannels/bigbrother.py +++ b/bot/cogs/watchchannels/bigbrother.py @@ -1,4 +1,5 @@ import logging +import textwrap from collections import ChainMap from discord.ext.commands import Cog, Context, group @@ -97,8 +98,8 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): if len(history) > 1: total = f"({len(history) // 2} previous infractions in total)" - end_reason = history[0]["reason"] - start_reason = f"Watched: {history[1]['reason']}" + end_reason = textwrap.shorten(history[0]["reason"], width=500, placeholder="...") + start_reason = f"Watched: {textwrap.shorten(history[1]['reason'], width=500, placeholder='...')}" msg += f"\n\nUser's previous watch reasons {total}:```{start_reason}\n\n{end_reason}```" else: msg = ":x: Failed to post the infraction: response was empty." -- cgit v1.2.3 From 7195a928952c1550a82d27706a57165606aa5f4a Mon Sep 17 00:00:00 2001 From: rohanjnr Date: Tue, 14 Apr 2020 18:57:39 +0530 Subject: Feature now is cross-platform Instead of using string methods to split the file path at `/` which is not cross-platform, I am now entirely using pathlib methods to get the parent folder and restrict the tags. --- bot/cogs/tags.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index bb74ab1ca..a59d28600 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -38,7 +38,7 @@ class Tags(Cog): cache = {} tag_files = Path("bot", "resources", "tags").glob("**/*") for file in tag_files: - file_path = str(file).split("/") + if file.is_file(): tag_title = file.stem tag = { @@ -48,8 +48,9 @@ class Tags(Cog): }, "restricted_to": "developers" } - if len(file_path) == 5: - tag["restricted_to"] = file_path[3] + parent_folder = file.parent.stem + if parent_folder != "tags": + tag["restricted_to"] = parent_folder cache[tag_title] = tag return cache @@ -212,11 +213,13 @@ class Tags(Cog): return if tag_name is not None: - founds = self._get_tag(tag_name) + temp_founds = self._get_tag(tag_name) + + founds = [] - for found_tag in founds: - if not self.check_accessibility(ctx.author, found_tag): - founds.remove(found_tag) + for found_tag in temp_founds: + if self.check_accessibility(ctx.author, found_tag): + founds.append(found_tag) if len(founds) == 1: tag = founds[0] -- cgit v1.2.3 From 39bac8873120801eb51a2c1a996d2760d9af64a4 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 14 Apr 2020 16:32:43 +0300 Subject: (ModLog): Applied force embed description truncating in `send_log_message` to avoid removing newlines. --- bot/cogs/moderation/modlog.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index e15a80c6d..fcc9d4e0a 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -99,7 +99,10 @@ class ModLog(Cog, name="ModLog"): footer: t.Optional[str] = None, ) -> Context: """Generate log embed and send to logging channel.""" - embed = discord.Embed(description=textwrap.shorten(text, width=2048, placeholder="...")) + # Truncate string directly here to avoid removing newlines + embed = discord.Embed( + description=text[:2046] + "..." if len(text) > 2048 else text + ) if title and icon_url: embed.set_author(name=title, icon_url=icon_url) -- cgit v1.2.3 From 0b719040e8eb52fd030322b720de5f5ffbcc9919 Mon Sep 17 00:00:00 2001 From: Rohan Reddy Alleti Date: Tue, 14 Apr 2020 19:02:58 +0530 Subject: simplify if statement Co-Authored-By: Mark --- bot/cogs/tags.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index a59d28600..79bea3b63 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -127,10 +127,8 @@ class Tags(Cog): matching_tags = [] for tag in self._cache.values(): - if ( - self.check_accessibility(user, tag) - and check(query in tag['embed']['description'].casefold() for query in keywords_processed) - ): + matches = (query in tag['embed']['description'].casefold() for query in keywords_processed) + if self.check_accessibility(user, tag) and check(matches): matching_tags.append(tag) return matching_tags -- cgit v1.2.3 From 8bf1df9438ecf456b02725e3f9689c5bd885b2a7 Mon Sep 17 00:00:00 2001 From: Rohan Reddy Alleti Date: Tue, 14 Apr 2020 19:03:38 +0530 Subject: simpl Co-Authored-By: Mark --- bot/cogs/tags.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index 79bea3b63..e5492971d 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -58,9 +58,7 @@ class Tags(Cog): @staticmethod def check_accessibility(user: Member, tag: dict) -> bool: """Check if user can access a tag.""" - if tag["restricted_to"].lower() in [role.name.lower() for role in user.roles]: - return True - return False + return tag["restricted_to"].lower() in [role.name.lower() for role in user.roles] @staticmethod def _fuzzy_search(search: str, target: str) -> float: -- cgit v1.2.3 From 6928c492c1d989567c47dfd49a396730c6c8bb27 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 14 Apr 2020 18:29:55 +0300 Subject: (Scheduler): Removed empty line when expiration not specified in `apply_infraction`. --- bot/cogs/moderation/scheduler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index b238cf4e2..5b59b4d4b 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -102,7 +102,7 @@ class InfractionScheduler(Scheduler): dm_result = "" dm_log_text = "" - expiry_log_text = f"Expires: {expiry}" if expiry else "" + expiry_log_text = f"\nExpires: {expiry}" if expiry else "" log_title = "applied" log_content = None @@ -181,8 +181,7 @@ class InfractionScheduler(Scheduler): thumbnail=user.avatar_url_as(static_format="png"), text=textwrap.dedent(f""" Member: {user.mention} (`{user.id}`) - Actor: {ctx.message.author}{dm_log_text} - {expiry_log_text} + Actor: {ctx.message.author}{dm_log_text} {expiry_log_text} Reason: {reason} """), content=log_content, -- cgit v1.2.3 From 32a5bf97e31addb32d17bd0479bc4cf2f4dd9eb7 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 14 Apr 2020 18:50:24 +0300 Subject: (Scheduler): Added removal of infraction in DB, when applying infraction fail. Also don't send DM in this case. --- bot/cogs/moderation/scheduler.py | 46 ++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 5b59b4d4b..58e363da6 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -105,23 +105,7 @@ class InfractionScheduler(Scheduler): expiry_log_text = f"\nExpires: {expiry}" if expiry else "" log_title = "applied" log_content = None - - # DM the user about the infraction if it's not a shadow/hidden infraction. - if not infraction["hidden"]: - dm_result = f"{constants.Emojis.failmail} " - dm_log_text = "\nDM: **Failed**" - - # Sometimes user is a discord.Object; make it a proper user. - try: - if not isinstance(user, (discord.Member, discord.User)): - user = await self.bot.fetch_user(user.id) - except discord.HTTPException as e: - log.error(f"Failed to DM {user.id}: could not fetch user (status {e.status})") - else: - # Accordingly display whether the user was successfully notified via DM. - if await utils.notify_infraction(user, infr_type, expiry, reason, icon): - dm_result = ":incoming_envelope: " - dm_log_text = "\nDM: Sent" + failed = False if infraction["actor"] == self.bot.user.id: log.trace( @@ -165,11 +149,37 @@ class InfractionScheduler(Scheduler): log.warning(f"{log_msg}: bot lacks permissions.") else: log.exception(log_msg) + failed = True + + # DM the user about the infraction if it's not a shadow/hidden infraction. + # Don't send DM when applying failed. + if not infraction["hidden"] and not failed: + dm_result = f"{constants.Emojis.failmail} " + dm_log_text = "\nDM: **Failed**" + + # Sometimes user is a discord.Object; make it a proper user. + try: + if not isinstance(user, (discord.Member, discord.User)): + user = await self.bot.fetch_user(user.id) + except discord.HTTPException as e: + log.error(f"Failed to DM {user.id}: could not fetch user (status {e.status})") + else: + # Accordingly display whether the user was successfully notified via DM. + if await utils.notify_infraction(user, infr_type, expiry, reason, icon): + dm_result = ":incoming_envelope: " + dm_log_text = "\nDM: Sent" + + if failed: + dm_log_text = "\nDM: **Canceled**" + dm_result = f"{constants.Emojis.failmail} " + log.trace(f"Deleted infraction {infraction['id']} from database because applying infraction failed.") + await self.bot.api_client.delete(f"bot/infractions/{infraction['id']}") # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} confirmation message.") await ctx.send( - f"{dm_result}{confirm_msg} **{infr_type}** to {user.mention}{expiry_msg}{end_msg}." + f"{dm_result}{confirm_msg} " + f"{f'**{infr_type}** to {user.mention}{expiry_msg}{end_msg}' if not failed else ''}." ) # Send a log message to the mod log. -- cgit v1.2.3 From 085decd12867f89a0803806928741fe6dd3c76bb Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 15 Apr 2020 08:18:19 +0300 Subject: (Test Helpers): Added `__ge__` function to `MockRole` for comparing. --- tests/helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/helpers.py b/tests/helpers.py index 8e13f0f28..227bac95f 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -205,6 +205,10 @@ class MockRole(CustomMockMixin, unittest.mock.Mock, ColourMixin, HashableMixin): """Simplified position-based comparisons similar to those of `discord.Role`.""" return self.position < other.position + def __ge__(self, other): + """Simplified position-based comparisons similar to those of `discord.Role`.""" + return self.position >= other.position + # Create a Member instance to get a realistic Mock of `discord.Member` member_data = {'user': 'lemon', 'roles': [1]} -- cgit v1.2.3 From 81f6efc2f4e9e157e2f7fb9f191ea410af066632 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 16 Apr 2020 11:15:16 +0300 Subject: (Infraction Tests): Created reason shortening tests for ban and kick. --- tests/bot/cogs/moderation/test_infractions.py | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/bot/cogs/moderation/test_infractions.py diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py new file mode 100644 index 000000000..39ea93952 --- /dev/null +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -0,0 +1,54 @@ +import textwrap +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from bot.cogs.moderation.infractions import Infractions +from tests.helpers import MockBot, MockContext, MockGuild, MockMember, MockRole + + +class ShorteningTests(unittest.IsolatedAsyncioTestCase): + """Tests for ban and kick command reason shortening.""" + + def setUp(self): + self.bot = MockBot() + self.cog = Infractions(self.bot) + self.user = MockMember(id=1234, top_role=MockRole(id=3577, position=10)) + self.target = MockMember(id=1265, top_role=MockRole(id=9876, position=0)) + self.guild = MockGuild(id=4567) + self.ctx = MockContext(bot=self.bot, author=self.user, guild=self.guild) + + @patch("bot.cogs.moderation.utils.has_active_infraction") + @patch("bot.cogs.moderation.utils.post_infraction") + async def test_apply_ban_reason_shortening(self, post_infraction_mock, has_active_mock): + """Should truncate reason for `ctx.guild.ban`.""" + has_active_mock.return_value = False + post_infraction_mock.return_value = {"foo": "bar"} + + self.cog.apply_infraction = AsyncMock() + self.bot.get_cog.return_value = AsyncMock() + self.cog.mod_log.ignore = Mock() + + await self.cog.apply_ban(self.ctx, self.target, "foo bar" * 3000) + ban = self.cog.apply_infraction.call_args[0][3] + self.assertEqual( + ban.cr_frame.f_locals["kwargs"]["reason"], + textwrap.shorten("foo bar" * 3000, 512, placeholder=" ...") + ) + # Await ban to avoid warning + await ban + + @patch("bot.cogs.moderation.utils.post_infraction") + async def test_apply_kick_reason_shortening(self, post_infraction_mock) -> None: + """Should truncate reason for `Member.kick`.""" + post_infraction_mock.return_value = {"foo": "bar"} + + self.cog.apply_infraction = AsyncMock() + self.cog.mod_log.ignore = Mock() + + await self.cog.apply_kick(self.ctx, self.target, "foo bar" * 3000) + kick = self.cog.apply_infraction.call_args[0][3] + self.assertEqual( + kick.cr_frame.f_locals["kwargs"]["reason"], + textwrap.shorten("foo bar" * 3000, 512, placeholder="...") + ) + await kick -- cgit v1.2.3 From 216953044a870f2440fe44fcd2f9ca3ee7cf37e9 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 16 Apr 2020 11:30:09 +0300 Subject: (ModLog Tests): Created reason shortening tests for `send_log_message`. --- tests/bot/cogs/moderation/test_modlog.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tests/bot/cogs/moderation/test_modlog.py diff --git a/tests/bot/cogs/moderation/test_modlog.py b/tests/bot/cogs/moderation/test_modlog.py new file mode 100644 index 000000000..46e01d2ea --- /dev/null +++ b/tests/bot/cogs/moderation/test_modlog.py @@ -0,0 +1,29 @@ +import unittest + +import discord + +from bot.cogs.moderation.modlog import ModLog +from tests.helpers import MockBot, MockTextChannel + + +class ModLogTests(unittest.IsolatedAsyncioTestCase): + """Tests for moderation logs.""" + + def setUp(self): + self.bot = MockBot() + self.cog = ModLog(self.bot) + self.channel = MockTextChannel() + + async def test_log_entry_description_shortening(self): + """Should truncate embed description for ModLog entry.""" + self.bot.get_channel.return_value = self.channel + await self.cog.send_log_message( + icon_url="foo", + colour=discord.Colour.blue(), + title="bar", + text="foo bar" * 3000 + ) + embed = self.channel.send.call_args[1]["embed"] + self.assertEqual( + embed.description, ("foo bar" * 3000)[:2046] + "..." + ) -- cgit v1.2.3 From b463cb2d21683b7184698f788419d325e2f5f5cc Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 16 Apr 2020 11:33:21 +0300 Subject: (ModLog): Removed unused `textwrap` import. --- bot/cogs/moderation/modlog.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index fcc9d4e0a..eb8bd65cf 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -2,7 +2,6 @@ import asyncio import difflib import itertools import logging -import textwrap import typing as t from datetime import datetime from itertools import zip_longest -- cgit v1.2.3 From 1a3fa6a395141c4fcdd1d388d6ce3e7bd89bcbf0 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 16 Apr 2020 13:40:47 +0300 Subject: (Infractions and ModLog Tests): Replaced `shortening` with `truncation`, removed unnecessary type hint and added comment to kick truncation test about awaiting `kick`. --- tests/bot/cogs/moderation/test_infractions.py | 9 +++++---- tests/bot/cogs/moderation/test_modlog.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 39ea93952..51a8cc645 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -6,8 +6,8 @@ from bot.cogs.moderation.infractions import Infractions from tests.helpers import MockBot, MockContext, MockGuild, MockMember, MockRole -class ShorteningTests(unittest.IsolatedAsyncioTestCase): - """Tests for ban and kick command reason shortening.""" +class TruncationTests(unittest.IsolatedAsyncioTestCase): + """Tests for ban and kick command reason truncation.""" def setUp(self): self.bot = MockBot() @@ -19,7 +19,7 @@ class ShorteningTests(unittest.IsolatedAsyncioTestCase): @patch("bot.cogs.moderation.utils.has_active_infraction") @patch("bot.cogs.moderation.utils.post_infraction") - async def test_apply_ban_reason_shortening(self, post_infraction_mock, has_active_mock): + async def test_apply_ban_reason_truncation(self, post_infraction_mock, has_active_mock): """Should truncate reason for `ctx.guild.ban`.""" has_active_mock.return_value = False post_infraction_mock.return_value = {"foo": "bar"} @@ -38,7 +38,7 @@ class ShorteningTests(unittest.IsolatedAsyncioTestCase): await ban @patch("bot.cogs.moderation.utils.post_infraction") - async def test_apply_kick_reason_shortening(self, post_infraction_mock) -> None: + async def test_apply_kick_reason_truncation(self, post_infraction_mock): """Should truncate reason for `Member.kick`.""" post_infraction_mock.return_value = {"foo": "bar"} @@ -51,4 +51,5 @@ class ShorteningTests(unittest.IsolatedAsyncioTestCase): kick.cr_frame.f_locals["kwargs"]["reason"], textwrap.shorten("foo bar" * 3000, 512, placeholder="...") ) + # Await kick to avoid warning await kick diff --git a/tests/bot/cogs/moderation/test_modlog.py b/tests/bot/cogs/moderation/test_modlog.py index 46e01d2ea..d60836474 100644 --- a/tests/bot/cogs/moderation/test_modlog.py +++ b/tests/bot/cogs/moderation/test_modlog.py @@ -14,7 +14,7 @@ class ModLogTests(unittest.IsolatedAsyncioTestCase): self.cog = ModLog(self.bot) self.channel = MockTextChannel() - async def test_log_entry_description_shortening(self): + async def test_log_entry_description_truncation(self): """Should truncate embed description for ModLog entry.""" self.bot.get_channel.return_value = self.channel await self.cog.send_log_message( -- cgit v1.2.3 From ce0ec7db55a9f12f55ac5ba019a95bc0b7d5cbd7 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 18 Apr 2020 21:00:38 -0700 Subject: Tags: always use top-most folder for role restrictions Ensures that nested directories aren't used as the value for the role name. --- bot/cogs/tags.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index e5492971d..1c124b25a 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -34,25 +34,28 @@ class Tags(Cog): @staticmethod def get_tags() -> dict: """Get all tags.""" - # Save all tags in memory. cache = {} - tag_files = Path("bot", "resources", "tags").glob("**/*") - for file in tag_files: + base_path = Path("bot", "resources", "tags") + for file in base_path.glob("**/*"): if file.is_file(): tag_title = file.stem tag = { "title": tag_title, "embed": { - "description": file.read_text() + "description": file.read_text(), }, - "restricted_to": "developers" + "restricted_to": "developers", } - parent_folder = file.parent.stem - if parent_folder != "tags": - tag["restricted_to"] = parent_folder + + # Convert to a list to allow negative indexing. + parents = list(file.relative_to(base_path).parents) + if len(parents) > 1: + # -1 would be '.' hence -2 is used as the index. + tag["restricted_to"] = parents[-2].name cache[tag_title] = tag + return cache @staticmethod -- cgit v1.2.3 From 3527b585fa407f7ef72af33eda2997334170075f Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 25 Apr 2020 14:40:45 -0700 Subject: Converters: handle ValueError when year for duration is out of range `datetime` objects only support a year up to 9999. Fixes #906 --- bot/converters.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/converters.py b/bot/converters.py index 72c46fdf0..4deb59f87 100644 --- a/bot/converters.py +++ b/bot/converters.py @@ -217,7 +217,10 @@ class Duration(Converter): delta = relativedelta(**duration_dict) now = datetime.utcnow() - return now + delta + try: + return now + delta + except ValueError: + raise BadArgument(f"`{duration}` results in a datetime outside the supported range.") class ISODateTime(Converter): -- cgit v1.2.3 From 96920935f9af6d325a2ff91d197285204b3221c9 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 28 Apr 2020 15:56:15 -0700 Subject: Test for out of range datetime in the Duration converter --- tests/bot/test_converters.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/bot/test_converters.py b/tests/bot/test_converters.py index ca8cb6825..e42bfc7ee 100644 --- a/tests/bot/test_converters.py +++ b/tests/bot/test_converters.py @@ -198,6 +198,17 @@ class ConverterTests(unittest.TestCase): with self.assertRaises(BadArgument, msg=exception_message): asyncio.run(converter.convert(self.context, invalid_duration)) + @patch("bot.converters.datetime") + def test_duration_converter_out_of_range(self, mock_datetime): + """Duration converter should raise BadArgument if datetime raises a ValueError.""" + mock_datetime.__add__.side_effect = ValueError + mock_datetime.utcnow.return_value = mock_datetime + + duration = f"{datetime.MAXYEAR}y" + exception_message = f"`{duration}` results in a datetime outside the supported range." + with self.assertRaisesRegex(BadArgument, exception_message): + asyncio.run(Duration().convert(self.context, duration)) + def test_isodatetime_converter_for_valid(self): """ISODateTime converter returns correct datetime for valid datetime string.""" test_values = ( -- cgit v1.2.3 From 298389f57166fb5c775e550175c8bb2685fa37ae Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 28 Apr 2020 15:58:29 -0700 Subject: Remove redundant parenthesis from test values --- tests/bot/test_converters.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/bot/test_converters.py b/tests/bot/test_converters.py index e42bfc7ee..51d7affba 100644 --- a/tests/bot/test_converters.py +++ b/tests/bot/test_converters.py @@ -166,28 +166,28 @@ class ConverterTests(unittest.TestCase): """Duration raises the right exception for invalid duration strings.""" test_values = ( # Units in wrong order - ('1d1w'), - ('1s1y'), + '1d1w', + '1s1y', # Duplicated units - ('1 year 2 years'), - ('1 M 10 minutes'), + '1 year 2 years', + '1 M 10 minutes', # Unknown substrings - ('1MVes'), - ('1y3breads'), + '1MVes', + '1y3breads', # Missing amount - ('ym'), + 'ym', # Incorrect whitespace - (" 1y"), - ("1S "), - ("1y 1m"), + " 1y", + "1S ", + "1y 1m", # Garbage - ('Guido van Rossum'), - ('lemon lemon lemon lemon lemon lemon lemon'), + 'Guido van Rossum', + 'lemon lemon lemon lemon lemon lemon lemon', ) converter = Duration() @@ -262,19 +262,19 @@ class ConverterTests(unittest.TestCase): """ISODateTime converter raises the correct exception for invalid datetime strings.""" test_values = ( # Make sure it doesn't interfere with the Duration converter - ('1Y'), - ('1d'), - ('1H'), + '1Y', + '1d', + '1H', # Check if it fails when only providing the optional time part - ('10:10:10'), - ('10:00'), + '10:10:10', + '10:00', # Invalid date format - ('19-01-01'), + '19-01-01', # Other non-valid strings - ('fisk the tag master'), + 'fisk the tag master', ) converter = ISODateTime() -- cgit v1.2.3 From 837bc230976328df8dabdc6e8be90188b2ff2ff3 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 28 Apr 2020 16:01:22 -0700 Subject: Use await instead of asyncio.run in converter tests --- tests/bot/test_converters.py | 55 ++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/tests/bot/test_converters.py b/tests/bot/test_converters.py index 51d7affba..146a8b5fa 100644 --- a/tests/bot/test_converters.py +++ b/tests/bot/test_converters.py @@ -1,4 +1,3 @@ -import asyncio import datetime import unittest from unittest.mock import MagicMock, patch @@ -16,7 +15,7 @@ from bot.converters import ( ) -class ConverterTests(unittest.TestCase): +class ConverterTests(unittest.IsolatedAsyncioTestCase): """Tests our custom argument converters.""" @classmethod @@ -26,7 +25,7 @@ class ConverterTests(unittest.TestCase): cls.fixed_utc_now = datetime.datetime.fromisoformat('2019-01-01T00:00:00') - def test_tag_content_converter_for_valid(self): + async def test_tag_content_converter_for_valid(self): """TagContentConverter should return correct values for valid input.""" test_values = ( ('hello', 'hello'), @@ -35,10 +34,10 @@ class ConverterTests(unittest.TestCase): for content, expected_conversion in test_values: with self.subTest(content=content, expected_conversion=expected_conversion): - conversion = asyncio.run(TagContentConverter.convert(self.context, content)) + conversion = await TagContentConverter.convert(self.context, content) self.assertEqual(conversion, expected_conversion) - def test_tag_content_converter_for_invalid(self): + async def test_tag_content_converter_for_invalid(self): """TagContentConverter should raise the proper exception for invalid input.""" test_values = ( ('', "Tag contents should not be empty, or filled with whitespace."), @@ -48,9 +47,9 @@ class ConverterTests(unittest.TestCase): for value, exception_message in test_values: with self.subTest(tag_content=value, exception_message=exception_message): with self.assertRaises(BadArgument, msg=exception_message): - asyncio.run(TagContentConverter.convert(self.context, value)) + await TagContentConverter.convert(self.context, value) - def test_tag_name_converter_for_valid(self): + async def test_tag_name_converter_for_valid(self): """TagNameConverter should return the correct values for valid tag names.""" test_values = ( ('tracebacks', 'tracebacks'), @@ -60,10 +59,10 @@ class ConverterTests(unittest.TestCase): for name, expected_conversion in test_values: with self.subTest(name=name, expected_conversion=expected_conversion): - conversion = asyncio.run(TagNameConverter.convert(self.context, name)) + conversion = await TagNameConverter.convert(self.context, name) self.assertEqual(conversion, expected_conversion) - def test_tag_name_converter_for_invalid(self): + async def test_tag_name_converter_for_invalid(self): """TagNameConverter should raise the correct exception for invalid tag names.""" test_values = ( ('👋', "Don't be ridiculous, you can't use that character!"), @@ -76,18 +75,18 @@ class ConverterTests(unittest.TestCase): for invalid_name, exception_message in test_values: with self.subTest(invalid_name=invalid_name, exception_message=exception_message): with self.assertRaises(BadArgument, msg=exception_message): - asyncio.run(TagNameConverter.convert(self.context, invalid_name)) + await TagNameConverter.convert(self.context, invalid_name) - def test_valid_python_identifier_for_valid(self): + async def test_valid_python_identifier_for_valid(self): """ValidPythonIdentifier returns valid identifiers unchanged.""" test_values = ('foo', 'lemon') for name in test_values: with self.subTest(identifier=name): - conversion = asyncio.run(ValidPythonIdentifier.convert(self.context, name)) + conversion = await ValidPythonIdentifier.convert(self.context, name) self.assertEqual(name, conversion) - def test_valid_python_identifier_for_invalid(self): + async def test_valid_python_identifier_for_invalid(self): """ValidPythonIdentifier raises the proper exception for invalid identifiers.""" test_values = ('nested.stuff', '#####') @@ -95,9 +94,9 @@ class ConverterTests(unittest.TestCase): with self.subTest(identifier=name): exception_message = f'`{name}` is not a valid Python identifier' with self.assertRaises(BadArgument, msg=exception_message): - asyncio.run(ValidPythonIdentifier.convert(self.context, name)) + await ValidPythonIdentifier.convert(self.context, name) - def test_duration_converter_for_valid(self): + async def test_duration_converter_for_valid(self): """Duration returns the correct `datetime` for valid duration strings.""" test_values = ( # Simple duration strings @@ -159,10 +158,10 @@ class ConverterTests(unittest.TestCase): mock_datetime.utcnow.return_value = self.fixed_utc_now with self.subTest(duration=duration, duration_dict=duration_dict): - converted_datetime = asyncio.run(converter.convert(self.context, duration)) + converted_datetime = await converter.convert(self.context, duration) self.assertEqual(converted_datetime, expected_datetime) - def test_duration_converter_for_invalid(self): + async def test_duration_converter_for_invalid(self): """Duration raises the right exception for invalid duration strings.""" test_values = ( # Units in wrong order @@ -196,10 +195,10 @@ class ConverterTests(unittest.TestCase): with self.subTest(invalid_duration=invalid_duration): exception_message = f'`{invalid_duration}` is not a valid duration string.' with self.assertRaises(BadArgument, msg=exception_message): - asyncio.run(converter.convert(self.context, invalid_duration)) + await converter.convert(self.context, invalid_duration) @patch("bot.converters.datetime") - def test_duration_converter_out_of_range(self, mock_datetime): + async def test_duration_converter_out_of_range(self, mock_datetime): """Duration converter should raise BadArgument if datetime raises a ValueError.""" mock_datetime.__add__.side_effect = ValueError mock_datetime.utcnow.return_value = mock_datetime @@ -207,9 +206,9 @@ class ConverterTests(unittest.TestCase): duration = f"{datetime.MAXYEAR}y" exception_message = f"`{duration}` results in a datetime outside the supported range." with self.assertRaisesRegex(BadArgument, exception_message): - asyncio.run(Duration().convert(self.context, duration)) + await Duration().convert(self.context, duration) - def test_isodatetime_converter_for_valid(self): + async def test_isodatetime_converter_for_valid(self): """ISODateTime converter returns correct datetime for valid datetime string.""" test_values = ( # `YYYY-mm-ddTHH:MM:SSZ` | `YYYY-mm-dd HH:MM:SSZ` @@ -254,11 +253,11 @@ class ConverterTests(unittest.TestCase): for datetime_string, expected_dt in test_values: with self.subTest(datetime_string=datetime_string, expected_dt=expected_dt): - converted_dt = asyncio.run(converter.convert(self.context, datetime_string)) + converted_dt = await converter.convert(self.context, datetime_string) self.assertIsNone(converted_dt.tzinfo) self.assertEqual(converted_dt, expected_dt) - def test_isodatetime_converter_for_invalid(self): + async def test_isodatetime_converter_for_invalid(self): """ISODateTime converter raises the correct exception for invalid datetime strings.""" test_values = ( # Make sure it doesn't interfere with the Duration converter @@ -282,9 +281,9 @@ class ConverterTests(unittest.TestCase): with self.subTest(datetime_string=datetime_string): exception_message = f"`{datetime_string}` is not a valid ISO-8601 datetime string" with self.assertRaises(BadArgument, msg=exception_message): - asyncio.run(converter.convert(self.context, datetime_string)) + await converter.convert(self.context, datetime_string) - def test_hush_duration_converter_for_valid(self): + async def test_hush_duration_converter_for_valid(self): """HushDurationConverter returns correct value for minutes duration or `"forever"` strings.""" test_values = ( ("0", 0), @@ -297,10 +296,10 @@ class ConverterTests(unittest.TestCase): converter = HushDurationConverter() for minutes_string, expected_minutes in test_values: with self.subTest(minutes_string=minutes_string, expected_minutes=expected_minutes): - converted = asyncio.run(converter.convert(self.context, minutes_string)) + converted = await converter.convert(self.context, minutes_string) self.assertEqual(expected_minutes, converted) - def test_hush_duration_converter_for_invalid(self): + async def test_hush_duration_converter_for_invalid(self): """HushDurationConverter raises correct exception for invalid minutes duration strings.""" test_values = ( ("16", "Duration must be at most 15 minutes."), @@ -311,4 +310,4 @@ class ConverterTests(unittest.TestCase): for invalid_minutes_string, exception_message in test_values: with self.subTest(invalid_minutes_string=invalid_minutes_string, exception_message=exception_message): with self.assertRaisesRegex(BadArgument, exception_message): - asyncio.run(converter.convert(self.context, invalid_minutes_string)) + await converter.convert(self.context, invalid_minutes_string) -- cgit v1.2.3 From 1e4766d9934396a72cc759649049b07e5814004a Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 28 Apr 2020 16:18:34 -0700 Subject: Fix exception message assertions in converter tests The `msg` arg is for displaying a message when the assertion fails. To match against the exception's message, `assertRaisesRegex` must be used. Since all of the messages are meant to be interpreted literally rather than as regex, `re.escape` is used. --- tests/bot/test_converters.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/bot/test_converters.py b/tests/bot/test_converters.py index 146a8b5fa..c42111f3f 100644 --- a/tests/bot/test_converters.py +++ b/tests/bot/test_converters.py @@ -1,4 +1,5 @@ import datetime +import re import unittest from unittest.mock import MagicMock, patch @@ -46,7 +47,7 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): for value, exception_message in test_values: with self.subTest(tag_content=value, exception_message=exception_message): - with self.assertRaises(BadArgument, msg=exception_message): + with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): await TagContentConverter.convert(self.context, value) async def test_tag_name_converter_for_valid(self): @@ -74,7 +75,7 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): for invalid_name, exception_message in test_values: with self.subTest(invalid_name=invalid_name, exception_message=exception_message): - with self.assertRaises(BadArgument, msg=exception_message): + with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): await TagNameConverter.convert(self.context, invalid_name) async def test_valid_python_identifier_for_valid(self): @@ -93,7 +94,7 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): for name in test_values: with self.subTest(identifier=name): exception_message = f'`{name}` is not a valid Python identifier' - with self.assertRaises(BadArgument, msg=exception_message): + with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): await ValidPythonIdentifier.convert(self.context, name) async def test_duration_converter_for_valid(self): @@ -194,7 +195,7 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): for invalid_duration in test_values: with self.subTest(invalid_duration=invalid_duration): exception_message = f'`{invalid_duration}` is not a valid duration string.' - with self.assertRaises(BadArgument, msg=exception_message): + with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): await converter.convert(self.context, invalid_duration) @patch("bot.converters.datetime") @@ -205,7 +206,7 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): duration = f"{datetime.MAXYEAR}y" exception_message = f"`{duration}` results in a datetime outside the supported range." - with self.assertRaisesRegex(BadArgument, exception_message): + with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): await Duration().convert(self.context, duration) async def test_isodatetime_converter_for_valid(self): @@ -280,7 +281,7 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): for datetime_string in test_values: with self.subTest(datetime_string=datetime_string): exception_message = f"`{datetime_string}` is not a valid ISO-8601 datetime string" - with self.assertRaises(BadArgument, msg=exception_message): + with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): await converter.convert(self.context, datetime_string) async def test_hush_duration_converter_for_valid(self): @@ -309,5 +310,5 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): converter = HushDurationConverter() for invalid_minutes_string, exception_message in test_values: with self.subTest(invalid_minutes_string=invalid_minutes_string, exception_message=exception_message): - with self.assertRaisesRegex(BadArgument, exception_message): + with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): await converter.convert(self.context, invalid_minutes_string) -- cgit v1.2.3 From 601ff03823deb842d74f4689fecb68f7ce1693e6 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 18:11:44 +0200 Subject: AntiMalware Tests - Added unittest for message without attachment --- tests/bot/cogs/test_antimalware.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/bot/cogs/test_antimalware.py diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py new file mode 100644 index 000000000..41ca19e17 --- /dev/null +++ b/tests/bot/cogs/test_antimalware.py @@ -0,0 +1,20 @@ +import asyncio +import unittest + +from bot.cogs import antimalware +from tests.helpers import MockBot, MockMessage + + +class AntiMalwareCogTests(unittest.TestCase): + """Test the AntiMalware cog.""" + + def setUp(self): + """Sets up fresh objects for each test.""" + self.bot = MockBot() + self.cog = antimalware.AntiMalware(self.bot) + self.message = MockMessage() + + def test_message_without_attachment(self): + """Messages without attachments should result in no action.""" + coroutine = self.cog.on_message(self.message) + self.assertIsNone(asyncio.run(coroutine)) -- cgit v1.2.3 From 9889f0fdd1ba403ae50ba20be38feca0932d1dda Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 19:06:46 +0200 Subject: AntiMalware Tests - Added unittests for deletion of message and ignoring of dms --- tests/bot/cogs/test_antimalware.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 41ca19e17..ebf3a1277 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,8 +1,9 @@ import asyncio import unittest +from unittest.mock import AsyncMock from bot.cogs import antimalware -from tests.helpers import MockBot, MockMessage +from tests.helpers import MockAttachment, MockBot, MockMessage class AntiMalwareCogTests(unittest.TestCase): @@ -13,8 +14,27 @@ class AntiMalwareCogTests(unittest.TestCase): self.bot = MockBot() self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() + self.message.delete = AsyncMock() def test_message_without_attachment(self): """Messages without attachments should result in no action.""" coroutine = self.cog.on_message(self.message) self.assertIsNone(asyncio.run(coroutine)) + self.message.delete.assert_not_called() + + def test_direct_message_with_attachment(self): + """Direct messages should have no action taken.""" + attachment = MockAttachment(filename="python.asdfsff") + self.message.attachments = [attachment] + self.message.guild = None + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + self.message.delete.assert_not_called() + + def test_message_with_illegal_extension_gets_deleted(self): + """A message containing an illegal extension should send an embed.""" + attachment = MockAttachment(filename="python.asdfsff") + self.message.attachments = [attachment] + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + self.message.delete.assert_called_once() -- cgit v1.2.3 From 90d2ce0e3717d4ddf79eb986e22f7542ca1770e1 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 19:13:45 +0200 Subject: AntiMalware Tests - Added unittest for messages send by staff --- tests/bot/cogs/test_antimalware.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index ebf3a1277..e3fd477fa 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -3,7 +3,8 @@ import unittest from unittest.mock import AsyncMock from bot.cogs import antimalware -from tests.helpers import MockAttachment, MockBot, MockMessage +from bot.constants import Roles +from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole class AntiMalwareCogTests(unittest.TestCase): @@ -38,3 +39,13 @@ class AntiMalwareCogTests(unittest.TestCase): coroutine = self.cog.on_message(self.message) asyncio.run(coroutine) self.message.delete.assert_called_once() + + def test_message_send_by_staff(self): + """A message send by a member of staff should be ignored.""" + moderator_role = MockRole(name="Moderator", id=Roles.moderators) + self.message.author.roles.append(moderator_role) + attachment = MockAttachment(filename="python.asdfsff") + self.message.attachments = [attachment] + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + self.message.delete.assert_not_called() -- cgit v1.2.3 From 3913a8eba46bf98bd09e13145da33f7a09f77960 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 19:45:57 +0200 Subject: AntiMalware Tests - Added unittest for the embed for a python file. --- tests/bot/cogs/test_antimalware.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index e3fd477fa..0bb5af943 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -3,7 +3,7 @@ import unittest from unittest.mock import AsyncMock from bot.cogs import antimalware -from bot.constants import Roles +from bot.constants import Roles, URLs from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole @@ -28,16 +28,20 @@ class AntiMalwareCogTests(unittest.TestCase): attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] self.message.guild = None + coroutine = self.cog.on_message(self.message) asyncio.run(coroutine) + self.message.delete.assert_not_called() def test_message_with_illegal_extension_gets_deleted(self): """A message containing an illegal extension should send an embed.""" attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] + coroutine = self.cog.on_message(self.message) asyncio.run(coroutine) + self.message.delete.assert_called_once() def test_message_send_by_staff(self): @@ -46,6 +50,25 @@ class AntiMalwareCogTests(unittest.TestCase): self.message.author.roles.append(moderator_role) attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] + coroutine = self.cog.on_message(self.message) asyncio.run(coroutine) + self.message.delete.assert_not_called() + + def test_python_file_redirect_embed(self): + """A message containing a .python file should result in an embed redirecting the user to our paste site""" + attachment = MockAttachment(filename="python.py") + self.message.attachments = [attachment] + self.message.channel.send = AsyncMock() + + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + args, kwargs = self.message.channel.send.call_args + embed = kwargs.pop("embed") + + self.assertEqual(args[0], f"Hey {self.message.author.mention}!") + self.assertEqual(embed.description, ( + "It looks like you tried to attach a Python file - " + f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" + )) -- cgit v1.2.3 From 19c15d957040b6857a4141e15c32fd0526f9920d Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 20:15:17 +0200 Subject: AntiMalware Tests - Added unittest for messages that were deleted in the meantime. --- tests/bot/cogs/test_antimalware.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 0bb5af943..da5cd9d11 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,6 +1,9 @@ import asyncio +import logging import unittest -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, Mock + +from discord import NotFound from bot.cogs import antimalware from bot.constants import Roles, URLs @@ -72,3 +75,18 @@ class AntiMalwareCogTests(unittest.TestCase): "It looks like you tried to attach a Python file - " f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" )) + + def test_removing_deleted_message_logs(self): + """Removing an already deleted message logs the correct message""" + attachment = MockAttachment(filename="python.py") + self.message.attachments = [attachment] + self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) + + coroutine = self.cog.on_message(self.message) + logger = logging.getLogger("bot.cogs.antimalware") + + with self.assertLogs(logger=logger, level="INFO") as logs: + asyncio.run(coroutine) + self.assertIn( + f"INFO:bot.cogs.antimalware:Tried to delete message `{self.message.id}`, but message could not be found.", + logs.output) -- cgit v1.2.3 From 4a0b3ea1ef182ddbbb1f9d731b28768a049a531d Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 20:23:00 +0200 Subject: AntiMalware Tests - Added unittest for cog setup --- tests/bot/cogs/test_antimalware.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index da5cd9d11..67c640d23 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -90,3 +90,13 @@ class AntiMalwareCogTests(unittest.TestCase): self.assertIn( f"INFO:bot.cogs.antimalware:Tried to delete message `{self.message.id}`, but message could not be found.", logs.output) + + +class AntiMalwareSetupTests(unittest.TestCase): + """Tests setup of the `AntiMalware` cog.""" + + def test_setup(self): + """Setup of the extension should call add_cog.""" + bot = MockBot() + antimalware.setup(bot) + bot.add_cog.assert_called_once() -- cgit v1.2.3 From 3090141f673279f2836cb3aca95397eb9950ad0f Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 20:41:31 +0200 Subject: AntiMalware Tests - Added unittest message deletion log --- tests/bot/cogs/test_antimalware.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 67c640d23..b4e31b5ce 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,14 +1,17 @@ import asyncio import logging import unittest +from os.path import splitext from unittest.mock import AsyncMock, Mock from discord import NotFound from bot.cogs import antimalware -from bot.constants import Roles, URLs +from bot.constants import AntiMalware as AntiMalwareConfig, Roles, URLs from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole +MODULE = "bot.cogs.antimalware" + class AntiMalwareCogTests(unittest.TestCase): """Test the AntiMalware cog.""" @@ -78,17 +81,38 @@ class AntiMalwareCogTests(unittest.TestCase): def test_removing_deleted_message_logs(self): """Removing an already deleted message logs the correct message""" - attachment = MockAttachment(filename="python.py") + attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) coroutine = self.cog.on_message(self.message) - logger = logging.getLogger("bot.cogs.antimalware") + logger = logging.getLogger(MODULE) with self.assertLogs(logger=logger, level="INFO") as logs: asyncio.run(coroutine) self.assertIn( - f"INFO:bot.cogs.antimalware:Tried to delete message `{self.message.id}`, but message could not be found.", + f"INFO:{MODULE}:Tried to delete message `{self.message.id}`, but message could not be found.", + logs.output) + + def test_message_with_illegal_attachment_logs(self): + """Deleting a message with an illegal attachment should result in a log.""" + attachment = MockAttachment(filename="python.asdfsff") + self.message.attachments = [attachment] + + coroutine = self.cog.on_message(self.message) + file_extensions = {splitext(attachment.filename.lower())[1] for attachment in self.message.attachments} + extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) + blocked_extensions_str = ', '.join(extensions_blocked) + logger = logging.getLogger(MODULE) + + with self.assertLogs(logger=logger, level="INFO") as logs: + asyncio.run(coroutine) + self.assertEqual( + [ + f"INFO:{MODULE}:" + f"User '{self.message.author}' ({self.message.author.id}) " + f"uploaded blacklisted file(s): {blocked_extensions_str}" + ], logs.output) -- cgit v1.2.3 From f0bc9d800dd141b9126c48251a80618e138d61f1 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 20:46:15 +0200 Subject: AntiMalware Tests - Added unittest for valid attachment --- tests/bot/cogs/test_antimalware.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index b4e31b5ce..407fa05c1 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -23,6 +23,15 @@ class AntiMalwareCogTests(unittest.TestCase): self.message = MockMessage() self.message.delete = AsyncMock() + def test_message_with_allowed_attachment(self): + """Messages with allowed extensions should not be deleted""" + attachment = MockAttachment(filename=f"python.{AntiMalwareConfig.whitelist[0]}") + self.message.attachments = [attachment] + + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + self.message.delete.assert_not_called() + def test_message_without_attachment(self): """Messages without attachments should result in no action.""" coroutine = self.cog.on_message(self.message) -- cgit v1.2.3 From 75f6ca6bd9b695a5deb4a4d78311bc63eb2a74d0 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 21:04:47 +0200 Subject: AntiMalware Tests - Added unittest for txt file attachment --- tests/bot/cogs/test_antimalware.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 407fa05c1..eba439afb 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -7,7 +7,7 @@ from unittest.mock import AsyncMock, Mock from discord import NotFound from bot.cogs import antimalware -from bot.constants import AntiMalware as AntiMalwareConfig, Roles, URLs +from bot.constants import AntiMalware as AntiMalwareConfig, Channels, Roles, URLs from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole MODULE = "bot.cogs.antimalware" @@ -21,7 +21,6 @@ class AntiMalwareCogTests(unittest.TestCase): self.bot = MockBot() self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() - self.message.delete = AsyncMock() def test_message_with_allowed_attachment(self): """Messages with allowed extensions should not be deleted""" @@ -88,6 +87,28 @@ class AntiMalwareCogTests(unittest.TestCase): f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" )) + def test_txt_file_redirect_embed(self): + attachment = MockAttachment(filename="python.txt") + self.message.attachments = [attachment] + self.message.channel.send = AsyncMock() + + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + args, kwargs = self.message.channel.send.call_args + embed = kwargs.pop("embed") + cmd_channel = self.bot.get_channel(Channels.bot_commands) + + self.assertEqual(args[0], f"Hey {self.message.author.mention}!") + self.assertEqual(embed.description, ( + "**Uh-oh!** It looks like your message got zapped by our spam filter. " + "We currently don't allow `.txt` attachments, so here are some tips to help you travel safely: \n\n" + "• If you attempted to send a message longer than 2000 characters, try shortening your message " + "to fit within the character limit or use a pasting service (see below) \n\n" + "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " + f"{cmd_channel.mention} for more information) or use a pasting service like: " + f"\n\n{URLs.site_schema}{URLs.site_paste}" + )) + def test_removing_deleted_message_logs(self): """Removing an already deleted message logs the correct message""" attachment = MockAttachment(filename="python.asdfsff") -- cgit v1.2.3 From c8bf44e30c286b27768601d5a04cd2459f170d4c Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 21:29:15 +0200 Subject: AntiMalware Tests - Switched to unittest.IsolatedAsyncioTestCase --- tests/bot/cogs/test_antimalware.py | 48 +++++++++++++++----------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index eba439afb..6fb7b399e 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,4 +1,3 @@ -import asyncio import logging import unittest from os.path import splitext @@ -13,7 +12,7 @@ from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole MODULE = "bot.cogs.antimalware" -class AntiMalwareCogTests(unittest.TestCase): +class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): """Test the AntiMalware cog.""" def setUp(self): @@ -22,62 +21,56 @@ class AntiMalwareCogTests(unittest.TestCase): self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() - def test_message_with_allowed_attachment(self): + async def test_message_with_allowed_attachment(self): """Messages with allowed extensions should not be deleted""" attachment = MockAttachment(filename=f"python.{AntiMalwareConfig.whitelist[0]}") self.message.attachments = [attachment] - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.message.delete.assert_not_called() - def test_message_without_attachment(self): + async def test_message_without_attachment(self): """Messages without attachments should result in no action.""" - coroutine = self.cog.on_message(self.message) - self.assertIsNone(asyncio.run(coroutine)) + self.assertIsNone(await self.cog.on_message(self.message)) self.message.delete.assert_not_called() - def test_direct_message_with_attachment(self): + async def test_direct_message_with_attachment(self): """Direct messages should have no action taken.""" attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] self.message.guild = None - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.message.delete.assert_not_called() - def test_message_with_illegal_extension_gets_deleted(self): + async def test_message_with_illegal_extension_gets_deleted(self): """A message containing an illegal extension should send an embed.""" attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.message.delete.assert_called_once() - def test_message_send_by_staff(self): + async def test_message_send_by_staff(self): """A message send by a member of staff should be ignored.""" moderator_role = MockRole(name="Moderator", id=Roles.moderators) self.message.author.roles.append(moderator_role) attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.message.delete.assert_not_called() - def test_python_file_redirect_embed(self): + async def test_python_file_redirect_embed(self): """A message containing a .python file should result in an embed redirecting the user to our paste site""" attachment = MockAttachment(filename="python.py") self.message.attachments = [attachment] self.message.channel.send = AsyncMock() - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) args, kwargs = self.message.channel.send.call_args embed = kwargs.pop("embed") @@ -87,13 +80,12 @@ class AntiMalwareCogTests(unittest.TestCase): f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" )) - def test_txt_file_redirect_embed(self): + async def test_txt_file_redirect_embed(self): attachment = MockAttachment(filename="python.txt") self.message.attachments = [attachment] self.message.channel.send = AsyncMock() - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) args, kwargs = self.message.channel.send.call_args embed = kwargs.pop("embed") cmd_channel = self.bot.get_channel(Channels.bot_commands) @@ -109,34 +101,32 @@ class AntiMalwareCogTests(unittest.TestCase): f"\n\n{URLs.site_schema}{URLs.site_paste}" )) - def test_removing_deleted_message_logs(self): + async def test_removing_deleted_message_logs(self): """Removing an already deleted message logs the correct message""" attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) - coroutine = self.cog.on_message(self.message) logger = logging.getLogger(MODULE) with self.assertLogs(logger=logger, level="INFO") as logs: - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.assertIn( f"INFO:{MODULE}:Tried to delete message `{self.message.id}`, but message could not be found.", logs.output) - def test_message_with_illegal_attachment_logs(self): + async def test_message_with_illegal_attachment_logs(self): """Deleting a message with an illegal attachment should result in a log.""" attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] - coroutine = self.cog.on_message(self.message) file_extensions = {splitext(attachment.filename.lower())[1] for attachment in self.message.attachments} extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) blocked_extensions_str = ', '.join(extensions_blocked) logger = logging.getLogger(MODULE) with self.assertLogs(logger=logger, level="INFO") as logs: - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.assertEqual( [ f"INFO:{MODULE}:" -- cgit v1.2.3 From bd9537ba85154ece1dca39ec03d36dd7d39a8388 Mon Sep 17 00:00:00 2001 From: MrGrote Date: Fri, 8 May 2020 22:11:54 +0200 Subject: Update tests/bot/cogs/test_antimalware.py Co-authored-by: Mark --- tests/bot/cogs/test_antimalware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 6fb7b399e..e0aa9d6d2 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -65,7 +65,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.message.delete.assert_not_called() async def test_python_file_redirect_embed(self): - """A message containing a .python file should result in an embed redirecting the user to our paste site""" + """A message containing a .py file should result in an embed redirecting the user to our paste site""" attachment = MockAttachment(filename="python.py") self.message.attachments = [attachment] self.message.channel.send = AsyncMock() -- cgit v1.2.3 From b06c60bc3456c36583d3d58cbf62e9ecd14e5f94 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 9 May 2020 23:19:39 -0700 Subject: Filtering: don't delete messages in DMs Bots are incapable of deleting direct messages authored by others. --- bot/cogs/filtering.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 6a703f5a1..1e21a4ce3 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -4,7 +4,7 @@ from typing import Optional, Union import discord.errors from dateutil.relativedelta import relativedelta -from discord import Colour, DMChannel, Member, Message, TextChannel +from discord import Colour, Member, Message, TextChannel from discord.ext.commands import Cog from discord.utils import escape_markdown @@ -161,8 +161,10 @@ class Filtering(Cog): match = await _filter["function"](msg) if match: - # If this is a filter (not a watchlist), we should delete the message. - if _filter["type"] == "filter": + is_private = msg.channel.type is discord.ChannelType.private + + # If this is a filter (not a watchlist) and not in a DM, delete the message. + if _filter["type"] == "filter" and not is_private: try: # Embeds (can?) trigger both the `on_message` and `on_message_edit` # event handlers, triggering filtering twice for the same message. @@ -181,7 +183,7 @@ class Filtering(Cog): if _filter["user_notification"]: await self.notify_member(msg.author, _filter["notification_msg"], msg.channel) - if isinstance(msg.channel, DMChannel): + if is_private: channel_str = "via DM" else: channel_str = f"in {msg.channel.mention}" -- cgit v1.2.3 From 67702dcb98323091a3681c5badc65403fa7147fe Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 11:06:53 -0700 Subject: ModLog: ignore DMs in the message delete listener --- bot/cogs/moderation/modlog.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index beef7a8ef..b434bc4b8 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -552,6 +552,10 @@ class ModLog(Cog, name="ModLog"): channel = message.channel author = message.author + # Ignore DMs. + if not message.guild: + return + if message.guild.id != GuildConstant.id or channel.id in GuildConstant.modlog_blacklist: return -- cgit v1.2.3 From 55b8aed9dc8d4f38824c8b5642a74df6a9948799 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 12:58:38 -0700 Subject: Filtering: don't attempt to send additional embeds for invalid invites Invalid invites won't have data available to put in the embeds. Fixes #929 Fixes BOT-3Z --- bot/cogs/filtering.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 6a703f5a1..0ad534741 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -212,7 +212,9 @@ class Filtering(Cog): additional_embeds = None additional_embeds_msg = None - if filter_name == "filter_invites": + # The function returns True for invalid invites. + # They have no data so additional embeds can't be created for them. + if filter_name == "filter_invites" and match is not True: additional_embeds = [] for invite, data in match.items(): embed = discord.Embed(description=( -- cgit v1.2.3 From 847a78a76c08a670e85d926e3afa43e1cc3180f4 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 19:41:46 +0200 Subject: AntiMalware Tests - implemented minor feedback --- tests/bot/cogs/test_antimalware.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index e0aa9d6d2..6e06df0a8 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,4 +1,3 @@ -import logging import unittest from os.path import splitext from unittest.mock import AsyncMock, Mock @@ -6,7 +5,7 @@ from unittest.mock import AsyncMock, Mock from discord import NotFound from bot.cogs import antimalware -from bot.constants import AntiMalware as AntiMalwareConfig, Channels, Roles, URLs +from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES, URLs from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole MODULE = "bot.cogs.antimalware" @@ -31,7 +30,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_message_without_attachment(self): """Messages without attachments should result in no action.""" - self.assertIsNone(await self.cog.on_message(self.message)) + await self.cog.on_message(self.message) self.message.delete.assert_not_called() async def test_direct_message_with_attachment(self): @@ -55,8 +54,8 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_message_send_by_staff(self): """A message send by a member of staff should be ignored.""" - moderator_role = MockRole(name="Moderator", id=Roles.moderators) - self.message.author.roles.append(moderator_role) + staff_role = MockRole(id=STAFF_ROLES[0]) + self.message.author.roles.append(staff_role) attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] @@ -71,6 +70,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.message.channel.send = AsyncMock() await self.cog.on_message(self.message) + self.message.channel.send.assert_called_once() args, kwargs = self.message.channel.send.call_args embed = kwargs.pop("embed") @@ -107,13 +107,13 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.message.attachments = [attachment] self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) - logger = logging.getLogger(MODULE) - - with self.assertLogs(logger=logger, level="INFO") as logs: + with self.assertLogs(logger=antimalware.log, level="INFO") as logs: await self.cog.on_message(self.message) + self.message.delete.assert_called_once() self.assertIn( f"INFO:{MODULE}:Tried to delete message `{self.message.id}`, but message could not be found.", - logs.output) + logs.output + ) async def test_message_with_illegal_attachment_logs(self): """Deleting a message with an illegal attachment should result in a log.""" @@ -123,9 +123,8 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): file_extensions = {splitext(attachment.filename.lower())[1] for attachment in self.message.attachments} extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) blocked_extensions_str = ', '.join(extensions_blocked) - logger = logging.getLogger(MODULE) - with self.assertLogs(logger=logger, level="INFO") as logs: + with self.assertLogs(logger=antimalware.log, level="INFO") as logs: await self.cog.on_message(self.message) self.assertEqual( [ @@ -133,7 +132,8 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): f"User '{self.message.author}' ({self.message.author.id}) " f"uploaded blacklisted file(s): {blocked_extensions_str}" ], - logs.output) + logs.output + ) class AntiMalwareSetupTests(unittest.TestCase): -- cgit v1.2.3 From ba71ac5b002dd3e1ee6a916ba2705a7cff697a66 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 20:24:20 +0200 Subject: AntiMalware Tests - extracted the method for determining disallowed extensions and added a test for it. --- tests/bot/cogs/test_antimalware.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 6e06df0a8..78ad996f2 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -19,10 +19,11 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.bot = MockBot() self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() + AntiMalwareConfig.whitelist = [".first", ".second", ".third"] async def test_message_with_allowed_attachment(self): """Messages with allowed extensions should not be deleted""" - attachment = MockAttachment(filename=f"python.{AntiMalwareConfig.whitelist[0]}") + attachment = MockAttachment(filename=f"python{AntiMalwareConfig.whitelist[0]}") self.message.attachments = [attachment] await self.cog.on_message(self.message) @@ -35,7 +36,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_direct_message_with_attachment(self): """Direct messages should have no action taken.""" - attachment = MockAttachment(filename="python.asdfsff") + attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] self.message.guild = None @@ -45,7 +46,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_message_with_illegal_extension_gets_deleted(self): """A message containing an illegal extension should send an embed.""" - attachment = MockAttachment(filename="python.asdfsff") + attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] await self.cog.on_message(self.message) @@ -56,7 +57,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): """A message send by a member of staff should be ignored.""" staff_role = MockRole(id=STAFF_ROLES[0]) self.message.author.roles.append(staff_role) - attachment = MockAttachment(filename="python.asdfsff") + attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] await self.cog.on_message(self.message) @@ -103,7 +104,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_removing_deleted_message_logs(self): """Removing an already deleted message logs the correct message""" - attachment = MockAttachment(filename="python.asdfsff") + attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) @@ -117,7 +118,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_message_with_illegal_attachment_logs(self): """Deleting a message with an illegal attachment should result in a log.""" - attachment = MockAttachment(filename="python.asdfsff") + attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] file_extensions = {splitext(attachment.filename.lower())[1] for attachment in self.message.attachments} @@ -135,6 +136,22 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): logs.output ) + async def test_get_disallowed_extensions(self): + """The return value should include all non-whitelisted extensions.""" + test_values = ( + (AntiMalwareConfig.whitelist, []), + ([".first"], []), + ([".first", ".disallowed"], [".disallowed"]), + ([".disallowed"], [".disallowed"]), + ([".disallowed", ".illegal"], [".disallowed", ".illegal"]), + ) + + for extensions, expected_disallowed_extensions in test_values: + with self.subTest(extensions=extensions, expected_disallowed_extensions=expected_disallowed_extensions): + self.message.attachments = [MockAttachment(filename=f"filename{extension}") for extension in extensions] + disallowed_extensions = self.cog.get_disallowed_extensions(self.message) + self.assertCountEqual(disallowed_extensions, expected_disallowed_extensions) + class AntiMalwareSetupTests(unittest.TestCase): """Tests setup of the `AntiMalware` cog.""" -- cgit v1.2.3 From 148b12603f4ad8799d135ec9956d1841cf1c7bf7 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 20:24:39 +0200 Subject: AntiMalware Tests - extracted the method for determining disallowed extensions and added a test for it. --- bot/cogs/antimalware.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bot/cogs/antimalware.py b/bot/cogs/antimalware.py index 66b5073e8..f5fd5e2d9 100644 --- a/bot/cogs/antimalware.py +++ b/bot/cogs/antimalware.py @@ -1,4 +1,5 @@ import logging +import typing as t from os.path import splitext from discord import Embed, Message, NotFound @@ -29,8 +30,7 @@ class AntiMalware(Cog): return embed = Embed() - file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} - extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) + extensions_blocked = self.get_disallowed_extensions(message) blocked_extensions_str = ', '.join(extensions_blocked) if ".py" in extensions_blocked: # Short-circuit on *.py files to provide a pastebin link @@ -73,6 +73,13 @@ class AntiMalware(Cog): except NotFound: log.info(f"Tried to delete message `{message.id}`, but message could not be found.") + @classmethod + def get_disallowed_extensions(cls, message: Message) -> t.Iterable[str]: + """Get an iterable containing all the disallowed extensions of attachments.""" + file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} + extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) + return extensions_blocked + def setup(bot: Bot) -> None: """Load the AntiMalware cog.""" -- cgit v1.2.3 From ecaddcedab6946ac4650b699a790471ef2a898c9 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 20:39:25 +0200 Subject: AntiMalware Tests - added a missing case for no extensions in test_get_disallowed_extensions --- tests/bot/cogs/test_antimalware.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 78ad996f2..7caee6f3c 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -139,6 +139,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_get_disallowed_extensions(self): """The return value should include all non-whitelisted extensions.""" test_values = ( + ([], []), (AntiMalwareConfig.whitelist, []), ([".first"], []), ([".first", ".disallowed"], [".disallowed"]), -- cgit v1.2.3 From fa467e4ef133186ff462b0178bcab08e8a3d6b2d Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 20:58:51 +0200 Subject: AntiMalware Tests - Removed exact log content checks --- tests/bot/cogs/test_antimalware.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 7caee6f3c..a2ce9a740 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,5 +1,4 @@ import unittest -from os.path import splitext from unittest.mock import AsyncMock, Mock from discord import NotFound @@ -108,33 +107,17 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.message.attachments = [attachment] self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) - with self.assertLogs(logger=antimalware.log, level="INFO") as logs: + with self.assertLogs(logger=antimalware.log, level="INFO"): await self.cog.on_message(self.message) self.message.delete.assert_called_once() - self.assertIn( - f"INFO:{MODULE}:Tried to delete message `{self.message.id}`, but message could not be found.", - logs.output - ) async def test_message_with_illegal_attachment_logs(self): """Deleting a message with an illegal attachment should result in a log.""" attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] - file_extensions = {splitext(attachment.filename.lower())[1] for attachment in self.message.attachments} - extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) - blocked_extensions_str = ', '.join(extensions_blocked) - - with self.assertLogs(logger=antimalware.log, level="INFO") as logs: + with self.assertLogs(logger=antimalware.log, level="INFO"): await self.cog.on_message(self.message) - self.assertEqual( - [ - f"INFO:{MODULE}:" - f"User '{self.message.author}' ({self.message.author.id}) " - f"uploaded blacklisted file(s): {blocked_extensions_str}" - ], - logs.output - ) async def test_get_disallowed_extensions(self): """The return value should include all non-whitelisted extensions.""" -- cgit v1.2.3 From b366d655af0e0f5a9ff3e053a693838d49884ea2 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 13:28:32 -0700 Subject: Token remover: catch ValueError when non-ASCII chars are present The token uses base64 and base64 only allows ASCII characters. Thus, if a match has non-ASCII characters, it's not a valid token. Catching the ValueError is simpler than trying to adjust the regex to only match valid base64. Fixes #928 Fixes BOT-3X --- bot/cogs/token_remover.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index 6721f0e02..860ae9f3a 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -135,7 +135,7 @@ class TokenRemover(Cog): try: content: bytes = base64.b64decode(b64_content) return content.decode('utf-8').isnumeric() - except (binascii.Error, UnicodeDecodeError): + except (binascii.Error, ValueError): return False @staticmethod @@ -150,7 +150,7 @@ class TokenRemover(Cog): try: content = base64.urlsafe_b64decode(b64_content) snowflake = struct.unpack('i', content)[0] - except (binascii.Error, struct.error): + except (binascii.Error, struct.error, ValueError): return False return snowflake_time(snowflake + TOKEN_EPOCH) < DISCORD_EPOCH_TIMESTAMP -- cgit v1.2.3 From f03ae8e49bb3d62776528e6339d6c713c93b7674 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 14:08:02 -0700 Subject: Token remover: reduce duplicated code in `on_message_edit` --- bot/cogs/token_remover.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index 860ae9f3a..e90d5ab8b 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -65,9 +65,7 @@ class TokenRemover(Cog): See: https://discordapp.com/developers/docs/reference#snowflakes """ - found_token = self.find_token_in_message(after) - if found_token: - await self.take_action(after, found_token) + await self.on_message(after) async def take_action(self, msg: Message, found_token: str) -> None: """Remove the `msg` containing a token an send a mod_log message.""" -- cgit v1.2.3 From d193a93828582965eb361dc6f3185291fff649a7 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 14:11:39 -0700 Subject: Test on_message_edit of token remover uses on_message --- tests/bot/cogs/test_token_remover.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 33d1ec170..e7b5a9bea 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -1,6 +1,7 @@ import asyncio import logging import unittest +from unittest import mock from unittest.mock import AsyncMock, MagicMock from discord import Colour @@ -14,7 +15,7 @@ from bot.constants import Channels, Colours, Event, Icons from tests.helpers import MockBot, MockMessage -class TokenRemoverTests(unittest.TestCase): +class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): """Tests the `TokenRemover` cog.""" def setUp(self): @@ -58,6 +59,13 @@ class TokenRemoverTests(unittest.TestCase): self.assertEqual(self.cog.mod_log, self.bot.get_cog.return_value) self.bot.get_cog.assert_called_once_with('ModLog') + async def test_on_message_edit_uses_on_message(self): + """The edit listener should delegate handling of the message to the normal listener.""" + self.cog.on_message = mock.create_autospec(self.cog.on_message, spec_set=True) + + await self.cog.on_message_edit(MockMessage(), self.msg) + self.cog.on_message.assert_awaited_once_with(self.msg) + def test_ignores_bot_messages(self): """When the message event handler is called with a bot message, nothing is done.""" self.msg.author.bot = True @@ -77,7 +85,7 @@ class TokenRemoverTests(unittest.TestCase): for content in ('foo.bar.baz', 'x.y.'): with self.subTest(content=content): self.msg.content = content - coroutine = self.cog.on_message(self.msg) + coroutine = self.cog.is_maybe_token(self.msg) self.assertIsNone(asyncio.run(coroutine)) def test_censors_valid_tokens(self): -- cgit v1.2.3 From 0bfd003dbfc5919220129f984dc043421e535f8c Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 14:38:12 -0700 Subject: Add a test helper function to patch multiple attributes with autospecs This helper reduces redundancy/boilerplate by setting default values. It also has the consequence of shortening the length of the invocation, which makes it faster to use and easier to read. --- tests/helpers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/helpers.py b/tests/helpers.py index 2b79a6c2a..d444cc49d 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -23,6 +23,15 @@ for logger in logging.Logger.manager.loggerDict.values(): logger.setLevel(logging.CRITICAL) +def autospec(target, *attributes: str, **kwargs) -> unittest.mock._patch: + """Patch multiple `attributes` of a `target` with autospecced mocks and `spec_set` as True.""" + # Caller's kwargs should take priority and overwrite the defaults. + kwargs = {'spec_set': True, 'autospec': True, **kwargs} + attributes = {attribute: unittest.mock.DEFAULT for attribute in attributes} + + return unittest.mock.patch.multiple(target, **attributes, **kwargs) + + class HashableMixin(discord.mixins.EqualityComparable): """ Mixin that provides similar hashing and equality functionality as discord.py's `Hashable` mixin. -- cgit v1.2.3 From e8bd69a6c556d78eca1a1eb2adfa26248273a1cd Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 14:42:07 -0700 Subject: Test token remover takes action if a token is found --- tests/bot/cogs/test_token_remover.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index e7b5a9bea..e0ec67684 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -12,7 +12,7 @@ from bot.cogs.token_remover import ( setup as setup_cog, ) from bot.constants import Channels, Colours, Event, Icons -from tests.helpers import MockBot, MockMessage +from tests.helpers import MockBot, MockMessage, autospec class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): @@ -66,6 +66,18 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): await self.cog.on_message_edit(MockMessage(), self.msg) self.cog.on_message.assert_awaited_once_with(self.msg) + @autospec(TokenRemover, "find_token_in_message", "take_action") + async def test_on_message_takes_action(self, find_token_in_message, take_action): + """Should take action if a valid token is found when a message is sent.""" + cog = TokenRemover(self.bot) + found_token = "foobar" + find_token_in_message.return_value = found_token + + await cog.on_message(self.msg) + + find_token_in_message.assert_called_once_with(self.msg) + take_action.assert_awaited_once_with(cog, self.msg, found_token) + def test_ignores_bot_messages(self): """When the message event handler is called with a bot message, nothing is done.""" self.msg.author.bot = True -- cgit v1.2.3 From 4cf7996a1d4630ccb05f57569ca62b1798dc7a93 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 14:44:54 -0700 Subject: Test token remover skips messages without tokens --- tests/bot/cogs/test_token_remover.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index e0ec67684..2b377e221 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -78,6 +78,17 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): find_token_in_message.assert_called_once_with(self.msg) take_action.assert_awaited_once_with(cog, self.msg, found_token) + @autospec(TokenRemover, "find_token_in_message", "take_action") + async def test_on_message_skips_missing_token(self, find_token_in_message, take_action): + """Shouldn't take action if a valid token isn't found when a message is sent.""" + cog = TokenRemover(self.bot) + find_token_in_message.return_value = False + + await cog.on_message(self.msg) + + find_token_in_message.assert_called_once_with(self.msg) + take_action.assert_not_awaited() + def test_ignores_bot_messages(self): """When the message event handler is called with a bot message, nothing is done.""" self.msg.author.bot = True -- cgit v1.2.3 From 593e09299c6e4115d41bfd5b074785a5e304a8d0 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 15:41:14 -0700 Subject: Allow using arbitrary parameter names with the autospec decorator This gives the caller more flexibility. Sometimes attribute names are too long or they don't follow a naming scheme accepted by the linter. --- tests/helpers.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index d444cc49d..1ab8b455f 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -24,12 +24,25 @@ for logger in logging.Logger.manager.loggerDict.values(): def autospec(target, *attributes: str, **kwargs) -> unittest.mock._patch: - """Patch multiple `attributes` of a `target` with autospecced mocks and `spec_set` as True.""" + """ + Patch multiple `attributes` of a `target` with autospecced mocks and `spec_set` as True. + + To allow for arbitrary parameter names to be used by the decorated function, the patchers have + no attribute names associated with them. As a consequence, it will not be possible to retrieve + mocks by their attribute names when using this as a context manager, + """ # Caller's kwargs should take priority and overwrite the defaults. kwargs = {'spec_set': True, 'autospec': True, **kwargs} attributes = {attribute: unittest.mock.DEFAULT for attribute in attributes} - return unittest.mock.patch.multiple(target, **attributes, **kwargs) + patcher = unittest.mock.patch.multiple(target, **attributes, **kwargs) + + # Unset attribute names to allow arbitrary parameter names for the decorator function. + patcher.attribute_name = None + for additional_patcher in patcher.additional_patchers: + additional_patcher.attribute_name = None + + return patcher class HashableMixin(discord.mixins.EqualityComparable): -- cgit v1.2.3 From b0dd290710799c342240d066abaebbe9e6940b54 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 15:09:22 -0700 Subject: Fix test for token remover ignoring bot messages It's not possible to test this via asserting the return value of `on_message` since it never returns anything. Instead, the actual relevant unit, `find_token_in_message,` should be tested. --- tests/bot/cogs/test_token_remover.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 2b377e221..e8b641101 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -89,11 +89,16 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): find_token_in_message.assert_called_once_with(self.msg) take_action.assert_not_awaited() - def test_ignores_bot_messages(self): - """When the message event handler is called with a bot message, nothing is done.""" + @autospec("bot.cogs.token_remover", "TOKEN_RE") + def test_find_token_ignores_bot_messages(self, token_re): + """The token finder should ignore messages authored by bots.""" + cog = TokenRemover(self.bot) self.msg.author.bot = True - coroutine = self.cog.on_message(self.msg) - self.assertIsNone(asyncio.run(coroutine)) + + return_value = cog.find_token_in_message(self.msg) + + self.assertIsNone(return_value) + token_re.findall.assert_not_called() def test_ignores_messages_without_tokens(self): """Messages without anything looking like a token are ignored.""" -- cgit v1.2.3 From 52f0f8a29d7f239c961beaa81881bf4b09da4749 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 15:53:06 -0700 Subject: Test `find_token_in_message` returns None if no matches found --- tests/bot/cogs/test_token_remover.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index e8b641101..5932cf4f0 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -100,6 +100,20 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): self.assertIsNone(return_value) token_re.findall.assert_not_called() + @autospec(TokenRemover, "is_maybe_token") + @autospec("bot.cogs.token_remover", "TOKEN_RE") + def test_find_token_no_matches_returns_none(self, token_re, is_maybe_token): + """None should be returned if the regex matches no tokens in a message.""" + cog = TokenRemover(self.bot) + token_re.findall.return_value = () + self.msg.content = "foobar" + + return_value = cog.find_token_in_message(self.msg) + + self.assertIsNone(return_value) + token_re.findall.assert_called_once_with(self.msg.content) + is_maybe_token.assert_not_called() + def test_ignores_messages_without_tokens(self): """Messages without anything looking like a token are ignored.""" for content in ('', 'lemon wins'): -- cgit v1.2.3 From cf658bd58559b2683527443f2908257f197ef0bb Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 16:06:47 -0700 Subject: Test `find_token_in_message` returns the found token --- tests/bot/cogs/test_token_remover.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 5932cf4f0..2b946778b 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -114,6 +114,30 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): token_re.findall.assert_called_once_with(self.msg.content) is_maybe_token.assert_not_called() + @autospec(TokenRemover, "is_maybe_token") + @autospec("bot.cogs.token_remover", "TOKEN_RE") + def test_find_token_returns_found_token(self, token_re, is_maybe_token): + """The found token should be returned.""" + true_index = 1 + matches = ("foo", "bar", "baz") + side_effects = [False] * len(matches) + side_effects[true_index] = True + + cog = TokenRemover(self.bot) + self.msg.content = "foobar" + token_re.findall.return_value = matches + is_maybe_token.side_effect = side_effects + + return_value = cog.find_token_in_message(self.msg) + + self.assertEqual(return_value, matches[true_index]) + token_re.findall.assert_called_once_with(self.msg.content) + + # assert_has_calls isn't used cause it'd allow for extra calls before or after. + # The function should short-circuit, so nothing past true_index should have been used. + calls = [mock.call(match) for match in matches[:true_index + 1]] + self.assertEqual(is_maybe_token.mock_calls, calls) + def test_ignores_messages_without_tokens(self): """Messages without anything looking like a token are ignored.""" for content in ('', 'lemon wins'): -- cgit v1.2.3 From f92bc80d6bddb5c57c190187adaa528ae44536f6 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 16:25:14 -0700 Subject: Test token regex doesn't match invalid tokens --- tests/bot/cogs/test_token_remover.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 2b946778b..b67602eb9 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -8,6 +8,7 @@ from discord import Colour from bot.cogs.token_remover import ( DELETION_MESSAGE_TEMPLATE, + TOKEN_RE, TokenRemover, setup as setup_cog, ) @@ -138,13 +139,30 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): calls = [mock.call(match) for match in matches[:true_index + 1]] self.assertEqual(is_maybe_token.mock_calls, calls) - def test_ignores_messages_without_tokens(self): - """Messages without anything looking like a token are ignored.""" - for content in ('', 'lemon wins'): - with self.subTest(content=content): - self.msg.content = content - coroutine = self.cog.on_message(self.msg) - self.assertIsNone(asyncio.run(coroutine)) + def test_regex_invalid_tokens(self): + """Messages without anything looking like a token are not matched.""" + tokens = ( + "", + "lemon wins", + "..", + "x.y", + "x.y.", + ".y.z", + ".y.", + "..z", + "x..z", + " . . ", + "\n.\n.\n", + "'.'.'", + '"."."', + "(.(.(", + ").).)" + ) + + for token in tokens: + with self.subTest(token=token): + results = TOKEN_RE.findall(token) + self.assertEquals(len(results), 0) def test_ignores_messages_with_invalid_tokens(self): """Messages with values that are invalid tokens are ignored.""" -- cgit v1.2.3 From 34b836a8eba0f006c77a7b3f48f7ab14c37d31ee Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 17:47:09 -0700 Subject: Fix autospec decorator when used with multiple attributes The original approach of messing with the `attribute_name` didn't work for reasons I won't discuss here (would require knowledge of patcher internals). The new approach doesn't use patch.multiple but mimics it by applying multiple patch decorators to the function. As a consequence, this can no longer be used as a context manager. --- tests/helpers.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 1ab8b455f..dfbe539ec 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -24,25 +24,21 @@ for logger in logging.Logger.manager.loggerDict.values(): def autospec(target, *attributes: str, **kwargs) -> unittest.mock._patch: - """ - Patch multiple `attributes` of a `target` with autospecced mocks and `spec_set` as True. - - To allow for arbitrary parameter names to be used by the decorated function, the patchers have - no attribute names associated with them. As a consequence, it will not be possible to retrieve - mocks by their attribute names when using this as a context manager, - """ + """Patch multiple `attributes` of a `target` with autospecced mocks and `spec_set` as True.""" # Caller's kwargs should take priority and overwrite the defaults. kwargs = {'spec_set': True, 'autospec': True, **kwargs} - attributes = {attribute: unittest.mock.DEFAULT for attribute in attributes} - - patcher = unittest.mock.patch.multiple(target, **attributes, **kwargs) - - # Unset attribute names to allow arbitrary parameter names for the decorator function. - patcher.attribute_name = None - for additional_patcher in patcher.additional_patchers: - additional_patcher.attribute_name = None - return patcher + # Import the target if it's a string. + # This is to support both object and string targets like patch.multiple. + if type(target) is str: + target = unittest.mock._importer(target) + + def decorator(func): + for attribute in attributes: + patcher = unittest.mock.patch.object(target, attribute, **kwargs) + func = patcher(func) + return func + return decorator class HashableMixin(discord.mixins.EqualityComparable): -- cgit v1.2.3 From 834bd543d1d301bb853e713560a7447dc75f1ab8 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 17:53:40 -0700 Subject: Test `is_maybe_token` returns False for missing parts In practice, this won't ever happen since the regex wouldn't match strings with missing parts. However, the function does check it so may as well test it. It's not necessarily bound to always use inputs from the regex either I suppose. --- tests/bot/cogs/test_token_remover.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index b67602eb9..9e1d96a37 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -164,6 +164,16 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): results = TOKEN_RE.findall(token) self.assertEquals(len(results), 0) + @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") + def test_is_maybe_token_missing_part_returns_false(self, valid_user, valid_time): + """False should be returned for tokens which do not have all 3 parts.""" + cog = TokenRemover(self.bot) + return_value = cog.is_maybe_token("x.y") + + self.assertFalse(return_value) + valid_user.assert_not_called() + valid_time.assert_not_called() + def test_ignores_messages_with_invalid_tokens(self): """Messages with values that are invalid tokens are ignored.""" for content in ('foo.bar.baz', 'x.y.'): -- cgit v1.2.3 From 4248f88a7407b6e9a5d80800a96f8707003634d3 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 18:07:17 -0700 Subject: Token remover: fix `is_maybe_token` returning None instead of False It's annotated as returning a bool and when the split fails it already returns False. To be consistent, it should always return a bool. --- bot/cogs/token_remover.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index e90d5ab8b..543f4c5a7 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -121,6 +121,8 @@ class TokenRemover(Cog): if cls.is_valid_user_id(user_id) and cls.is_valid_timestamp(creation_timestamp): return True + return False + @staticmethod def is_valid_user_id(b64_content: str) -> bool: """ -- cgit v1.2.3 From ab5d194b90a7e068c8ab7171939f471e252ee073 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 18:11:31 -0700 Subject: Test is_maybe_token --- tests/bot/cogs/test_token_remover.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 9e1d96a37..85bbbdf6b 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -174,13 +174,30 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): valid_user.assert_not_called() valid_time.assert_not_called() - def test_ignores_messages_with_invalid_tokens(self): - """Messages with values that are invalid tokens are ignored.""" - for content in ('foo.bar.baz', 'x.y.'): - with self.subTest(content=content): - self.msg.content = content - coroutine = self.cog.is_maybe_token(self.msg) - self.assertIsNone(asyncio.run(coroutine)) + @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") + def test_is_maybe_token(self, valid_user, valid_time): + """Should return True if the user ID and timestamp are valid or return False otherwise.""" + cog = TokenRemover(self.bot) + subtests = ( + (False, True, False), + (True, False, False), + (True, True, True), + ) + + for user_return, time_return, expected in subtests: + valid_user.reset_mock() + valid_time.reset_mock() + + with self.subTest(user_return=user_return, time_return=time_return, expected=expected): + valid_user.return_value = user_return + valid_time.return_value = time_return + + actual = cog.is_maybe_token("x.y.z") + self.assertIs(actual, expected) + + valid_user.assert_called_once_with("x") + if user_return: + valid_time.assert_called_once_with("y") def test_censors_valid_tokens(self): """Valid tokens are censored.""" -- cgit v1.2.3 From 4b6fde69a7e193382701dccf80a5471ea7ccea72 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 18:22:31 -0700 Subject: Test token regex matches valid tokens --- tests/bot/cogs/test_token_remover.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 85bbbdf6b..7310b4637 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -164,6 +164,27 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): results = TOKEN_RE.findall(token) self.assertEquals(len(results), 0) + def test_regex_valid_tokens(self): + """Messages that look like tokens should be matched.""" + # Don't worry, the token's been invalidated. + tokens = ( + "x1.y2.z_3", + "NDcyMjY1OTQzMDYyNDEzMzMy.Xrim9Q.Ysnu2wacjaKs7qnoo46S8Dm2us8" + ) + + for token in tokens: + with self.subTest(token=token): + results = TOKEN_RE.findall(token) + self.assertIn(token, results) + + def test_regex_matches_multiple_valid(self): + """Should support multiple matches in the middle of a string.""" + tokens = ["x.y.z", "a.b.c"] + message = f"garbage {tokens[0]} hello {tokens[1]} world" + + results = TOKEN_RE.findall(message) + self.assertEquals(tokens, results) + @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") def test_is_maybe_token_missing_part_returns_false(self, valid_user, valid_time): """False should be returned for tokens which do not have all 3 parts.""" -- cgit v1.2.3 From d8d8e144adfe4c2de15dbbf4346e2eec548a9f67 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 18:28:06 -0700 Subject: Correct the return type annotation for the autospec decorator --- tests/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index dfbe539ec..3cd8a63c0 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -4,7 +4,7 @@ import collections import itertools import logging import unittest.mock -from typing import Iterable, Optional +from typing import Callable, Iterable, Optional import discord from discord.ext.commands import Context @@ -23,7 +23,7 @@ for logger in logging.Logger.manager.loggerDict.values(): logger.setLevel(logging.CRITICAL) -def autospec(target, *attributes: str, **kwargs) -> unittest.mock._patch: +def autospec(target, *attributes: str, **kwargs) -> Callable: """Patch multiple `attributes` of a `target` with autospecced mocks and `spec_set` as True.""" # Caller's kwargs should take priority and overwrite the defaults. kwargs = {'spec_set': True, 'autospec': True, **kwargs} -- cgit v1.2.3 From ab860e23a7e6206e68cb350257b63083cfbe1a15 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 18:53:42 -0700 Subject: Token remover: split some of `take_action` into separate functions --- bot/cogs/token_remover.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index 543f4c5a7..d6919839e 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -68,31 +68,41 @@ class TokenRemover(Cog): await self.on_message(after) async def take_action(self, msg: Message, found_token: str) -> None: - """Remove the `msg` containing a token an send a mod_log message.""" - user_id, creation_timestamp, hmac = found_token.split('.') + """Remove the `msg` containing the `found_token` and send a mod log message.""" self.mod_log.ignore(Event.message_delete, msg.id) - await msg.delete() - await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention)) + await self.delete_message(msg) - message = ( - "Censored a seemingly valid token sent by " - f"{msg.author} (`{msg.author.id}`) in {msg.channel.mention}, token was " - f"`{user_id}.{creation_timestamp}.{'x' * len(hmac)}`" - ) - log.debug(message) + log_message = self.format_log_message(msg, found_token) + log.debug(log_message) # Send pretty mod log embed to mod-alerts await self.mod_log.send_log_message( icon_url=Icons.token_removed, colour=Colour(Colours.soft_red), title="Token removed!", - text=message, + text=log_message, thumbnail=msg.author.avatar_url_as(static_format="png"), channel_id=Channels.mod_alerts, ) self.bot.stats.incr("tokens.removed_tokens") + @staticmethod + async def delete_message(msg: Message) -> None: + """Remove a `msg` containing a token and send an explanatory message in the same channel.""" + await msg.delete() + await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention)) + + @staticmethod + def format_log_message(msg: Message, found_token: str) -> str: + """Return the log message to send for `found_token` being censored in `msg`.""" + user_id, creation_timestamp, hmac = found_token.split('.') + return ( + "Censored a seemingly valid token sent by " + f"{msg.author} (`{msg.author.id}`) in {msg.channel.mention}, token was " + f"`{user_id}.{creation_timestamp}.{'x' * len(hmac)}`" + ) + @classmethod def find_token_in_message(cls, msg: Message) -> t.Optional[str]: """Return a seemingly valid token found in `msg` or `None` if no token is found.""" -- cgit v1.2.3 From 09a6c2e211c0f209b258a02d9677240282c4fab3 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 18:55:24 -0700 Subject: Token remover: use a string template for the log message --- bot/cogs/token_remover.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index d6919839e..c576a67d0 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -16,6 +16,10 @@ from bot.constants import Channels, Colours, Event, Icons log = logging.getLogger(__name__) +LOG_MESSAGE = ( + "Censored a seemingly valid token sent by {author} (`{author_id}`) in {channel}," + "token was `{user_id}.{timestamp}.{hmac}`" +) DELETION_MESSAGE_TEMPLATE = ( "Hey {mention}! I noticed you posted a seemingly valid Discord API " "token in your message and have removed your message. " @@ -97,10 +101,13 @@ class TokenRemover(Cog): def format_log_message(msg: Message, found_token: str) -> str: """Return the log message to send for `found_token` being censored in `msg`.""" user_id, creation_timestamp, hmac = found_token.split('.') - return ( - "Censored a seemingly valid token sent by " - f"{msg.author} (`{msg.author.id}`) in {msg.channel.mention}, token was " - f"`{user_id}.{creation_timestamp}.{'x' * len(hmac)}`" + return LOG_MESSAGE.format( + author=msg.author, + author_id=msg.author.id, + channel=msg.channel.mention, + user_id=user_id, + timestamp=creation_timestamp, + hmac='x' * len(hmac), ) @classmethod -- cgit v1.2.3 From 5b9bf9aba686f570322cb9996dd35d3ab669a162 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 11 May 2020 10:26:16 -0700 Subject: Avoid instantiating the cog when testing static/class methods --- tests/bot/cogs/test_token_remover.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 7310b4637..6a8247070 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -93,10 +93,9 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): @autospec("bot.cogs.token_remover", "TOKEN_RE") def test_find_token_ignores_bot_messages(self, token_re): """The token finder should ignore messages authored by bots.""" - cog = TokenRemover(self.bot) self.msg.author.bot = True - return_value = cog.find_token_in_message(self.msg) + return_value = TokenRemover.find_token_in_message(self.msg) self.assertIsNone(return_value) token_re.findall.assert_not_called() @@ -105,11 +104,10 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): @autospec("bot.cogs.token_remover", "TOKEN_RE") def test_find_token_no_matches_returns_none(self, token_re, is_maybe_token): """None should be returned if the regex matches no tokens in a message.""" - cog = TokenRemover(self.bot) token_re.findall.return_value = () self.msg.content = "foobar" - return_value = cog.find_token_in_message(self.msg) + return_value = TokenRemover.find_token_in_message(self.msg) self.assertIsNone(return_value) token_re.findall.assert_called_once_with(self.msg.content) @@ -124,12 +122,11 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): side_effects = [False] * len(matches) side_effects[true_index] = True - cog = TokenRemover(self.bot) self.msg.content = "foobar" token_re.findall.return_value = matches is_maybe_token.side_effect = side_effects - return_value = cog.find_token_in_message(self.msg) + return_value = TokenRemover.find_token_in_message(self.msg) self.assertEqual(return_value, matches[true_index]) token_re.findall.assert_called_once_with(self.msg.content) @@ -188,8 +185,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") def test_is_maybe_token_missing_part_returns_false(self, valid_user, valid_time): """False should be returned for tokens which do not have all 3 parts.""" - cog = TokenRemover(self.bot) - return_value = cog.is_maybe_token("x.y") + return_value = TokenRemover.is_maybe_token("x.y") self.assertFalse(return_value) valid_user.assert_not_called() @@ -198,7 +194,6 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") def test_is_maybe_token(self, valid_user, valid_time): """Should return True if the user ID and timestamp are valid or return False otherwise.""" - cog = TokenRemover(self.bot) subtests = ( (False, True, False), (True, False, False), @@ -213,7 +208,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): valid_user.return_value = user_return valid_time.return_value = time_return - actual = cog.is_maybe_token("x.y.z") + actual = TokenRemover.is_maybe_token("x.y.z") self.assertIs(actual, expected) valid_user.assert_called_once_with("x") -- cgit v1.2.3 From 2127239840085ba523d411899e0b7a188530df07 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 11 May 2020 10:33:05 -0700 Subject: Simplify token remover's message mock * Rely on default values for the author * Set the content to a non-empty string --- tests/bot/cogs/test_token_remover.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 6a8247070..5ca863926 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -26,14 +26,10 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): self.bot.get_cog.return_value.send_log_message = AsyncMock() self.cog = TokenRemover(bot=self.bot) - self.msg = MockMessage(id=555, content='') - self.msg.author.__str__ = MagicMock() - self.msg.author.__str__.return_value = 'lemon' - self.msg.author.bot = False - self.msg.author.avatar_url_as.return_value = 'picture-lemon.png' - self.msg.author.id = 42 - self.msg.author.mention = '@lemon' + self.msg = MockMessage(id=555, content="hello world") self.msg.channel.mention = "#lemonade-stand" + self.msg.author.__str__ = MagicMock(return_value=self.msg.author.name) + self.msg.author.avatar_url_as.return_value = "picture-lemon.png" def test_is_valid_user_id_is_true_for_numeric_content(self): """A string decoding to numeric characters is a valid user ID.""" @@ -105,7 +101,6 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): def test_find_token_no_matches_returns_none(self, token_re, is_maybe_token): """None should be returned if the regex matches no tokens in a message.""" token_re.findall.return_value = () - self.msg.content = "foobar" return_value = TokenRemover.find_token_in_message(self.msg) @@ -122,7 +117,6 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): side_effects = [False] * len(matches) side_effects[true_index] = True - self.msg.content = "foobar" token_re.findall.return_value = matches is_maybe_token.side_effect = side_effects -- cgit v1.2.3 From e4790b330da1605573b5d23615bfe62b481e1e04 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 11 May 2020 10:37:59 -0700 Subject: Test token remover's message deletion --- tests/bot/cogs/test_token_remover.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 5ca863926..d65ce2ce5 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -209,6 +209,15 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): if user_return: valid_time.assert_called_once_with("y") + async def test_delete_message(self): + """The message should be deleted, and a message should be sent to the same channel.""" + await TokenRemover.delete_message(self.msg) + + self.msg.delete.assert_called_once_with() + self.msg.channel.send.assert_called_once_with( + DELETION_MESSAGE_TEMPLATE.format(mention=self.msg.author.mention) + ) + def test_censors_valid_tokens(self): """Valid tokens are censored.""" cases = ( -- cgit v1.2.3 From 567a5f9242912d6a3340c088c0ae1a62977a141e Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 11 May 2020 10:46:02 -0700 Subject: Test TokenRemover.format_log_message --- tests/bot/cogs/test_token_remover.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index d65ce2ce5..f5412e692 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -218,6 +218,22 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): DELETION_MESSAGE_TEMPLATE.format(mention=self.msg.author.mention) ) + @autospec("bot.cogs.token_remover", "LOG_MESSAGE") + async def test_format_log_message(self, log_message): + """Should correctly format the log message with info from the message and token.""" + log_message.format.return_value = "Howdy" + return_value = TokenRemover.format_log_message(self.msg, "MTIz.DN9R_A.xyz") + + self.assertEqual(return_value, log_message.format.return_value) + log_message.format.assert_called_once_with( + author=self.msg.author, + author_id=self.msg.author.id, + channel=self.msg.channel.mention, + user_id="MTIz", + timestamp="DN9R_A", + hmac="xxx", + ) + def test_censors_valid_tokens(self): """Valid tokens are censored.""" cases = ( -- cgit v1.2.3 From f47cbef0b47ef11b8c1fd63076105e4cb7d73601 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 11 May 2020 11:29:28 -0700 Subject: Test TokenRemover.take_action * Remove `bot.get_cog` mocks in `setUp` * Mock the logger cause it's easier to assert logs * Remove subtests * Assert helper functions were called * Create an autospec for ModLog --- tests/bot/cogs/test_token_remover.py | 73 +++++++++++++++--------------------- 1 file changed, 30 insertions(+), 43 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index f5412e692..3546e7964 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -1,11 +1,10 @@ -import asyncio -import logging import unittest from unittest import mock -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock from discord import Colour +from bot.cogs.moderation import ModLog from bot.cogs.token_remover import ( DELETION_MESSAGE_TEMPLATE, TOKEN_RE, @@ -22,8 +21,6 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): def setUp(self): """Adds the cog, a bot, and a message to the instance for usage in tests.""" self.bot = MockBot() - self.bot.get_cog.return_value = MagicMock() - self.bot.get_cog.return_value.send_log_message = AsyncMock() self.cog = TokenRemover(bot=self.bot) self.msg = MockMessage(id=555, content="hello world") @@ -234,46 +231,36 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): hmac="xxx", ) - def test_censors_valid_tokens(self): - """Valid tokens are censored.""" - cases = ( - # (content, censored_token) - ('MTIz.DN9R_A.xyz', 'MTIz.DN9R_A.xxx'), + @mock.patch.object(TokenRemover, "mod_log", new_callable=mock.PropertyMock) + @autospec("bot.cogs.token_remover", "log") + @autospec(TokenRemover, "delete_message", "format_log_message") + async def test_take_action(self, delete_message, format_log_message, logger, mod_log_property): + """Should delete the message and send a mod log.""" + cog = TokenRemover(self.bot) + mod_log = mock.create_autospec(ModLog, spec_set=True, instance=True) + token = "MTIz.DN9R_A.xyz" + log_msg = "testing123" + + mod_log_property.return_value = mod_log + format_log_message.return_value = log_msg + + await cog.take_action(self.msg, token) + + delete_message.assert_awaited_once_with(self.msg) + format_log_message.assert_called_once_with(self.msg, token) + logger.debug.assert_called_with(log_msg) + self.bot.stats.incr.assert_called_once_with("tokens.removed_tokens") + + mod_log.ignore.assert_called_once_with(Event.message_delete, self.msg.id) + mod_log.send_log_message.assert_called_once_with( + icon_url=Icons.token_removed, + colour=Colour(Colours.soft_red), + title="Token removed!", + text=log_msg, + thumbnail=self.msg.author.avatar_url_as.return_value, + channel_id=Channels.mod_alerts ) - for content, censored_token in cases: - with self.subTest(content=content, censored_token=censored_token): - self.msg.content = content - coroutine = self.cog.on_message(self.msg) - with self.assertLogs(logger='bot.cogs.token_remover', level=logging.DEBUG) as cm: - self.assertIsNone(asyncio.run(coroutine)) # no return value - - [line] = cm.output - log_message = ( - "Censored a seemingly valid token sent by " - "lemon (`42`) in #lemonade-stand, " - f"token was `{censored_token}`" - ) - self.assertIn(log_message, line) - - self.msg.delete.assert_called_once_with() - self.msg.channel.send.assert_called_once_with( - DELETION_MESSAGE_TEMPLATE.format(mention='@lemon') - ) - self.bot.get_cog.assert_called_with('ModLog') - self.msg.author.avatar_url_as.assert_called_once_with(static_format='png') - - mod_log = self.bot.get_cog.return_value - mod_log.ignore.assert_called_once_with(Event.message_delete, self.msg.id) - mod_log.send_log_message.assert_called_once_with( - icon_url=Icons.token_removed, - colour=Colour(Colours.soft_red), - title="Token removed!", - text=log_message, - thumbnail='picture-lemon.png', - channel_id=Channels.mod_alerts - ) - class TokenRemoverSetupTests(unittest.TestCase): """Tests setup of the `TokenRemover` cog.""" -- cgit v1.2.3 From 5734a4d84922a9497014dfeb3eba31ad3c57536f Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 11 May 2020 11:44:08 -0700 Subject: Refactor `TokenRemoverSetupTests` and add a more thorough test The test now ensures the cog is instantiated and that the instance is passed as an argument to `add_cog`. --- tests/bot/cogs/test_token_remover.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 3546e7964..c377de7b2 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -262,11 +262,15 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): ) -class TokenRemoverSetupTests(unittest.TestCase): - """Tests setup of the `TokenRemover` cog.""" +class TokenRemoverExtensionTests(unittest.TestCase): + """Tests for the token_remover extension.""" - def test_setup(self): - """Setup of the extension should call add_cog.""" + @autospec("bot.cogs.token_remover", "TokenRemover") + def test_extension_setup(self, cog): + """The TokenRemover cog should be added.""" bot = MockBot() setup_cog(bot) + + cog.assert_called_once_with(bot) bot.add_cog.assert_called_once() + self.assertTrue(isinstance(bot.add_cog.call_args.args[0], TokenRemover)) -- cgit v1.2.3 From d0303d715d485842a2d5c906099d767d74cf8bd8 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 11 May 2020 11:45:50 -0700 Subject: Replace deprecated assertion methods --- tests/bot/cogs/test_token_remover.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index c377de7b2..aecb51403 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -150,7 +150,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): for token in tokens: with self.subTest(token=token): results = TOKEN_RE.findall(token) - self.assertEquals(len(results), 0) + self.assertEqual(len(results), 0) def test_regex_valid_tokens(self): """Messages that look like tokens should be matched.""" @@ -171,7 +171,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): message = f"garbage {tokens[0]} hello {tokens[1]} world" results = TOKEN_RE.findall(message) - self.assertEquals(tokens, results) + self.assertEqual(tokens, results) @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") def test_is_maybe_token_missing_part_returns_false(self, valid_user, valid_time): -- cgit v1.2.3 From 862153f2e4ab5b1408719fb2c1abc5143cfb15ce Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 11 May 2020 11:47:40 -0700 Subject: Clean up token remover test imports --- tests/bot/cogs/test_token_remover.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index aecb51403..5cc8c7ad1 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -4,14 +4,10 @@ from unittest.mock import MagicMock from discord import Colour +from bot import constants +from bot.cogs import token_remover from bot.cogs.moderation import ModLog -from bot.cogs.token_remover import ( - DELETION_MESSAGE_TEMPLATE, - TOKEN_RE, - TokenRemover, - setup as setup_cog, -) -from bot.constants import Channels, Colours, Event, Icons +from bot.cogs.token_remover import TokenRemover from tests.helpers import MockBot, MockMessage, autospec @@ -149,7 +145,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): for token in tokens: with self.subTest(token=token): - results = TOKEN_RE.findall(token) + results = token_remover.TOKEN_RE.findall(token) self.assertEqual(len(results), 0) def test_regex_valid_tokens(self): @@ -162,7 +158,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): for token in tokens: with self.subTest(token=token): - results = TOKEN_RE.findall(token) + results = token_remover.TOKEN_RE.findall(token) self.assertIn(token, results) def test_regex_matches_multiple_valid(self): @@ -170,7 +166,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): tokens = ["x.y.z", "a.b.c"] message = f"garbage {tokens[0]} hello {tokens[1]} world" - results = TOKEN_RE.findall(message) + results = token_remover.TOKEN_RE.findall(message) self.assertEqual(tokens, results) @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") @@ -212,7 +208,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): self.msg.delete.assert_called_once_with() self.msg.channel.send.assert_called_once_with( - DELETION_MESSAGE_TEMPLATE.format(mention=self.msg.author.mention) + token_remover.DELETION_MESSAGE_TEMPLATE.format(mention=self.msg.author.mention) ) @autospec("bot.cogs.token_remover", "LOG_MESSAGE") @@ -251,14 +247,14 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): logger.debug.assert_called_with(log_msg) self.bot.stats.incr.assert_called_once_with("tokens.removed_tokens") - mod_log.ignore.assert_called_once_with(Event.message_delete, self.msg.id) + mod_log.ignore.assert_called_once_with(constants.Event.message_delete, self.msg.id) mod_log.send_log_message.assert_called_once_with( - icon_url=Icons.token_removed, - colour=Colour(Colours.soft_red), + icon_url=constants.Icons.token_removed, + colour=Colour(constants.Colours.soft_red), title="Token removed!", text=log_msg, thumbnail=self.msg.author.avatar_url_as.return_value, - channel_id=Channels.mod_alerts + channel_id=constants.Channels.mod_alerts ) @@ -269,7 +265,7 @@ class TokenRemoverExtensionTests(unittest.TestCase): def test_extension_setup(self, cog): """The TokenRemover cog should be added.""" bot = MockBot() - setup_cog(bot) + token_remover.setup(bot) cog.assert_called_once_with(bot) bot.add_cog.assert_called_once() -- cgit v1.2.3 From 4701b0da36c7f42792c0af258b785076237fd661 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 11 May 2020 11:56:15 -0700 Subject: Use subtests for valid ID/timestamp tests and test non-ASCII inputs --- tests/bot/cogs/test_token_remover.py | 43 +++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 5cc8c7ad1..f1a56c235 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -24,24 +24,31 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): self.msg.author.__str__ = MagicMock(return_value=self.msg.author.name) self.msg.author.avatar_url_as.return_value = "picture-lemon.png" - def test_is_valid_user_id_is_true_for_numeric_content(self): - """A string decoding to numeric characters is a valid user ID.""" - # MTIz = base64(123) - self.assertTrue(TokenRemover.is_valid_user_id('MTIz')) - - def test_is_valid_user_id_is_false_for_alphabetic_content(self): - """A string decoding to alphabetic characters is not a valid user ID.""" - # YWJj = base64(abc) - self.assertFalse(TokenRemover.is_valid_user_id('YWJj')) - - def test_is_valid_timestamp_is_true_for_valid_timestamps(self): - """A string decoding to a valid timestamp should be recognized as such.""" - self.assertTrue(TokenRemover.is_valid_timestamp('DN9r_A')) - - def test_is_valid_timestamp_is_false_for_invalid_values(self): - """A string not decoding to a valid timestamp should not be recognized as such.""" - # MTIz = base64(123) - self.assertFalse(TokenRemover.is_valid_timestamp('MTIz')) + def test_is_valid_user_id(self): + """Should correctly discern valid user IDs and ignore non-numeric and non-ASCII IDs.""" + subtests = ( + ("MTIz", True), # base64(123) + ("YWJj", False), # base64(abc) + ("λδµ", False), + ) + + for user_id, is_valid in subtests: + with self.subTest(user_id=user_id, is_valid=is_valid): + result = TokenRemover.is_valid_user_id(user_id) + self.assertIs(result, is_valid) + + def test_is_valid_timestamp(self): + """Should correctly discern valid timestamps.""" + subtests = ( + ("DN9r_A", True), + ("MTIz", False), # base64(123) + ("λδµ", False), + ) + + for timestamp, is_valid in subtests: + with self.subTest(timestamp=timestamp, is_valid=is_valid): + result = TokenRemover.is_valid_timestamp(timestamp) + self.assertIs(result, is_valid) def test_mod_log_property(self): """The `mod_log` property should ask the bot to return the `ModLog` cog.""" -- cgit v1.2.3 From ddfe583d0b1e72f98855f628ff01b72c82fa491d Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 21:56:39 +0200 Subject: AntiMalware Refactor - Moved embed descriptions into constants, added tests for embed descriptions --- bot/cogs/antimalware.py | 44 ++++++++++++++++++++-------------- tests/bot/cogs/test_antimalware.py | 48 ++++++++++++++++++++++++-------------- 2 files changed, 56 insertions(+), 36 deletions(-) diff --git a/bot/cogs/antimalware.py b/bot/cogs/antimalware.py index f5fd5e2d9..ea257442e 100644 --- a/bot/cogs/antimalware.py +++ b/bot/cogs/antimalware.py @@ -10,6 +10,27 @@ from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLE log = logging.getLogger(__name__) +PY_EMBED_DESCRIPTION = ( + "It looks like you tried to attach a Python file - " + f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" +) + +TXT_EMBED_DESCRIPTION = ( + "**Uh-oh!** It looks like your message got zapped by our spam filter. " + "We currently don't allow `.txt` attachments, so here are some tips to help you travel safely: \n\n" + "• If you attempted to send a message longer than 2000 characters, try shortening your message " + "to fit within the character limit or use a pasting service (see below) \n\n" + "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " + "{cmd_channel_mention} for more information) or use a pasting service like: " + f"\n\n{URLs.site_schema}{URLs.site_paste}" +) + +DISALLOWED_EMBED_DESCRIPTION = ( + "It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " + f"We currently allow the following file types: **{', '.join(AntiMalwareConfig.whitelist)}**.\n\n" + "Feel free to ask in {meta_channel_mention} if you think this is a mistake." +) + class AntiMalware(Cog): """Delete messages which contain attachments with non-whitelisted file extensions.""" @@ -34,29 +55,16 @@ class AntiMalware(Cog): blocked_extensions_str = ', '.join(extensions_blocked) if ".py" in extensions_blocked: # Short-circuit on *.py files to provide a pastebin link - embed.description = ( - "It looks like you tried to attach a Python file - " - f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" - ) + embed.description = PY_EMBED_DESCRIPTION elif ".txt" in extensions_blocked: # Work around Discord AutoConversion of messages longer than 2000 chars to .txt cmd_channel = self.bot.get_channel(Channels.bot_commands) - embed.description = ( - "**Uh-oh!** It looks like your message got zapped by our spam filter. " - "We currently don't allow `.txt` attachments, so here are some tips to help you travel safely: \n\n" - "• If you attempted to send a message longer than 2000 characters, try shortening your message " - "to fit within the character limit or use a pasting service (see below) \n\n" - "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " - f"{cmd_channel.mention} for more information) or use a pasting service like: " - f"\n\n{URLs.site_schema}{URLs.site_paste}" - ) + embed.description = TXT_EMBED_DESCRIPTION.format(cmd_channel_mention=cmd_channel.mention) elif extensions_blocked: - whitelisted_types = ', '.join(AntiMalwareConfig.whitelist) meta_channel = self.bot.get_channel(Channels.meta) - embed.description = ( - f"It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " - f"We currently allow the following file types: **{whitelisted_types}**.\n\n" - f"Feel free to ask in {meta_channel.mention} if you think this is a mistake." + embed.description = DISALLOWED_EMBED_DESCRIPTION.format( + blocked_extensions_str=blocked_extensions_str, + meta_channel_mention=meta_channel.mention, ) if embed.description: diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index a2ce9a740..fab063201 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -4,7 +4,7 @@ from unittest.mock import AsyncMock, Mock from discord import NotFound from bot.cogs import antimalware -from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES, URLs +from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole MODULE = "bot.cogs.antimalware" @@ -63,7 +63,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.message.delete.assert_not_called() - async def test_python_file_redirect_embed(self): + async def test_python_file_redirect_embed_description(self): """A message containing a .py file should result in an embed redirecting the user to our paste site""" attachment = MockAttachment(filename="python.py") self.message.attachments = [attachment] @@ -74,32 +74,44 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): args, kwargs = self.message.channel.send.call_args embed = kwargs.pop("embed") - self.assertEqual(args[0], f"Hey {self.message.author.mention}!") - self.assertEqual(embed.description, ( - "It looks like you tried to attach a Python file - " - f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" - )) + self.assertEqual(embed.description, antimalware.PY_EMBED_DESCRIPTION) - async def test_txt_file_redirect_embed(self): + async def test_txt_file_redirect_embed_description(self): + """A message containing a .txt file should result in the correct embed.""" attachment = MockAttachment(filename="python.txt") self.message.attachments = [attachment] self.message.channel.send = AsyncMock() + antimalware.TXT_EMBED_DESCRIPTION = Mock() + antimalware.TXT_EMBED_DESCRIPTION.format.return_value = "test" await self.cog.on_message(self.message) + self.message.channel.send.assert_called_once() args, kwargs = self.message.channel.send.call_args embed = kwargs.pop("embed") cmd_channel = self.bot.get_channel(Channels.bot_commands) - self.assertEqual(args[0], f"Hey {self.message.author.mention}!") - self.assertEqual(embed.description, ( - "**Uh-oh!** It looks like your message got zapped by our spam filter. " - "We currently don't allow `.txt` attachments, so here are some tips to help you travel safely: \n\n" - "• If you attempted to send a message longer than 2000 characters, try shortening your message " - "to fit within the character limit or use a pasting service (see below) \n\n" - "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " - f"{cmd_channel.mention} for more information) or use a pasting service like: " - f"\n\n{URLs.site_schema}{URLs.site_paste}" - )) + self.assertEqual(embed.description, antimalware.TXT_EMBED_DESCRIPTION.format.return_value) + antimalware.TXT_EMBED_DESCRIPTION.format.assert_called_with(cmd_channel_mention=cmd_channel.mention) + + async def test_other_disallowed_extention_embed_description(self): + """Test the description for a non .py/.txt disallowed extension.""" + attachment = MockAttachment(filename="python.disallowed") + self.message.attachments = [attachment] + self.message.channel.send = AsyncMock() + antimalware.DISALLOWED_EMBED_DESCRIPTION = Mock() + antimalware.DISALLOWED_EMBED_DESCRIPTION.format.return_value = "test" + + await self.cog.on_message(self.message) + self.message.channel.send.assert_called_once() + args, kwargs = self.message.channel.send.call_args + embed = kwargs.pop("embed") + meta_channel = self.bot.get_channel(Channels.meta) + + self.assertEqual(embed.description, antimalware.DISALLOWED_EMBED_DESCRIPTION.format.return_value) + antimalware.DISALLOWED_EMBED_DESCRIPTION.format.assert_called_with( + blocked_extensions_str=".disallowed", + meta_channel_mention=meta_channel.mention + ) async def test_removing_deleted_message_logs(self): """Removing an already deleted message logs the correct message""" -- cgit v1.2.3 From d7e6bed7b5b0f61312165e5b0b2b9291cd8df0c9 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 13 May 2020 14:17:51 +0300 Subject: Add message publishing to `Reddit` cog --- bot/cogs/reddit.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index 5a7fa100f..371b65434 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -218,7 +218,10 @@ class Reddit(Cog): for subreddit in RedditConfig.subreddits: top_posts = await self.get_top_posts(subreddit=subreddit, time="day") - await self.webhook.send(username=f"{subreddit} Top Daily Posts", embed=top_posts) + message = await self.webhook.send(username=f"{subreddit} Top Daily Posts", embed=top_posts) + + if message.channel.is_news(): + await message.publish() async def top_weekly_posts(self) -> None: """Post a summary of the top posts.""" @@ -242,6 +245,9 @@ class Reddit(Cog): await message.pin() + if message.channel.is_news(): + await message.publish() + @group(name="reddit", invoke_without_command=True) async def reddit_group(self, ctx: Context) -> None: """View the top posts from various subreddits.""" -- cgit v1.2.3 From 31aff51655d3783bc70f04628f189cf3c3591028 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Wed, 13 May 2020 18:58:43 -0700 Subject: Fix a test needlessly being a coroutine --- tests/bot/cogs/test_token_remover.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index f1a56c235..8e743a715 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -219,7 +219,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): ) @autospec("bot.cogs.token_remover", "LOG_MESSAGE") - async def test_format_log_message(self, log_message): + def test_format_log_message(self, log_message): """Should correctly format the log message with info from the message and token.""" log_message.format.return_value = "Howdy" return_value = TokenRemover.format_log_message(self.msg, "MTIz.DN9R_A.xyz") -- cgit v1.2.3 From ab44bb38d874dfdec9d7dc61bbf13b06144b9a0e Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Wed, 13 May 2020 19:18:50 -0700 Subject: Add missing comma to token remover log message --- bot/cogs/token_remover.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index c576a67d0..c57e7764e 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -17,7 +17,7 @@ from bot.constants import Channels, Colours, Event, Icons log = logging.getLogger(__name__) LOG_MESSAGE = ( - "Censored a seemingly valid token sent by {author} (`{author_id}`) in {channel}," + "Censored a seemingly valid token sent by {author} (`{author_id}`) in {channel}, " "token was `{user_id}.{timestamp}.{hmac}`" ) DELETION_MESSAGE_TEMPLATE = ( -- cgit v1.2.3 From 297089cde278ea09a27240f71f41006fab2b2ca4 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Wed, 13 May 2020 19:36:44 -0700 Subject: Token remover: add logs to clarify why token is invalid --- bot/cogs/token_remover.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index c57e7764e..244d52edb 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -133,12 +133,14 @@ class TokenRemover(Cog): try: user_id, creation_timestamp, hmac = test_str.split('.') except ValueError: + log.debug(f"Invalid token format in '{test_str}': does not have all 3 parts.") return False if cls.is_valid_user_id(user_id) and cls.is_valid_timestamp(creation_timestamp): return True - - return False + else: + log.debug(f"Invalid user ID or timestamp in '{test_str}'.") + return False @staticmethod def is_valid_user_id(b64_content: str) -> bool: -- cgit v1.2.3 From 206aed70f6185057ccbe4f8478ed456e5ab0c197 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 14 May 2020 16:58:55 +0300 Subject: Python News: Implement stats Add stat increaser to PEP and maillist posting. --- bot/cogs/python_news.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/cogs/python_news.py b/bot/cogs/python_news.py index 57ce61638..d28af4a0b 100644 --- a/bot/cogs/python_news.py +++ b/bot/cogs/python_news.py @@ -109,6 +109,9 @@ class PythonNews(Cog): ) payload["data"]["pep"].append(pep_nr) + # Increase overall PEP new stat + self.bot.stats.incr("python_news.posted.pep") + if msg.channel.is_news(): log.trace("Publishing PEP annnouncement because it was in a news channel") await msg.publish() @@ -168,6 +171,9 @@ class PythonNews(Cog): ) payload["data"][maillist].append(thread_information["thread_id"]) + # Increase this specific maillist counter in stats + self.bot.stats.incr(f"python_news.posted.{maillist.replace('-', '_')}") + if msg.channel.is_news(): log.trace("Publishing mailing list message because it was in a news channel") await msg.publish() -- cgit v1.2.3 From 73bcb2b434a30761494bbedd914508964c6fbbad Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 14 May 2020 10:34:37 -0700 Subject: Token remover: fix timestamp check The timestamp calculation was incorrect. The bytes need to be interpreted as big-endian and the result is just a timestamp rather than a snowflake. --- bot/cogs/token_remover.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index 244d52edb..957c8a690 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -2,13 +2,10 @@ import base64 import binascii import logging import re -import struct import typing as t -from datetime import datetime from discord import Colour, Message from discord.ext.commands import Cog -from discord.utils import snowflake_time from bot.bot import Bot from bot.cogs.moderation import ModLog @@ -29,7 +26,7 @@ DELETION_MESSAGE_TEMPLATE = ( "Feel free to re-post it with the token removed. " "If you believe this was a mistake, please let us know!" ) -DISCORD_EPOCH_TIMESTAMP = datetime(2017, 1, 1) +DISCORD_EPOCH = 1_420_070_400_000 TOKEN_EPOCH = 1_293_840_000 TOKEN_RE = re.compile( r"[^\s\.()\"']+" # Matches token part 1: The user ID string, encoded as base64 @@ -160,18 +157,27 @@ class TokenRemover(Cog): @staticmethod def is_valid_timestamp(b64_content: str) -> bool: """ - Check potential token to see if it contains a valid timestamp. + Return True if `b64_content` decodes to a valid timestamp. - See: https://discordapp.com/developers/docs/reference#snowflakes + If the timestamp is greater than the Discord epoch, it's probably valid. + See: https://i.imgur.com/7WdehGn.png """ b64_content += '=' * (-len(b64_content) % 4) try: - content = base64.urlsafe_b64decode(b64_content) - snowflake = struct.unpack('i', content)[0] - except (binascii.Error, struct.error, ValueError): + decoded_bytes = base64.urlsafe_b64decode(b64_content) + timestamp = int.from_bytes(decoded_bytes, byteorder="big") + except (binascii.Error, ValueError) as e: + log.debug(f"Failed to decode token timestamp '{b64_content}': {e}") + return False + + # Seems like newer tokens don't need the epoch added, but add anyway since an upper bound + # is not checked. + if timestamp + TOKEN_EPOCH >= DISCORD_EPOCH: + return True + else: + log.debug(f"Invalid token timestamp '{b64_content}': smaller than Discord epoch") return False - return snowflake_time(snowflake + TOKEN_EPOCH) < DISCORD_EPOCH_TIMESTAMP def setup(bot: Bot) -> None: -- cgit v1.2.3 From 4a73c24678d4a893304f0b2f3a5f1e326cae817a Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 15 May 2020 08:54:36 -0700 Subject: Token remover: use strict check for digits in token ID `isnumeric` would be true for a wide range of characters in Unicode, but the ID must only consist of the characters 0-9 (ASCII digits). In fact, `isdigit` on its own would also match other Unicode characters too. --- bot/cogs/token_remover.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index 957c8a690..43c12c4f7 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -149,8 +149,11 @@ class TokenRemover(Cog): b64_content += '=' * (-len(b64_content) % 4) try: - content: bytes = base64.b64decode(b64_content) - return content.decode('utf-8').isnumeric() + decoded_bytes: bytes = base64.b64decode(b64_content) + string = decoded_bytes.decode('utf-8') + + # isdigit on its own would match a lot of other Unicode characters, hence the isascii. + return string.isascii() and string.isdigit() except (binascii.Error, ValueError): return False -- cgit v1.2.3 From d2c538e23c20c5c4b22d7b2eb2bcf03067593374 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Fri, 15 May 2020 19:33:24 +0200 Subject: Increase snekbox re eval timeout. --- bot/cogs/snekbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 8d4688114..611e80f61 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -227,7 +227,7 @@ class Snekbox(Cog): _, new_message = await self.bot.wait_for( 'message_edit', check=_predicate_eval_message_edit, - timeout=10 + timeout=30 ) await ctx.message.add_reaction(REEVAL_EMOJI) await self.bot.wait_for( -- cgit v1.2.3 From 1c06d2a9d873ced2e54bf16a96573a46c583c12f Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Fri, 15 May 2020 19:51:19 +0200 Subject: Move the re eval timeout to a module constant --- bot/cogs/snekbox.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 611e80f61..9fa75a929 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -47,6 +47,7 @@ EVAL_ROLES = (Roles.helpers, Roles.moderators, Roles.admins, Roles.owners, Roles SIGKILL = 9 REEVAL_EMOJI = '\U0001f501' # :repeat: +REEVAL_TIMEOUT = 30 class Snekbox(Cog): @@ -227,7 +228,7 @@ class Snekbox(Cog): _, new_message = await self.bot.wait_for( 'message_edit', check=_predicate_eval_message_edit, - timeout=30 + timeout=REEVAL_TIMEOUT ) await ctx.message.add_reaction(REEVAL_EMOJI) await self.bot.wait_for( -- cgit v1.2.3 From 5a48ed0d60ebc9984cae27b19953b50b52df83d9 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Fri, 15 May 2020 19:52:26 +0200 Subject: Change tests to use the new timeout constant --- tests/bot/cogs/test_snekbox.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_snekbox.py b/tests/bot/cogs/test_snekbox.py index 1dec0ccaf..ccc090f02 100644 --- a/tests/bot/cogs/test_snekbox.py +++ b/tests/bot/cogs/test_snekbox.py @@ -291,7 +291,11 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(actual, expected) self.bot.wait_for.assert_has_awaits( ( - call('message_edit', check=partial_mock(snekbox.predicate_eval_message_edit, ctx), timeout=10), + call( + 'message_edit', + check=partial_mock(snekbox.predicate_eval_message_edit, ctx), + timeout=snekbox.REEVAL_TIMEOUT, + ), call('reaction_add', check=partial_mock(snekbox.predicate_eval_emoji_reaction, ctx), timeout=10) ) ) -- cgit v1.2.3 From 3aeb9e6ac6258fe3446788fc8a731fc8bb5922d4 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 12:23:52 +0200 Subject: Adding redis to docker-compose file. This is almost hilariously easy since we can just use the official image for it. --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 11deceae8..1bcf1008e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,9 @@ services: POSTGRES_PASSWORD: pysite POSTGRES_USER: pysite + redis: + image: redis:5.0.9 + web: image: pythondiscord/site:latest command: ["run", "--debug"] @@ -41,6 +44,7 @@ services: tty: true depends_on: - web + - redis environment: BOT_TOKEN: ${BOT_TOKEN} BOT_API_KEY: badbot13m0n8f570f942013fc818f234916ca531 -- cgit v1.2.3 From 5382dd80611dcf24124477f0e09dfde668df1ace Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 12:24:34 +0200 Subject: Adding redis-py to the Pipfile This is the module we will be using to interface with Redis. --- Pipfile | 1 + Pipfile.lock | 131 ++++++++++++++++++++++++++++++----------------------------- 2 files changed, 67 insertions(+), 65 deletions(-) diff --git a/Pipfile b/Pipfile index 14c9ef926..5f85b1e51 100644 --- a/Pipfile +++ b/Pipfile @@ -23,6 +23,7 @@ colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} statsd = "~=3.3" feedparser = "~=5.2" beautifulsoup4 = "~=4.9" +redis = ">=3.5" [dev-packages] coverage = "~=5.0" diff --git a/Pipfile.lock b/Pipfile.lock index 4e7050a13..1a420182d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "64620e7e825c74fd3010821fb30843b19f5dafb2b5a1f6eafedc0a5febd99b69" + "sha256": "c6b4d38c4034e55a4bd598399f2e1f48b70a76693c986d0db0fae7442e224d41" }, "pipfile-spec": 6, "requires": { @@ -52,10 +52,10 @@ }, "aiormq": { "hashes": [ - "sha256:286e0b0772075580466e45f98f051b9728a9316b9c36f0c14c7bc1409be375b0", - "sha256:7ed7d6df6b57af7f8bce7d1ebcbdfc32b676192e46703e81e9e217316e56b5bd" + "sha256:41a9d4eb17db805f30ed172f3f609fe0c2b16657fb15b1b67df19d251dd93c0d", + "sha256:7c19477a9450824cb79f9949fd238f4148e2c0dca67756a2868863c387209f04" ], - "version": "==3.2.1" + "version": "==3.2.2" }, "alabaster": { "hashes": [ @@ -305,39 +305,39 @@ }, "more-itertools": { "hashes": [ - "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c", - "sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507" + "sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be", + "sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982" ], "index": "pypi", - "version": "==8.2.0" + "version": "==8.3.0" }, "multidict": { "hashes": [ - "sha256:317f96bc0950d249e96d8d29ab556d01dd38888fbe68324f46fd834b430169f1", - "sha256:42f56542166040b4474c0c608ed051732033cd821126493cf25b6c276df7dd35", - "sha256:4b7df040fb5fe826d689204f9b544af469593fb3ff3a069a6ad3409f742f5928", - "sha256:544fae9261232a97102e27a926019100a9db75bec7b37feedd74b3aa82f29969", - "sha256:620b37c3fea181dab09267cd5a84b0f23fa043beb8bc50d8474dd9694de1fa6e", - "sha256:6e6fef114741c4d7ca46da8449038ec8b1e880bbe68674c01ceeb1ac8a648e78", - "sha256:7774e9f6c9af3f12f296131453f7b81dabb7ebdb948483362f5afcaac8a826f1", - "sha256:85cb26c38c96f76b7ff38b86c9d560dea10cf3459bb5f4caf72fc1bb932c7136", - "sha256:a326f4240123a2ac66bb163eeba99578e9d63a8654a59f4688a79198f9aa10f8", - "sha256:ae402f43604e3b2bc41e8ea8b8526c7fa7139ed76b0d64fc48e28125925275b2", - "sha256:aee283c49601fa4c13adc64c09c978838a7e812f85377ae130a24d7198c0331e", - "sha256:b51249fdd2923739cd3efc95a3d6c363b67bbf779208e9f37fd5e68540d1a4d4", - "sha256:bb519becc46275c594410c6c28a8a0adc66fe24fef154a9addea54c1adb006f5", - "sha256:c2c37185fb0af79d5c117b8d2764f4321eeb12ba8c141a95d0aa8c2c1d0a11dd", - "sha256:dc561313279f9d05a3d0ffa89cd15ae477528ea37aa9795c4654588a3287a9ab", - "sha256:e439c9a10a95cb32abd708bb8be83b2134fa93790a4fb0535ca36db3dda94d20", - "sha256:fc3b4adc2ee8474cb3cd2a155305d5f8eda0a9c91320f83e55748e1fcb68f8e3" - ], - "version": "==4.7.5" + "sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a", + "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000", + "sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2", + "sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507", + "sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5", + "sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7", + "sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d", + "sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463", + "sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19", + "sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3", + "sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b", + "sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c", + "sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87", + "sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7", + "sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430", + "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255", + "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d" + ], + "version": "==4.7.6" }, "ordered-set": { "hashes": [ - "sha256:a7bfa858748c73b096e43db14eb23e2bc714a503f990c89fac8fab9b0ee79724" + "sha256:a31008c57f9c9776b12eb8841b1f61d1e4d70dfbbe8875ccfa2403c54af3d51b" ], - "version": "==3.1.1" + "version": "==4.0.1" }, "packaging": { "hashes": [ @@ -418,10 +418,10 @@ }, "pytz": { "hashes": [ - "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", - "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" + "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", + "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048" ], - "version": "==2019.3" + "version": "==2020.1" }, "pyyaml": { "hashes": [ @@ -440,6 +440,14 @@ "index": "pypi", "version": "==5.3.1" }, + "redis": { + "hashes": [ + "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", + "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" + ], + "index": "pypi", + "version": "==3.5.2" + }, "requests": { "hashes": [ "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", @@ -450,11 +458,11 @@ }, "sentry-sdk": { "hashes": [ - "sha256:23808d571d2461a4ce3784ec12bbee5bdb8c026c143fe79d36cef8a6d653e71f", - "sha256:bb90a4e19c7233a580715fc986cc44be2c48fc10b31e71580a2037e1c94b6950" + "sha256:0e5e947d0f7a969314aa23669a94a9712be5a688ff069ff7b9fc36c66adc160c", + "sha256:799a8bf76b012e3030a881be00e97bc0b922ce35dde699c6537122b751d80e2c" ], "index": "pypi", - "version": "==0.14.3" + "version": "==0.14.4" }, "six": { "hashes": [ @@ -595,10 +603,10 @@ "develop": { "appdirs": { "hashes": [ - "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", - "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" ], - "version": "==1.4.3" + "version": "==1.4.4" }, "attrs": { "hashes": [ @@ -657,13 +665,6 @@ ], "version": "==0.3.0" }, - "entrypoints": { - "hashes": [ - "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", - "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" - ], - "version": "==0.3" - }, "filelock": { "hashes": [ "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", @@ -673,11 +674,11 @@ }, "flake8": { "hashes": [ - "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb", - "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca" + "sha256:6c1193b0c3f853ef763969238f6c81e9e63ace9d024518edc020d5f1d6d93195", + "sha256:ea6623797bf9a52f4c9577d780da0bb17d65f870213f7b5bcc9fca82540c31d5" ], "index": "pypi", - "version": "==3.7.9" + "version": "==3.8.1" }, "flake8-annotations": { "hashes": [ @@ -743,10 +744,10 @@ }, "identify": { "hashes": [ - "sha256:2bb8760d97d8df4408f4e805883dad26a2d076f04be92a10a3e43f09c6060742", - "sha256:faffea0fd8ec86bb146ac538ac350ed0c73908326426d387eded0bcc9d077522" + "sha256:23c18d97bb50e05be1a54917ee45cc61d57cb96aedc06aabb2b02331edf0dbf0", + "sha256:88ed90632023e52a6495749c6732e61e08ec9f4f04e95484a5c37b9caf40283c" ], - "version": "==1.4.14" + "version": "==1.4.15" }, "mccabe": { "hashes": [ @@ -771,18 +772,18 @@ }, "pre-commit": { "hashes": [ - "sha256:487c675916e6f99d355ec5595ad77b325689d423ef4839db1ed2f02f639c9522", - "sha256:c0aa11bce04a7b46c5544723aedf4e81a4d5f64ad1205a30a9ea12d5e81969e1" + "sha256:5559e09afcac7808933951ffaf4ff9aac524f31efbc3f24d021540b6c579813c", + "sha256:703e2e34cbe0eedb0d319eff9f7b83e2022bb5a3ab5289a6a8841441076514d0" ], "index": "pypi", - "version": "==2.2.0" + "version": "==2.4.0" }, "pycodestyle": { "hashes": [ - "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", - "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" + "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367", + "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e" ], - "version": "==2.5.0" + "version": "==2.6.0" }, "pydocstyle": { "hashes": [ @@ -793,10 +794,10 @@ }, "pyflakes": { "hashes": [ - "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", - "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" + "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92", + "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8" ], - "version": "==2.1.1" + "version": "==2.2.0" }, "pyyaml": { "hashes": [ @@ -831,10 +832,10 @@ }, "toml": { "hashes": [ - "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", - "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" + "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", + "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88" ], - "version": "==0.10.0" + "version": "==0.10.1" }, "unittest-xml-reporting": { "hashes": [ @@ -846,10 +847,10 @@ }, "virtualenv": { "hashes": [ - "sha256:5021396e8f03d0d002a770da90e31e61159684db2859d0ba4850fbea752aa675", - "sha256:ac53ade75ca189bc97b6c1d9ec0f1a50efe33cbf178ae09452dcd9fd309013c1" + "sha256:b4c14d4d73a0c23db267095383c4276ef60e161f94fde0427f2f21a0132dde74", + "sha256:fd0e54dec8ac96c1c7c87daba85f0a59a7c37fe38748e154306ca21c73244637" ], - "version": "==20.0.18" + "version": "==20.0.20" } } } -- cgit v1.2.3 From 7a501fdecaae186590177fd4ebd6cea64119629e Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 12:40:06 +0200 Subject: Boilerplate for the RedisCacheMixin We're using __init_subclass__ to initialize our RedisDict with the subclass name as a namespace. This will be prefixed to all data that we store, so that there won't be collisions between different subclasses. --- bot/mixins/__init__.py | 3 +++ bot/mixins/redis.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 bot/mixins/__init__.py create mode 100644 bot/mixins/redis.py diff --git a/bot/mixins/__init__.py b/bot/mixins/__init__.py new file mode 100644 index 000000000..ff1f0c50d --- /dev/null +++ b/bot/mixins/__init__.py @@ -0,0 +1,3 @@ +from .redis import RedisCacheMixin + +__all__ = ['RedisCacheMixin'] diff --git a/bot/mixins/redis.py b/bot/mixins/redis.py new file mode 100644 index 000000000..f19108576 --- /dev/null +++ b/bot/mixins/redis.py @@ -0,0 +1,56 @@ +import redis as redis_py + +redis = redis_py.Redis(host="redis") + + +class RedisDict(dict): + """ + A dictionary interface for a Redis database. + + Objects created by this class should mostly behave like a normal dictionary, + but will store all the data in our Redis database for persistence between restarts. + + There are, however, a few limitations to what kinds of data types can be + stored on Redis, so this is a little bit more limited than a regular dict. + """ + + def __init__(self, namespace: str = "global"): + """Initialize the RedisDict with the right namespace.""" + # TODO: Make namespace collision impossible! + # Append a number or something if it exists already. + self.namespace = namespace + + # redis.mset({"firedog": "donkeykong"}) + # + # print(redis.get("firedog").decode("utf-8") + + +class RedisCacheMixin: + """ + A mixin which adds a cls.cache parameter which can be used for persistent caching. + + This adds a dictionary-like object called cache which can be treated like a regular dictionary, + but which can only store simple data types like ints, strings, and floats. + + To use it, simply subclass it into your class like this: + + class MyCog(Cog, RedisCacheMixin): + def some_command(self): + # You can now do this! + self.cache['some_data'] = some_data + + All the data stored in this cache will probably be available permanently, even if the bot restarts or + is updated. However, Redis is not meant to be used for reliable, permanent storage. It may be cleared + from time to time, so please only use it for caching data that you can afford to lose. + + If it's really important that your data should never disappear, please use our postgres database instead. + """ + + def __init_subclass__(cls, **kwargs): + """ + Initialize the cache when subclass is created. + + When this mixin is subclassed, we create a cache using the subclass name as the namespace. + This is to prevent collisions between subclasses. + """ + cls.cache = RedisDict(cls.__name__) -- cgit v1.2.3 From 93cce50846b1cfcf520535d69a9fe223c2cd4d7a Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 16 May 2020 16:49:48 +0300 Subject: Stats: Create guild boost stat collection Collect Guild boost amount + level and post it to StatsD every hour in task. Added starting to cog `__init__.py` and stopping to `cog_unload`. --- bot/cogs/stats.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py index d253db913..acee1f5a9 100644 --- a/bot/cogs/stats.py +++ b/bot/cogs/stats.py @@ -2,8 +2,10 @@ import string from datetime import datetime from discord import Member, Message, Status -from discord.ext.commands import Bot, Cog, Context +from discord.ext.commands import Cog, Context +from discord.ext.tasks import loop +from bot.bot import Bot from bot.constants import Channels, Guild, Stats as StatConf @@ -23,6 +25,7 @@ class Stats(Cog): def __init__(self, bot: Bot): self.bot = bot self.last_presence_update = None + self.update_guild_boost.start() @Cog.listener() async def on_message(self, message: Message) -> None: @@ -101,6 +104,18 @@ class Stats(Cog): self.bot.stats.gauge("guild.status.do_not_disturb", dnd) self.bot.stats.gauge("guild.status.offline", offline) + @loop(hours=1) + async def update_guild_boost(self) -> None: + """Update every hour guild boosts amount + level.""" + await self.bot.wait_until_guild_available() + g = self.bot.get_guild(Guild.id) + self.bot.stats.gauge("boost.amount", g.premium_subscription_count) + self.bot.stats.gauge("boost.tier", g.premium_tier) + + def cog_unload(self) -> None: + """Stop guild boost stat collecting task on Cog unload.""" + self.update_guild_boost.stop() + def setup(bot: Bot) -> None: """Load the stats cog.""" -- cgit v1.2.3 From 1a6abaac12eb2e6ab0d26065108fe1cce9a7be45 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 16 May 2020 17:00:25 +0300 Subject: Stats: Added codeblock correction stats --- bot/cogs/bot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/cogs/bot.py b/bot/cogs/bot.py index a6929b431..67ff8f95d 100644 --- a/bot/cogs/bot.py +++ b/bot/cogs/bot.py @@ -326,6 +326,8 @@ class BotCog(Cog, name="Bot"): log.trace("The code consists only of expressions, not sending instructions") if howto != "": + # Increase amount of codeblock correction in stats + self.bot.stats.incr("codeblock_corrections") howto_embed = Embed(description=howto) bot_message = await msg.channel.send(f"Hey {msg.author.mention}!", embed=howto_embed) self.codeblock_message_ids[msg.id] = bot_message.id -- cgit v1.2.3 From 158e19a6fcb2056c6bcc244a1f02d8b75d7fe503 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 16 May 2020 17:13:54 +0300 Subject: Stats: Added stats for eval successes + fails --- bot/cogs/snekbox.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 8d4688114..1c64c893b 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -205,6 +205,12 @@ class Snekbox(Cog): if paste_link: msg = f"{msg}\nFull output: {paste_link}" + # Collect stats of eval fails + successes + if icon == ":x:": + self.bot.stats.incr("evals.fail") + elif icon in (":warning:", ":white_check_mark:"): + self.bot.stats.incr("evals.success") + response = await ctx.send(msg) self.bot.loop.create_task( wait_for_deletion(response, user_ids=(ctx.author.id,), client=ctx.bot) -- cgit v1.2.3 From 5878ec93c5b883038b0f738e9bbbdee8fd1929ad Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 16 May 2020 17:25:25 +0300 Subject: Stats: Added stats for eval role uses (Helpers/Developers) --- bot/cogs/snekbox.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 1c64c893b..04c0a5ae1 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -298,6 +298,11 @@ class Snekbox(Cog): await ctx.invoke(self.bot.get_command("help"), "eval") return + if Roles.helpers in (role.id for role in ctx.author.roles): + self.bot.stats.incr("evals.roles.helpers") + else: + self.bot.stats.incr("evals.roles.developers") + log.info(f"Received code from {ctx.author} for evaluation:\n{code}") while True: -- cgit v1.2.3 From 613b00a5ec060f409d5838cb1a648d9770cecfde Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 16 May 2020 18:19:18 +0300 Subject: Stats: Added stats for eval channel using (Help/Bot commands/Topical) --- bot/cogs/snekbox.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 04c0a5ae1..1d240d8d8 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -303,6 +303,13 @@ class Snekbox(Cog): else: self.bot.stats.incr("evals.roles.developers") + if ctx.channel.category_id == Categories.help_in_use: + self.bot.stats.incr("evals.channels.help") + elif ctx.channel.id == Channels.bot_commands: + self.bot.stats.incr("evals.channels.bot_commands") + else: + self.bot.stats.incr("evals.channels.topical") + log.info(f"Received code from {ctx.author} for evaluation:\n{code}") while True: -- cgit v1.2.3 From 588521c82403f6d66693512c6d33272cc370d755 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 20:22:05 +0200 Subject: Refactor - no more mixins! It was brought to my attention that we may need several caches per Cog for some of our Cogs. This means that the original approach of having this be a mixin is a little bit problematic. Instead, RedisDict will be instantiated directly inside the class you want it in. By leveraging __set_name__, we can create a namespace containing both the class name and the variable name without the user having to provide anything. For example, if you create an attribute MyClass.cache = RedisDict(), this will be using the redis namespace 'MyClass.cache.' before anything you store in it. With this approach, it is also possible to instantiate a RedisDict with a custom namespace by simply passing it into the constructor. - RedisDict("firedog") will create items with the 'firedog.your_item' prefix. - If there are multiple RedisDicts using the same namespace, an underscore will be appended to the namespace, such that the second RedisDict("firedog") will actually create items in the 'firedog_.your_item' namespace. This is also possible to use outside of classes, so long as you provide a custom namespace when you instantiate it. Custom namespaces will always take precedence over automatic 'Class.attribute_name' ones. --- bot/mixins/__init__.py | 3 --- bot/mixins/redis.py | 56 ---------------------------------------- bot/utils/__init__.py | 4 +++ bot/utils/redis.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 59 deletions(-) delete mode 100644 bot/mixins/__init__.py delete mode 100644 bot/mixins/redis.py create mode 100644 bot/utils/redis.py diff --git a/bot/mixins/__init__.py b/bot/mixins/__init__.py deleted file mode 100644 index ff1f0c50d..000000000 --- a/bot/mixins/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .redis import RedisCacheMixin - -__all__ = ['RedisCacheMixin'] diff --git a/bot/mixins/redis.py b/bot/mixins/redis.py deleted file mode 100644 index f19108576..000000000 --- a/bot/mixins/redis.py +++ /dev/null @@ -1,56 +0,0 @@ -import redis as redis_py - -redis = redis_py.Redis(host="redis") - - -class RedisDict(dict): - """ - A dictionary interface for a Redis database. - - Objects created by this class should mostly behave like a normal dictionary, - but will store all the data in our Redis database for persistence between restarts. - - There are, however, a few limitations to what kinds of data types can be - stored on Redis, so this is a little bit more limited than a regular dict. - """ - - def __init__(self, namespace: str = "global"): - """Initialize the RedisDict with the right namespace.""" - # TODO: Make namespace collision impossible! - # Append a number or something if it exists already. - self.namespace = namespace - - # redis.mset({"firedog": "donkeykong"}) - # - # print(redis.get("firedog").decode("utf-8") - - -class RedisCacheMixin: - """ - A mixin which adds a cls.cache parameter which can be used for persistent caching. - - This adds a dictionary-like object called cache which can be treated like a regular dictionary, - but which can only store simple data types like ints, strings, and floats. - - To use it, simply subclass it into your class like this: - - class MyCog(Cog, RedisCacheMixin): - def some_command(self): - # You can now do this! - self.cache['some_data'] = some_data - - All the data stored in this cache will probably be available permanently, even if the bot restarts or - is updated. However, Redis is not meant to be used for reliable, permanent storage. It may be cleared - from time to time, so please only use it for caching data that you can afford to lose. - - If it's really important that your data should never disappear, please use our postgres database instead. - """ - - def __init_subclass__(cls, **kwargs): - """ - Initialize the cache when subclass is created. - - When this mixin is subclassed, we create a cache using the subclass name as the namespace. - This is to prevent collisions between subclasses. - """ - cls.cache = RedisDict(cls.__name__) diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 9b32e515d..7ae2db8fe 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -2,6 +2,10 @@ from abc import ABCMeta from discord.ext.commands import CogMeta +from bot.utils.redis import RedisDict + +__all__ = ['RedisDict', 'CogABCMeta'] + class CogABCMeta(CogMeta, ABCMeta): """Metaclass for ABCs meant to be implemented as Cogs.""" diff --git a/bot/utils/redis.py b/bot/utils/redis.py new file mode 100644 index 000000000..8b33e8977 --- /dev/null +++ b/bot/utils/redis.py @@ -0,0 +1,70 @@ +from collections.abc import MutableMapping +from typing import Optional + +import redis as redis_py + +redis = redis_py.Redis(host="redis") + + +class RedisDict(MutableMapping): + """ + A dictionary interface for a Redis database. + + Objects created by this class should mostly behave like a normal dictionary, + but will store all the data in our Redis database for persistence between restarts. + + Redis is limited to simple types, so to allow you to store collections like lists + and dictionaries, we JSON deserialize every value. That means that it will not be possible + to store complex objects, only stuff like strings, numbers, and collections of strings and numbers. + + TODO: Implement these: + __delitem__ + __getitem__ + __setitem__ + __iter__ + __len__ + clear (just use DEL and the hash goes) + copy (convert to dict maybe?) + pop + popitem + setdefault + update + + TODO: TEST THESE + .get + .items + .keys + .values + .__eg__ + .__ne__ + """ + + namespaces = [] + + def _set_namespace(self, namespace: str) -> None: + """Try to set the namespace, but do not permit collisions.""" + while namespace in self.namespaces: + namespace = namespace + "_" + + self.namespaces.append(namespace) + self.namespace = namespace + + def __init__(self, namespace: Optional[str] = None) -> None: + """Initialize the RedisDict with the right namespace.""" + super().__init__() + self.has_custom_namespace = namespace is not None + self._set_namespace(namespace) + + def __set_name__(self, owner: object, attribute_name: str) -> None: + """ + Set the namespace to Class.attribute_name. + + Called automatically when this class is constructed inside a class as an attribute, as long as + no custom namespace is provided to the constructor. + """ + if not self.has_custom_namespace: + self._set_namespace(f"{owner.__name__}.{attribute_name}") + + def __repr__(self) -> str: + """Return a beautiful representation of this object instance.""" + return f"RedisDict(namespace={self.namespace!r})" -- cgit v1.2.3 From ee8386e67aa7d298b4761eef79b927c3066fe037 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 23:40:21 +0200 Subject: Add basic dict methods for RedisDict. The rest of the features should be provided by the MutableMapping abc we're interfacing. Specifically, MutableMapping provides these: .pop, .popitem, .clear, .update, .setdefault, __contains__, .keys, .items, .values, .get, __eq__, and __ne__. --- bot/utils/redis.py | 80 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/bot/utils/redis.py b/bot/utils/redis.py index 8b33e8977..1e8c6422d 100644 --- a/bot/utils/redis.py +++ b/bot/utils/redis.py @@ -1,9 +1,12 @@ +import json from collections.abc import MutableMapping -from typing import Optional +from enum import Enum +from typing import Dict, List, Optional, Tuple, Union import redis as redis_py -redis = redis_py.Redis(host="redis") +ValidRedisKey = Union[str, int, float] +JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] class RedisDict(MutableMapping): @@ -16,45 +19,25 @@ class RedisDict(MutableMapping): Redis is limited to simple types, so to allow you to store collections like lists and dictionaries, we JSON deserialize every value. That means that it will not be possible to store complex objects, only stuff like strings, numbers, and collections of strings and numbers. - - TODO: Implement these: - __delitem__ - __getitem__ - __setitem__ - __iter__ - __len__ - clear (just use DEL and the hash goes) - copy (convert to dict maybe?) - pop - popitem - setdefault - update - - TODO: TEST THESE - .get - .items - .keys - .values - .__eg__ - .__ne__ """ - namespaces = [] - - def _set_namespace(self, namespace: str) -> None: - """Try to set the namespace, but do not permit collisions.""" - while namespace in self.namespaces: - namespace = namespace + "_" - - self.namespaces.append(namespace) - self.namespace = namespace + _namespaces = [] + _redis = redis_py.Redis(host="redis") # Can be overridden for testing def __init__(self, namespace: Optional[str] = None) -> None: """Initialize the RedisDict with the right namespace.""" super().__init__() - self.has_custom_namespace = namespace is not None + self._has_custom_namespace = namespace is not None self._set_namespace(namespace) + def _set_namespace(self, namespace: str) -> None: + """Try to set the namespace, but do not permit collisions.""" + while namespace in self._namespaces: + namespace = namespace + "_" + + self._namespaces.append(namespace) + self._namespace = namespace + def __set_name__(self, owner: object, attribute_name: str) -> None: """ Set the namespace to Class.attribute_name. @@ -62,9 +45,36 @@ class RedisDict(MutableMapping): Called automatically when this class is constructed inside a class as an attribute, as long as no custom namespace is provided to the constructor. """ - if not self.has_custom_namespace: + if not self._has_custom_namespace: self._set_namespace(f"{owner.__name__}.{attribute_name}") def __repr__(self) -> str: """Return a beautiful representation of this object instance.""" - return f"RedisDict(namespace={self.namespace!r})" + return f"RedisDict(namespace={self._namespace!r})" + + def __setitem__(self, key: ValidRedisKey, value: JSONSerializableType): + """Store an item in the Redis cache.""" + # JSON serialize the value before storing it. + json_value = json.dumps(value) + self._redis.hset(self._namespace, key, json_value) + + def __getitem__(self, key: ValidRedisKey): + """Get an item from the Redis cache.""" + value = self._redis.hget(self._namespace, key) + return json.loads(value) + + def __delitem__(self, key: ValidRedisKey): + """Delete an item from the Redis cache.""" + self._redis.hdel(self._namespace, key) + + def __iter__(self): + """Iterate all the items in the Redis cache.""" + return iter(self._redis.hkeys(self._namespace)) + + def __len__(self): + """Return the number of items in the Redis cache.""" + return self._redis.hlen(self._namespace) + + def copy(self) -> Dict: + """Convert to dict and return.""" + return dict(self) -- cgit v1.2.3 From 5859711a05e634d7b788944b8292071cb1e7cf72 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 23:43:22 +0200 Subject: copy should dictify the .items(), not just keys. --- bot/utils/redis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/redis.py b/bot/utils/redis.py index 1e8c6422d..470de47b7 100644 --- a/bot/utils/redis.py +++ b/bot/utils/redis.py @@ -77,4 +77,4 @@ class RedisDict(MutableMapping): def copy(self) -> Dict: """Convert to dict and return.""" - return dict(self) + return dict(self.items()) -- cgit v1.2.3 From 9eeee1ce303b7ebac4fa9db37193921d052d0f8d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 00:10:29 +0200 Subject: Implements .clear with hash deletion. This would've been implemented by MutableMapping, but that implementation is O(n) instead of O(1) since it just iterates the entire hash and does HDEL. Feels wasteful. --- bot/utils/redis.py | 80 ---------------------------------------------- bot/utils/redis_dict.py | 84 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 80 deletions(-) delete mode 100644 bot/utils/redis.py create mode 100644 bot/utils/redis_dict.py diff --git a/bot/utils/redis.py b/bot/utils/redis.py deleted file mode 100644 index 470de47b7..000000000 --- a/bot/utils/redis.py +++ /dev/null @@ -1,80 +0,0 @@ -import json -from collections.abc import MutableMapping -from enum import Enum -from typing import Dict, List, Optional, Tuple, Union - -import redis as redis_py - -ValidRedisKey = Union[str, int, float] -JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] - - -class RedisDict(MutableMapping): - """ - A dictionary interface for a Redis database. - - Objects created by this class should mostly behave like a normal dictionary, - but will store all the data in our Redis database for persistence between restarts. - - Redis is limited to simple types, so to allow you to store collections like lists - and dictionaries, we JSON deserialize every value. That means that it will not be possible - to store complex objects, only stuff like strings, numbers, and collections of strings and numbers. - """ - - _namespaces = [] - _redis = redis_py.Redis(host="redis") # Can be overridden for testing - - def __init__(self, namespace: Optional[str] = None) -> None: - """Initialize the RedisDict with the right namespace.""" - super().__init__() - self._has_custom_namespace = namespace is not None - self._set_namespace(namespace) - - def _set_namespace(self, namespace: str) -> None: - """Try to set the namespace, but do not permit collisions.""" - while namespace in self._namespaces: - namespace = namespace + "_" - - self._namespaces.append(namespace) - self._namespace = namespace - - def __set_name__(self, owner: object, attribute_name: str) -> None: - """ - Set the namespace to Class.attribute_name. - - Called automatically when this class is constructed inside a class as an attribute, as long as - no custom namespace is provided to the constructor. - """ - if not self._has_custom_namespace: - self._set_namespace(f"{owner.__name__}.{attribute_name}") - - def __repr__(self) -> str: - """Return a beautiful representation of this object instance.""" - return f"RedisDict(namespace={self._namespace!r})" - - def __setitem__(self, key: ValidRedisKey, value: JSONSerializableType): - """Store an item in the Redis cache.""" - # JSON serialize the value before storing it. - json_value = json.dumps(value) - self._redis.hset(self._namespace, key, json_value) - - def __getitem__(self, key: ValidRedisKey): - """Get an item from the Redis cache.""" - value = self._redis.hget(self._namespace, key) - return json.loads(value) - - def __delitem__(self, key: ValidRedisKey): - """Delete an item from the Redis cache.""" - self._redis.hdel(self._namespace, key) - - def __iter__(self): - """Iterate all the items in the Redis cache.""" - return iter(self._redis.hkeys(self._namespace)) - - def __len__(self): - """Return the number of items in the Redis cache.""" - return self._redis.hlen(self._namespace) - - def copy(self) -> Dict: - """Convert to dict and return.""" - return dict(self.items()) diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py new file mode 100644 index 000000000..b2fd7d2e9 --- /dev/null +++ b/bot/utils/redis_dict.py @@ -0,0 +1,84 @@ +import json +from collections.abc import MutableMapping +from enum import Enum +from typing import Dict, List, Optional, Tuple, Union + +import redis as redis_py + +ValidRedisKey = Union[str, int, float] +JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] + + +class RedisDict(MutableMapping): + """ + A dictionary interface for a Redis database. + + Objects created by this class should mostly behave like a normal dictionary, + but will store all the data in our Redis database for persistence between restarts. + + Redis is limited to simple types, so to allow you to store collections like lists + and dictionaries, we JSON deserialize every value. That means that it will not be possible + to store complex objects, only stuff like strings, numbers, and collections of strings and numbers. + """ + + _namespaces = [] + _redis = redis_py.Redis(host="redis") # Can be overridden for testing + + def __init__(self, namespace: Optional[str] = None) -> None: + """Initialize the RedisDict with the right namespace.""" + super().__init__() + self._has_custom_namespace = namespace is not None + self._set_namespace(namespace) + + def _set_namespace(self, namespace: str) -> None: + """Try to set the namespace, but do not permit collisions.""" + while namespace in self._namespaces: + namespace = namespace + "_" + + self._namespaces.append(namespace) + self._namespace = namespace + + def __set_name__(self, owner: object, attribute_name: str) -> None: + """ + Set the namespace to Class.attribute_name. + + Called automatically when this class is constructed inside a class as an attribute, as long as + no custom namespace is provided to the constructor. + """ + if not self._has_custom_namespace: + self._set_namespace(f"{owner.__name__}.{attribute_name}") + + def __repr__(self) -> str: + """Return a beautiful representation of this object instance.""" + return f"RedisDict(namespace={self._namespace!r})" + + def __setitem__(self, key: ValidRedisKey, value: JSONSerializableType): + """Store an item in the Redis cache.""" + # JSON serialize the value before storing it. + json_value = json.dumps(value) + self._redis.hset(self._namespace, key, json_value) + + def __getitem__(self, key: ValidRedisKey): + """Get an item from the Redis cache.""" + value = self._redis.hget(self._namespace, key) + return json.loads(value) + + def __delitem__(self, key: ValidRedisKey): + """Delete an item from the Redis cache.""" + self._redis.hdel(self._namespace, key) + + def __iter__(self): + """Iterate all the items in the Redis cache.""" + return iter(self._redis.hkeys(self._namespace)) + + def __len__(self): + """Return the number of items in the Redis cache.""" + return self._redis.hlen(self._namespace) + + def copy(self) -> Dict: + """Convert to dict and return.""" + return dict(self.items()) + + def clear(self) -> None: + """Deletes the entire hash from the Redis cache.""" + self._redis.delete(self._namespace) -- cgit v1.2.3 From 677a7f755a15f8fdf0cd97e399c4265dd8e702d9 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 02:30:28 +0200 Subject: Implement .get, equality, and membership check This is supposed to be provided by our MutableMapping mixin, but unit tests are demonstrating that these don't really work as intended. --- bot/utils/redis_dict.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py index b2fd7d2e9..35439b2f3 100644 --- a/bot/utils/redis_dict.py +++ b/bot/utils/redis_dict.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json from collections.abc import MutableMapping from enum import Enum @@ -28,7 +30,11 @@ class RedisDict(MutableMapping): """Initialize the RedisDict with the right namespace.""" super().__init__() self._has_custom_namespace = namespace is not None - self._set_namespace(namespace) + + if self._has_custom_namespace: + self._set_namespace(namespace) + else: + self.namespace = "general" def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" @@ -52,6 +58,14 @@ class RedisDict(MutableMapping): """Return a beautiful representation of this object instance.""" return f"RedisDict(namespace={self._namespace!r})" + def __eq__(self, other: RedisDict) -> bool: + """Check equality between two RedisDicts.""" + return self.items() == other.items() and self._namespace == other._namespace + + def __ne__(self, other: RedisDict) -> bool: + """Check inequality between two RedisDicts.""" + return self.items() != other.items() or self._namespace != other._namespace + def __setitem__(self, key: ValidRedisKey, value: JSONSerializableType): """Store an item in the Redis cache.""" # JSON serialize the value before storing it. @@ -61,12 +75,18 @@ class RedisDict(MutableMapping): def __getitem__(self, key: ValidRedisKey): """Get an item from the Redis cache.""" value = self._redis.hget(self._namespace, key) - return json.loads(value) + + if value: + return json.loads(value) def __delitem__(self, key: ValidRedisKey): """Delete an item from the Redis cache.""" self._redis.hdel(self._namespace, key) + def __contains__(self, key: ValidRedisKey): + """Check if a key exists in the Redis cache.""" + return self._redis.hexists(self._namespace, key) + def __iter__(self): """Iterate all the items in the Redis cache.""" return iter(self._redis.hkeys(self._namespace)) @@ -82,3 +102,10 @@ class RedisDict(MutableMapping): def clear(self) -> None: """Deletes the entire hash from the Redis cache.""" self._redis.delete(self._namespace) + + def get(self, key: ValidRedisKey, default: Optional[str] = None) -> JSONSerializableType: + """Get the item, but provide a default if not found.""" + if key in self: + return self[key] + else: + return default -- cgit v1.2.3 From 0843728927fe73be2f2e2c37381ead1497debe11 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 02:40:30 +0200 Subject: Add fakeredis to the Pipfile --- Pipfile | 3 ++- Pipfile.lock | 25 ++++++++++++++++++++++++- bot/utils/__init__.py | 2 +- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/Pipfile b/Pipfile index 5f85b1e51..40ae52761 100644 --- a/Pipfile +++ b/Pipfile @@ -23,10 +23,11 @@ colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} statsd = "~=3.3" feedparser = "~=5.2" beautifulsoup4 = "~=4.9" -redis = ">=3.5" +redis = "~=3.5" [dev-packages] coverage = "~=5.0" +fakeredis = "~=1.4" flake8 = "~=3.7" flake8-annotations = "~=2.0" flake8-bugbear = "~=20.1" diff --git a/Pipfile.lock b/Pipfile.lock index 1a420182d..414f4a053 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c6b4d38c4034e55a4bd598399f2e1f48b70a76693c986d0db0fae7442e224d41" + "sha256": "8ec71e9c46d52bf3b8c72939519e993715c79b4bc9e6ad164c1cf88951dc48b4" }, "pipfile-spec": 6, "requires": { @@ -665,6 +665,14 @@ ], "version": "==0.3.0" }, + "fakeredis": { + "hashes": [ + "sha256:4d170886865a91dbc8b7f8cbd4e5d488f4c5f2f25dfae127f001617bbe9e8f97", + "sha256:647b2593d349d9d4e566c8dadb2e4c71ba35be5bdc4f1f7ac2d565a12a965053" + ], + "index": "pypi", + "version": "==1.4.1" + }, "filelock": { "hashes": [ "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", @@ -816,6 +824,14 @@ "index": "pypi", "version": "==5.3.1" }, + "redis": { + "hashes": [ + "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", + "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" + ], + "index": "pypi", + "version": "==3.5.2" + }, "six": { "hashes": [ "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", @@ -830,6 +846,13 @@ ], "version": "==2.0.0" }, + "sortedcontainers": { + "hashes": [ + "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", + "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" + ], + "version": "==2.1.0" + }, "toml": { "hashes": [ "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 7ae2db8fe..5ce383bf2 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -2,7 +2,7 @@ from abc import ABCMeta from discord.ext.commands import CogMeta -from bot.utils.redis import RedisDict +from bot.utils.redis_dict import RedisDict __all__ = ['RedisDict', 'CogABCMeta'] -- cgit v1.2.3 From 7cf0e83d1079ed34a3839948ce6823d95e0ebb62 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 03:16:03 +0200 Subject: Implement .pop, .popitem and .setdefault. Turns out the MutableMapping class doesn't give us servicable implementations of these, so we need to implement them ourselves. Also, let's not have keys returned as bytestrings. --- bot/utils/redis_dict.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py index 35439b2f3..c89765a24 100644 --- a/bot/utils/redis_dict.py +++ b/bot/utils/redis_dict.py @@ -89,7 +89,8 @@ class RedisDict(MutableMapping): def __iter__(self): """Iterate all the items in the Redis cache.""" - return iter(self._redis.hkeys(self._namespace)) + keys = self._redis.hkeys(self._namespace) + return iter([key.decode('utf-8') for key in keys]) def __len__(self): """Return the number of items in the Redis cache.""" @@ -103,9 +104,28 @@ class RedisDict(MutableMapping): """Deletes the entire hash from the Redis cache.""" self._redis.delete(self._namespace) - def get(self, key: ValidRedisKey, default: Optional[str] = None) -> JSONSerializableType: + def get(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: """Get the item, but provide a default if not found.""" if key in self: return self[key] else: return default + + def pop(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + """Get the item, remove it from the cache, and provide a default if not found.""" + value = self.get(key, default) + del self[key] + return value + + def popitem(self) -> JSONSerializableType: + """Get the last item added to the cache.""" + key = list(self.keys())[-1] + return self.pop(key) + + def setdefault(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + """Try to get the item. If the item does not exist, set it to `default` and return that.""" + value = self.get(key) + + if value is None: + self[key] = default + return default -- cgit v1.2.3 From bf6c113319d47594e103c71f8ff5b0ea48d15b38 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 03:18:19 +0200 Subject: Test suite for the redis dict. --- tests/bot/utils/test_redis_dict.py | 189 +++++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 tests/bot/utils/test_redis_dict.py diff --git a/tests/bot/utils/test_redis_dict.py b/tests/bot/utils/test_redis_dict.py new file mode 100644 index 000000000..f422887ce --- /dev/null +++ b/tests/bot/utils/test_redis_dict.py @@ -0,0 +1,189 @@ +import unittest + +import fakeredis +from redis import DataError + +from bot.utils import RedisDict + +redis_server = fakeredis.FakeServer() +RedisDict._redis = fakeredis.FakeStrictRedis(server=redis_server) + + +class RedisDictTests(unittest.TestCase): + """Tests the RedisDict class from utils.redis_dict.py.""" + + redis = RedisDict() + + def test_class_attribute_namespace(self): + """Test that RedisDict creates a namespace automatically for class attributes.""" + self.assertEqual(self.redis._namespace, "RedisDictTests.redis") + + def test_custom_namespace(self): + """Test that users can set a custom namespaces which never collide.""" + test_cases = ( + (RedisDict("firedog")._namespace, "firedog"), + (RedisDict("firedog")._namespace, "firedog_"), + (RedisDict("firedog")._namespace, "firedog__"), + ) + + for test_case, result in test_cases: + self.assertEqual(test_case, result) + + def test_custom_namespace_takes_precedence(self): + """Test that custom namespaces take precedence over class attribute ones.""" + class LemonJuice: + citrus = RedisDict("citrus") + watercat = RedisDict() + + test_class = LemonJuice() + self.assertEqual(test_class.citrus._namespace, "citrus") + self.assertEqual(test_class.watercat._namespace, "LemonJuice.watercat") + + def test_set_get_item(self): + """Test that users can set and get items from the RedisDict.""" + self.redis['favorite_fruit'] = 'melon' + self.redis['favorite_number'] = 86 + self.assertEqual(self.redis['favorite_fruit'], 'melon') + self.assertEqual(self.redis['favorite_number'], 86) + + def test_set_item_value_types(self): + """Test that setitem rejects values that are not JSON serializable.""" + with self.assertRaises(TypeError): + self.redis['favorite_thing'] = object + self.redis['favorite_stuff'] = RedisDict + + def test_set_item_key_types(self): + """Test that setitem rejects keys that are not strings, ints or floats.""" + fruits = ["lemon", "melon", "apple"] + + with self.assertRaises(DataError): + self.redis[fruits] = "nice" + + def test_get_method(self): + """Test that the .get method works like in a dict.""" + self.redis['favorite_movie'] = 'Code Jam Highlights' + + self.assertEqual(self.redis.get('favorite_movie'), 'Code Jam Highlights') + self.assertEqual(self.redis.get('favorite_youtuber', 'pydis'), 'pydis') + self.assertIsNone(self.redis.get('favorite_dog')) + + def test_membership(self): + """Test that we can reliably use the `in` operator with our RedisDict.""" + self.redis['favorite_country'] = "Burkina Faso" + + self.assertIn('favorite_country', self.redis) + self.assertNotIn('favorite_dentist', self.redis) + + def test_del_item(self): + """Test that users can delete items from the RedisDict.""" + self.redis['favorite_band'] = "Radiohead" + self.assertIn('favorite_band', self.redis) + + del self.redis['favorite_band'] + self.assertNotIn('favorite_band', self.redis) + + def test_iter(self): + """Test that the RedisDict can be iterated.""" + self.redis.clear() + test_cases = ( + ('favorite_turtle', 'Donatello'), + ('second_favorite_turtle', 'Leonardo'), + ('third_favorite_turtle', 'Raphael'), + ) + for key, value in test_cases: + self.redis[key] = value + + # Test regular iteration + for test_case, key in zip(test_cases, self.redis): + value = test_case[1] + self.assertEqual(self.redis[key], value) + + # Test .items iteration + for key, value in self.redis.items(): + self.assertEqual(self.redis[key], value) + + # Test .keys iteration + for test_case, key in zip(test_cases, self.redis.keys()): + value = test_case[1] + self.assertEqual(self.redis[key], value) + + def test_len(self): + """Test that we can get the correct len() from the RedisDict.""" + self.redis.clear() + self.redis['one'] = 1 + self.redis['two'] = 2 + self.redis['three'] = 3 + self.assertEqual(len(self.redis), 3) + + self.redis['four'] = 4 + self.assertEqual(len(self.redis), 4) + + def test_copy(self): + """Test that the .copy method returns a workable dictionary copy.""" + copy = self.redis.copy() + local_copy = dict(self.redis.items()) + self.assertIs(type(copy), dict) + self.assertEqual(copy, local_copy) + + def test_clear(self): + """Test that the .clear method removes the entire hash.""" + self.redis.clear() + self.redis['teddy'] = "with me" + self.redis['in my dreams'] = "you have a weird hat" + self.assertEqual(len(self.redis), 2) + + self.redis.clear() + self.assertEqual(len(self.redis), 0) + + def test_pop(self): + """Test that we can .pop an item from the RedisDict.""" + self.redis.clear() + self.redis['john'] = 'was afraid' + + self.assertEqual(self.redis.pop('john'), 'was afraid') + self.assertEqual(self.redis.pop('pete', 'breakneck'), 'breakneck') + self.assertEqual(len(self.redis), 0) + + def test_popitem(self): + """Test that we can .popitem an item from the RedisDict.""" + self.redis.clear() + self.redis['john'] = 'the revalator' + self.redis['teddy'] = 'big bear' + + self.assertEqual(len(self.redis), 2) + self.assertEqual(self.redis.popitem(), 'big bear') + self.assertEqual(len(self.redis), 1) + + def test_setdefault(self): + """Test that we can .setdefault an item from the RedisDict.""" + self.redis.clear() + self.redis.setdefault('john', 'is yellow and weak') + self.assertEqual(self.redis['john'], 'is yellow and weak') + + with self.assertRaises(TypeError): + self.redis.setdefault('geisha', object) + + def test_update(self): + """Test that we can .update the RedisDict with multiple items.""" + self.redis.clear() + self.redis["reckfried"] = "lona" + self.redis["bel air"] = "prince" + self.redis.update({ + "reckfried": "jona", + "mega": "hungry, though", + }) + + result = { + "reckfried": "jona", + "bel air": "prince", + "mega": "hungry, though", + } + self.assertEqual(self.redis.copy(), result) + + def test_equals(self): + """Test that RedisDicts can be compared with == and !=.""" + new_redis_dict = RedisDict("firedog_the_sequel") + new_new_redis_dict = new_redis_dict + + self.assertEqual(new_redis_dict, new_new_redis_dict) + self.assertNotEqual(new_redis_dict, self.redis) -- cgit v1.2.3 From 258ad9d9c5601e01d135e706c256c0cd7f7fdbe0 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 03:29:53 +0200 Subject: Make redis host and port configurable. --- bot/constants.py | 3 +++ bot/utils/redis_dict.py | 7 ++++++- config-default.yml | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/bot/constants.py b/bot/constants.py index fd280e9de..01e8ac3a3 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -199,6 +199,9 @@ class Bot(metaclass=YAMLGetter): prefix: str token: str sentry_dsn: str + redis_host: str + redis_port: int + class Filter(metaclass=YAMLGetter): section = "filter" diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py index c89765a24..dfb1c7252 100644 --- a/bot/utils/redis_dict.py +++ b/bot/utils/redis_dict.py @@ -7,6 +7,8 @@ from typing import Dict, List, Optional, Tuple, Union import redis as redis_py +from bot import constants + ValidRedisKey = Union[str, int, float] JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] @@ -24,7 +26,10 @@ class RedisDict(MutableMapping): """ _namespaces = [] - _redis = redis_py.Redis(host="redis") # Can be overridden for testing + _redis = redis_py.Redis( + host=constants.Bot.redis_host, + port=constants.Bot.redis_port, + ) # Can be overridden for testing def __init__(self, namespace: Optional[str] = None) -> None: """Initialize the RedisDict with the right namespace.""" diff --git a/config-default.yml b/config-default.yml index 83ea59016..722afa41b 100644 --- a/config-default.yml +++ b/config-default.yml @@ -2,6 +2,8 @@ bot: prefix: "!" token: !ENV "BOT_TOKEN" sentry_dsn: !ENV "BOT_SENTRY_DSN" + redis_host: "redis" + redis_port: 6379 stats: statsd_host: "graphite" -- cgit v1.2.3 From 8456023252c2d4c91c6566ee0a3f83e9033d45d9 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 03:39:52 +0200 Subject: namespace "general" -> "global" --- bot/utils/redis_dict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py index dfb1c7252..47905314a 100644 --- a/bot/utils/redis_dict.py +++ b/bot/utils/redis_dict.py @@ -39,7 +39,7 @@ class RedisDict(MutableMapping): if self._has_custom_namespace: self._set_namespace(namespace) else: - self.namespace = "general" + self.namespace = "global" def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" -- cgit v1.2.3 From ad154f7f0d7daa3f962433f77d1cdd11cc66bfe0 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 16 May 2020 22:43:00 -0700 Subject: Add a utility function to pad base64 data --- bot/cogs/token_remover.py | 5 +++-- bot/utils/__init__.py | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index 43c12c4f7..cae482e6e 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -7,6 +7,7 @@ import typing as t from discord import Colour, Message from discord.ext.commands import Cog +from bot import utils from bot.bot import Bot from bot.cogs.moderation import ModLog from bot.constants import Channels, Colours, Event, Icons @@ -146,7 +147,7 @@ class TokenRemover(Cog): See: https://discordapp.com/developers/docs/reference#snowflakes """ - b64_content += '=' * (-len(b64_content) % 4) + b64_content = utils.pad_base64(b64_content) try: decoded_bytes: bytes = base64.b64decode(b64_content) @@ -165,7 +166,7 @@ class TokenRemover(Cog): If the timestamp is greater than the Discord epoch, it's probably valid. See: https://i.imgur.com/7WdehGn.png """ - b64_content += '=' * (-len(b64_content) % 4) + b64_content = utils.pad_base64(b64_content) try: decoded_bytes = base64.urlsafe_b64decode(b64_content) diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 9b32e515d..1dd0636df 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -7,3 +7,8 @@ class CogABCMeta(CogMeta, ABCMeta): """Metaclass for ABCs meant to be implemented as Cogs.""" pass + + +def pad_base64(data: str) -> str: + """Return base64 `data` with padding characters to ensure its length is a multiple of 4.""" + return data + "=" * (-len(data) % 4) -- cgit v1.2.3 From e993566fe5d816fadee64aaca454ce6ba463bca2 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 11:35:34 +0200 Subject: Fix linting errors introduced by flake8 3.8 Turns out that bumping the flake8 version up to 3.8 introduces a long list of new linting errors. Since this PR is the one that bumps the version, I suppose we will also fix all the linting errors in this branch. --- bot/cogs/antispam.py | 4 ++-- bot/cogs/defcon.py | 2 +- bot/cogs/duck_pond.py | 2 +- bot/cogs/error_handler.py | 4 ++-- bot/cogs/help_channels.py | 2 +- bot/cogs/moderation/management.py | 10 +++++----- bot/cogs/moderation/scheduler.py | 10 +++++----- bot/cogs/moderation/silence.py | 2 +- bot/cogs/stats.py | 4 ++-- bot/cogs/utils.py | 4 ++-- bot/cogs/watchchannels/talentpool.py | 2 +- bot/cogs/watchchannels/watchchannel.py | 14 +++++++------- bot/decorators.py | 2 +- bot/pagination.py | 4 ++-- bot/utils/messages.py | 2 +- 15 files changed, 34 insertions(+), 34 deletions(-) diff --git a/bot/cogs/antispam.py b/bot/cogs/antispam.py index d63acbc4a..0bcca578d 100644 --- a/bot/cogs/antispam.py +++ b/bot/cogs/antispam.py @@ -94,7 +94,7 @@ class DeletionContext: await modlog.send_log_message( icon_url=Icons.filtering, colour=Colour(Colours.soft_red), - title=f"Spam detected!", + title="Spam detected!", text=mod_alert_message, thumbnail=last_message.author.avatar_url_as(static_format="png"), channel_id=Channels.mod_alerts, @@ -130,7 +130,7 @@ class AntiSpam(Cog): body += "\n\n**The cog has been unloaded.**" await self.mod_log.send_log_message( - title=f"Error: AntiSpam configuration validation failed!", + title="Error: AntiSpam configuration validation failed!", text=body, ping_everyone=True, icon_url=Icons.token_removed, diff --git a/bot/cogs/defcon.py b/bot/cogs/defcon.py index 56fca002a..f4cb0aa58 100644 --- a/bot/cogs/defcon.py +++ b/bot/cogs/defcon.py @@ -81,7 +81,7 @@ class Defcon(Cog): else: self.enabled = False self.days = timedelta(days=0) - log.info(f"DEFCON disabled") + log.info("DEFCON disabled") await self.update_channel_topic() diff --git a/bot/cogs/duck_pond.py b/bot/cogs/duck_pond.py index 1f84a0609..37d1786a2 100644 --- a/bot/cogs/duck_pond.py +++ b/bot/cogs/duck_pond.py @@ -117,7 +117,7 @@ class DuckPond(Cog): avatar_url=message.author.avatar_url ) except discord.HTTPException: - log.exception(f"Failed to send an attachment to the webhook") + log.exception("Failed to send an attachment to the webhook") await message.add_reaction("✅") diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index b2f4c59f6..16790c769 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -173,7 +173,7 @@ class ErrorHandler(Cog): await ctx.invoke(*help_command) self.bot.stats.incr("errors.missing_required_argument") elif isinstance(e, errors.TooManyArguments): - await ctx.send(f"Too many arguments provided.") + await ctx.send("Too many arguments provided.") await ctx.invoke(*help_command) self.bot.stats.incr("errors.too_many_arguments") elif isinstance(e, errors.BadArgument): @@ -213,7 +213,7 @@ class ErrorHandler(Cog): if isinstance(e, bot_missing_errors): ctx.bot.stats.incr("errors.bot_permission_error") await ctx.send( - f"Sorry, it looks like I don't have the permissions or roles I need to do that." + "Sorry, it looks like I don't have the permissions or roles I need to do that." ) elif isinstance(e, (InWhitelistCheckFailure, errors.NoPrivateMessage)): ctx.bot.stats.incr("errors.wrong_channel_or_dm_error") diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 1bd1f9d68..a20fe2b05 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -391,7 +391,7 @@ class HelpChannels(Scheduler, commands.Cog): self.in_use_category = await self.try_get_channel(constants.Categories.help_in_use) self.dormant_category = await self.try_get_channel(constants.Categories.help_dormant) except discord.HTTPException: - log.exception(f"Failed to get a category; cog will be removed") + log.exception("Failed to get a category; cog will be removed") self.bot.remove_cog(self.qualified_name) async def init_cog(self) -> None: diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index 250a24247..82ec6b0d9 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -49,8 +49,8 @@ class ModManagement(commands.Cog): async def infraction_edit( self, ctx: Context, - infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], - duration: t.Union[Expiry, allowed_strings("p", "permanent"), None], + infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], # noqa: F821 + duration: t.Union[Expiry, allowed_strings("p", "permanent"), None], # noqa: F821 *, reason: str = None ) -> None: @@ -83,14 +83,14 @@ class ModManagement(commands.Cog): "actor__id": ctx.author.id, "ordering": "-inserted_at" } - infractions = await self.bot.api_client.get(f"bot/infractions", params=params) + infractions = await self.bot.api_client.get("bot/infractions", params=params) if infractions: old_infraction = infractions[0] infraction_id = old_infraction["id"] else: await ctx.send( - f":x: Couldn't find most recent infraction; you have never given an infraction." + ":x: Couldn't find most recent infraction; you have never given an infraction." ) return else: @@ -224,7 +224,7 @@ class ModManagement(commands.Cog): ) -> None: """Send a paginated embed of infractions for the specified user.""" if not infractions: - await ctx.send(f":warning: No infractions could be found for that query.") + await ctx.send(":warning: No infractions could be found for that query.") return lines = tuple( diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index dc42bee2e..012432e60 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -91,7 +91,7 @@ class InfractionScheduler(Scheduler): log.trace(f"Applying {infr_type} infraction #{id_} to {user}.") # Default values for the confirmation message and mod log. - confirm_msg = f":ok_hand: applied" + confirm_msg = ":ok_hand: applied" # Specifying an expiry for a note or warning makes no sense. if infr_type in ("note", "warning"): @@ -154,7 +154,7 @@ class InfractionScheduler(Scheduler): self.schedule_task(infraction["id"], infraction) except discord.HTTPException as e: # Accordingly display that applying the infraction failed. - confirm_msg = f":x: failed to apply" + confirm_msg = ":x: failed to apply" expiry_msg = "" log_content = ctx.author.mention log_title = "failed to apply" @@ -281,7 +281,7 @@ class InfractionScheduler(Scheduler): log.warning(f"Failed to pardon {infr_type} infraction #{id_} for {user}.") else: - confirm_msg = f":ok_hand: pardoned" + confirm_msg = ":ok_hand: pardoned" log_title = "pardoned" log.info(f"Pardoned {infr_type} infraction #{id_} for {user}.") @@ -353,7 +353,7 @@ class InfractionScheduler(Scheduler): ) except discord.Forbidden: log.warning(f"Failed to deactivate infraction #{id_} ({type_}): bot lacks permissions.") - log_text["Failure"] = f"The bot lacks permissions to do this (role hierarchy?)" + log_text["Failure"] = "The bot lacks permissions to do this (role hierarchy?)" log_content = mod_role.mention except discord.HTTPException as e: log.exception(f"Failed to deactivate infraction #{id_} ({type_})") @@ -402,7 +402,7 @@ class InfractionScheduler(Scheduler): # Send a log message to the mod log. if send_log: - log_title = f"expiration failed" if "Failure" in log_text else "expired" + log_title = "expiration failed" if "Failure" in log_text else "expired" user = self.bot.get_user(user_id) avatar = user.avatar_url_as(static_format="png") if user else None diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index 1ef3967a9..25febfa51 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -91,7 +91,7 @@ class Silence(commands.Cog): await ctx.send(f"{Emojis.check_mark} silenced current channel for {duration} minute(s).") await asyncio.sleep(duration*60) - log.info(f"Unsilencing channel after set delay.") + log.info("Unsilencing channel after set delay.") await ctx.invoke(self.unsilence) @commands.command(aliases=("unhush",)) diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py index d253db913..e088c2b87 100644 --- a/bot/cogs/stats.py +++ b/bot/cogs/stats.py @@ -59,7 +59,7 @@ class Stats(Cog): if member.guild.id != Guild.id: return - self.bot.stats.gauge(f"guild.total_members", len(member.guild.members)) + self.bot.stats.gauge("guild.total_members", len(member.guild.members)) @Cog.listener() async def on_member_leave(self, member: Member) -> None: @@ -67,7 +67,7 @@ class Stats(Cog): if member.guild.id != Guild.id: return - self.bot.stats.gauge(f"guild.total_members", len(member.guild.members)) + self.bot.stats.gauge("guild.total_members", len(member.guild.members)) @Cog.listener() async def on_member_update(self, _before: Member, after: Member) -> None: diff --git a/bot/cogs/utils.py b/bot/cogs/utils.py index 89d556f58..f76daedac 100644 --- a/bot/cogs/utils.py +++ b/bot/cogs/utils.py @@ -253,8 +253,8 @@ class Utils(Cog): async def send_pep_zero(self, ctx: Context) -> None: """Send information about PEP 0.""" pep_embed = Embed( - title=f"**PEP 0 - Index of Python Enhancement Proposals (PEPs)**", - description=f"[Link](https://www.python.org/dev/peps/)" + title="**PEP 0 - Index of Python Enhancement Proposals (PEPs)**", + description="[Link](https://www.python.org/dev/peps/)" ) pep_embed.set_thumbnail(url=ICON_URL) pep_embed.add_field(name="Status", value="Active") diff --git a/bot/cogs/watchchannels/talentpool.py b/bot/cogs/watchchannels/talentpool.py index ad0c51fa6..68b220233 100644 --- a/bot/cogs/watchchannels/talentpool.py +++ b/bot/cogs/watchchannels/talentpool.py @@ -61,7 +61,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): return if isinstance(user, Member) and any(role.id in STAFF_ROLES for role in user.roles): - await ctx.send(f":x: Nominating staff members, eh? Here's a cookie :cookie:") + await ctx.send(":x: Nominating staff members, eh? Here's a cookie :cookie:") return if not await self.fetch_user_cache(): diff --git a/bot/cogs/watchchannels/watchchannel.py b/bot/cogs/watchchannels/watchchannel.py index 479820444..643cd46e4 100644 --- a/bot/cogs/watchchannels/watchchannel.py +++ b/bot/cogs/watchchannels/watchchannel.py @@ -82,7 +82,7 @@ class WatchChannel(metaclass=CogABCMeta): exc = self._consume_task.exception() if exc: self.log.exception( - f"The message queue consume task has failed with:", + "The message queue consume task has failed with:", exc_info=exc ) return False @@ -146,7 +146,7 @@ class WatchChannel(metaclass=CogABCMeta): try: data = await self.bot.api_client.get(self.api_endpoint, params=self.api_default_params) except ResponseCodeError as err: - self.log.exception(f"Failed to fetch the watched users from the API", exc_info=err) + self.log.exception("Failed to fetch the watched users from the API", exc_info=err) return False self.watched_users = defaultdict(dict) @@ -173,7 +173,7 @@ class WatchChannel(metaclass=CogABCMeta): self.log.trace(f"Sleeping {BigBrotherConfig.log_delay} seconds before consuming message queue") await asyncio.sleep(BigBrotherConfig.log_delay) - self.log.trace(f"Started consuming the message queue") + self.log.trace("Started consuming the message queue") # If the previous consumption Task failed, first consume the existing comsumption_queue if not self.consumption_queue: @@ -208,7 +208,7 @@ class WatchChannel(metaclass=CogABCMeta): await self.webhook.send(content=content, username=username, avatar_url=avatar_url, embed=embed) except discord.HTTPException as exc: self.log.exception( - f"Failed to send a message to the webhook", + "Failed to send a message to the webhook", exc_info=exc ) @@ -254,7 +254,7 @@ class WatchChannel(metaclass=CogABCMeta): ) except discord.HTTPException as exc: self.log.exception( - f"Failed to send an attachment to the webhook", + "Failed to send an attachment to the webhook", exc_info=exc ) @@ -326,13 +326,13 @@ class WatchChannel(metaclass=CogABCMeta): def cog_unload(self) -> None: """Takes care of unloading the cog and canceling the consumption task.""" - self.log.trace(f"Unloading the cog") + self.log.trace("Unloading the cog") if self._consume_task and not self._consume_task.done(): self._consume_task.cancel() try: self._consume_task.result() except asyncio.CancelledError as e: self.log.exception( - f"The consume task was canceled. Messages may be lost.", + "The consume task was canceled. Messages may be lost.", exc_info=e ) diff --git a/bot/decorators.py b/bot/decorators.py index 2ee5879f2..dc9c7d439 100644 --- a/bot/decorators.py +++ b/bot/decorators.py @@ -121,7 +121,7 @@ def locked() -> Callable: embed = Embed() embed.colour = Colour.red() - log.debug(f"User tried to invoke a locked command.") + log.debug("User tried to invoke a locked command.") embed.description = ( "You're already using this command. Please wait until it is done before you use it again." ) diff --git a/bot/pagination.py b/bot/pagination.py index 90c8f849c..09759d5be 100644 --- a/bot/pagination.py +++ b/bot/pagination.py @@ -147,7 +147,7 @@ class LinePaginator(Paginator): if not lines: if exception_on_empty_embed: - log.exception(f"Pagination asked for empty lines iterable") + log.exception("Pagination asked for empty lines iterable") raise EmptyPaginatorEmbed("No lines to paginate") log.debug("No lines to add to paginator, adding '(nothing to display)' message") @@ -357,7 +357,7 @@ class ImagePaginator(Paginator): if not pages: if exception_on_empty_embed: - log.exception(f"Pagination asked for empty image list") + log.exception("Pagination asked for empty image list") raise EmptyPaginatorEmbed("No images to paginate") log.debug("No images to add to paginator, adding '(no images to display)' message") diff --git a/bot/utils/messages.py b/bot/utils/messages.py index e969ee590..de8e186f3 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -100,7 +100,7 @@ async def send_attachments( log.warning(f"{failure_msg} with status {e.status}.") if link_large and large: - desc = f"\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large) + desc = "\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large) embed = Embed(description=desc) embed.set_footer(text="Attachments exceed upload size limit.") -- cgit v1.2.3 From adf50a6ddc6868dec108ad471c5f3f4033ccd69b Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 19:44:29 +0200 Subject: Changes discord-py to discord.py in Pipfile The `discord-py` package is no longer the official release, and so making this change silences some warnings about deprecation. --- Pipfile | 2 +- Pipfile.lock | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Pipfile b/Pipfile index 40ae52761..1d6cd7015 100644 --- a/Pipfile +++ b/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -discord-py = "~=1.3.2" +discord.py = "~=1.3.2" aiodns = "~=2.0" aiohttp = "~=3.5" sphinx = "~=2.2" diff --git a/Pipfile.lock b/Pipfile.lock index 414f4a053..25383b355 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8ec71e9c46d52bf3b8c72939519e993715c79b4bc9e6ad164c1cf88951dc48b4" + "sha256": "49c231092320b48c5a7618bf048a477f7e0ed33dcbfb71c6dc8f18ef819dd935" }, "pipfile-spec": 6, "requires": { @@ -166,11 +166,19 @@ "index": "pypi", "version": "==4.3.2" }, - "discord-py": { + "discord": { "hashes": [ - "sha256:406871b06d86c3dc49fba63238519f28628dac946fef8a0e22988ff58ec05580" + "sha256:9d4debb4a37845543bd4b92cb195bc53a302797333e768e70344222857ff1559", + "sha256:ff6653655e342e7721dfb3f10421345fd852c2a33f2cca912b1c39b3778a9429" ], "index": "pypi", + "version": "==1.0.1" + }, + "discord.py": { + "hashes": [ + "sha256:406871b06d86c3dc49fba63238519f28628dac946fef8a0e22988ff58ec05580", + "sha256:ad00e34c72d2faa8db2157b651d05f3c415d7d05078e7e41dc9e8dc240051beb" + ], "version": "==1.3.3" }, "docutils": { @@ -480,10 +488,10 @@ }, "soupsieve": { "hashes": [ - "sha256:e914534802d7ffd233242b785229d5ba0766a7f487385e3f714446a07bf540ae", - "sha256:fcd71e08c0aee99aca1b73f45478549ee7e7fc006d51b37bec9e9def7dc22b69" + "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", + "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232" ], - "version": "==2.0" + "version": "==2.0.1" }, "sphinx": { "hashes": [ -- cgit v1.2.3 From ecf7f24f05b9baa8705f3b2c2d044a42292fbc07 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 19:50:12 +0200 Subject: Add the REDIS_PASSWORD environment variable In production, we will need this password to make a connection to Redis. --- bot/constants.py | 11 +++++++++-- bot/utils/redis_dict.py | 5 +++-- config-default.yml | 7 +++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/bot/constants.py b/bot/constants.py index 01e8ac3a3..5d854dd7a 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -199,8 +199,15 @@ class Bot(metaclass=YAMLGetter): prefix: str token: str sentry_dsn: str - redis_host: str - redis_port: int + + +class Redis(metaclass=YAMLGetter): + section = "bot" + subsection = "redis" + + host: str + port: int + password: str class Filter(metaclass=YAMLGetter): diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py index 47905314a..4a5e34249 100644 --- a/bot/utils/redis_dict.py +++ b/bot/utils/redis_dict.py @@ -27,8 +27,9 @@ class RedisDict(MutableMapping): _namespaces = [] _redis = redis_py.Redis( - host=constants.Bot.redis_host, - port=constants.Bot.redis_port, + host=constants.Redis.host, + port=constants.Redis.port, + password=constants.Redis.password, ) # Can be overridden for testing def __init__(self, namespace: Optional[str] = None) -> None: diff --git a/config-default.yml b/config-default.yml index 722afa41b..3b58d9099 100644 --- a/config-default.yml +++ b/config-default.yml @@ -2,8 +2,11 @@ bot: prefix: "!" token: !ENV "BOT_TOKEN" sentry_dsn: !ENV "BOT_SENTRY_DSN" - redis_host: "redis" - redis_port: 6379 + + redis: + host: "redis" + port: 6379 + password: !ENV "REDIS_PASSWORD" stats: statsd_host: "graphite" -- cgit v1.2.3 From bd4b439bb7f4abb6b22ad6e0d33bbe9203317475 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Tue, 19 May 2020 01:30:33 +0100 Subject: [bug] Adjustment to changes in #941, return message sent by webhook so publish can take place --- bot/cogs/reddit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index a300cfe0f..3b77538a0 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -218,7 +218,7 @@ class Reddit(Cog): for subreddit in RedditConfig.subreddits: top_posts = await self.get_top_posts(subreddit=subreddit, time="day") - message = await self.webhook.send(username=f"{subreddit} Top Daily Posts", embed=top_posts) + message = await self.webhook.send(username=f"{subreddit} Top Daily Posts", embed=top_posts, wait=True) if message.channel.is_news(): await message.publish() -- cgit v1.2.3 From 1d36ec7001c35d96ff24f7493d26997c02891e93 Mon Sep 17 00:00:00 2001 From: "S. Co1" Date: Mon, 18 May 2020 21:17:53 -0400 Subject: Add Steam gift card scam to domain blacklist --- config-default.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config-default.yml b/config-default.yml index 83ea59016..c0b5b062f 100644 --- a/config-default.yml +++ b/config-default.yml @@ -316,6 +316,8 @@ filter: - poweredbydialup.online - poweredbysecurity.org - poweredbysecurity.online + - ssteam.site + - steamwalletgift.com word_watchlist: - goo+ks* -- cgit v1.2.3 From 75385da574dfdd622ab4b0c7d5771ebd3218542d Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 19 May 2020 07:58:40 +0300 Subject: Stats: Fixed stat names Co-authored-by: Joseph Banks --- bot/cogs/snekbox.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 1d240d8d8..efff6d815 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -207,9 +207,9 @@ class Snekbox(Cog): # Collect stats of eval fails + successes if icon == ":x:": - self.bot.stats.incr("evals.fail") + self.bot.stats.incr("snekbox.python.fail") elif icon in (":warning:", ":white_check_mark:"): - self.bot.stats.incr("evals.success") + self.bot.stats.incr("snekbox.python.success") response = await ctx.send(msg) self.bot.loop.create_task( @@ -299,16 +299,16 @@ class Snekbox(Cog): return if Roles.helpers in (role.id for role in ctx.author.roles): - self.bot.stats.incr("evals.roles.helpers") + self.bot.stats.incr("snekbox_usages.roles.helpers") else: - self.bot.stats.incr("evals.roles.developers") + self.bot.stats.incr("snekbox_usages.roles.developers") if ctx.channel.category_id == Categories.help_in_use: - self.bot.stats.incr("evals.channels.help") + self.bot.stats.incr("snekbox_usages.channels.help") elif ctx.channel.id == Channels.bot_commands: - self.bot.stats.incr("evals.channels.bot_commands") + self.bot.stats.incr("snekbox_usages.channels.bot_commands") else: - self.bot.stats.incr("evals.channels.topical") + self.bot.stats.incr("snekbox_usages.channels.topical") log.info(f"Received code from {ctx.author} for evaluation:\n{code}") -- cgit v1.2.3 From 3081aad0cb360436c451b9a4515d494711adaf81 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 19 May 2020 07:59:21 +0300 Subject: Stats: Fix docstrings Co-authored-by: Joseph Banks --- bot/cogs/stats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py index acee1f5a9..9baf222e2 100644 --- a/bot/cogs/stats.py +++ b/bot/cogs/stats.py @@ -106,14 +106,14 @@ class Stats(Cog): @loop(hours=1) async def update_guild_boost(self) -> None: - """Update every hour guild boosts amount + level.""" + """Post the server boost level and tier every hour.""" await self.bot.wait_until_guild_available() g = self.bot.get_guild(Guild.id) self.bot.stats.gauge("boost.amount", g.premium_subscription_count) self.bot.stats.gauge("boost.tier", g.premium_tier) def cog_unload(self) -> None: - """Stop guild boost stat collecting task on Cog unload.""" + """Stop the boost statistic task on unload of the Cog.""" self.update_guild_boost.stop() -- cgit v1.2.3 From 21916ad9c19a326eb8406ea751e5fd9f80e9d912 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:42:11 +0300 Subject: ModLog Tests: Fix truncation tests docstring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Leon Sandøy --- tests/bot/cogs/moderation/test_modlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/moderation/test_modlog.py b/tests/bot/cogs/moderation/test_modlog.py index d60836474..b5ad21a09 100644 --- a/tests/bot/cogs/moderation/test_modlog.py +++ b/tests/bot/cogs/moderation/test_modlog.py @@ -15,7 +15,7 @@ class ModLogTests(unittest.IsolatedAsyncioTestCase): self.channel = MockTextChannel() async def test_log_entry_description_truncation(self): - """Should truncate embed description for ModLog entry.""" + """Test that embed description for ModLog entry is truncated.""" self.bot.get_channel.return_value = self.channel await self.cog.send_log_message( icon_url="foo", -- cgit v1.2.3 From 1432e5ba36fc09c7233e5be4745f540c2c4af792 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:47:59 +0300 Subject: Infraction Tests: Small fixes - Remove unnecessary space from placeholder - Rename `has_active_infraction` to `get_active_infraction` --- tests/bot/cogs/moderation/test_infractions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 51a8cc645..2b1ff5728 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -17,11 +17,11 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): self.guild = MockGuild(id=4567) self.ctx = MockContext(bot=self.bot, author=self.user, guild=self.guild) - @patch("bot.cogs.moderation.utils.has_active_infraction") + @patch("bot.cogs.moderation.utils.get_active_infraction") @patch("bot.cogs.moderation.utils.post_infraction") - async def test_apply_ban_reason_truncation(self, post_infraction_mock, has_active_mock): + async def test_apply_ban_reason_truncation(self, post_infraction_mock, get_active_mock): """Should truncate reason for `ctx.guild.ban`.""" - has_active_mock.return_value = False + get_active_mock.return_value = 'foo' post_infraction_mock.return_value = {"foo": "bar"} self.cog.apply_infraction = AsyncMock() @@ -32,7 +32,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): ban = self.cog.apply_infraction.call_args[0][3] self.assertEqual( ban.cr_frame.f_locals["kwargs"]["reason"], - textwrap.shorten("foo bar" * 3000, 512, placeholder=" ...") + textwrap.shorten("foo bar" * 3000, 512, placeholder="...") ) # Await ban to avoid warning await ban -- cgit v1.2.3 From 787c106fb4a55eacc7af04afb9bcd8f206099e81 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:50:46 +0300 Subject: Infractions: Remove space from placeholder --- bot/cogs/moderation/infractions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 01266d346..5bfaad796 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -259,7 +259,7 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - truncated_reason = textwrap.shorten(reason, width=512, placeholder=" ...") + truncated_reason = textwrap.shorten(reason, width=512, placeholder="...") action = ctx.guild.ban(user, reason=truncated_reason, delete_message_days=0) await self.apply_infraction(ctx, infraction, user, action) -- cgit v1.2.3 From caac9b92d7c3f73ca5428597606105730e56cefc Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:53:58 +0300 Subject: ModLog: Fix embed description truncation --- bot/cogs/moderation/modlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index c6497b38d..9d28030d9 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -100,7 +100,7 @@ class ModLog(Cog, name="ModLog"): """Generate log embed and send to logging channel.""" # Truncate string directly here to avoid removing newlines embed = discord.Embed( - description=text[:2046] + "..." if len(text) > 2048 else text + description=text[:2045] + "..." if len(text) > 2048 else text ) if title and icon_url: -- cgit v1.2.3 From 5989bcfefa244eb05f37b76d1e1df2f45e5782fa Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:54:49 +0300 Subject: ModLog Tests: Fix embed description truncate test --- tests/bot/cogs/moderation/test_modlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/moderation/test_modlog.py b/tests/bot/cogs/moderation/test_modlog.py index b5ad21a09..f2809f40a 100644 --- a/tests/bot/cogs/moderation/test_modlog.py +++ b/tests/bot/cogs/moderation/test_modlog.py @@ -25,5 +25,5 @@ class ModLogTests(unittest.IsolatedAsyncioTestCase): ) embed = self.channel.send.call_args[1]["embed"] self.assertEqual( - embed.description, ("foo bar" * 3000)[:2046] + "..." + embed.description, ("foo bar" * 3000)[:2045] + "..." ) -- cgit v1.2.3 From 874cb001df91ea8223385dd2b32ab4e3c280e183 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:57:07 +0300 Subject: Infr. Tests: Add more content to await comment --- tests/bot/cogs/moderation/test_infractions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 2b1ff5728..f8f340c2e 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -34,7 +34,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): ban.cr_frame.f_locals["kwargs"]["reason"], textwrap.shorten("foo bar" * 3000, 512, placeholder="...") ) - # Await ban to avoid warning + # Await ban to avoid not awaited coroutine warning await ban @patch("bot.cogs.moderation.utils.post_infraction") @@ -51,5 +51,5 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): kick.cr_frame.f_locals["kwargs"]["reason"], textwrap.shorten("foo bar" * 3000, 512, placeholder="...") ) - # Await kick to avoid warning + # Await kick to avoid not awaited coroutine warning await kick -- cgit v1.2.3 From e9bd09d90c5acf61caa955533f406851e1a65aec Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:59:11 +0300 Subject: Infr. Tests: Replace `str` with `dict` To allow `.get`, I had to replace `str` return value with `dict` --- tests/bot/cogs/moderation/test_infractions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index f8f340c2e..139209749 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -21,7 +21,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): @patch("bot.cogs.moderation.utils.post_infraction") async def test_apply_ban_reason_truncation(self, post_infraction_mock, get_active_mock): """Should truncate reason for `ctx.guild.ban`.""" - get_active_mock.return_value = 'foo' + get_active_mock.return_value = {"foo": "bar"} post_infraction_mock.return_value = {"foo": "bar"} self.cog.apply_infraction = AsyncMock() -- cgit v1.2.3 From d9730e41b3144862fdd9c221d160a40144a7c881 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 11:02:37 +0300 Subject: Infr. Test: Replace `get_active_mock` return value Replace `{"foo": "bar"}` with `{"id": 1}` --- tests/bot/cogs/moderation/test_infractions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 139209749..925439bf3 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -21,7 +21,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): @patch("bot.cogs.moderation.utils.post_infraction") async def test_apply_ban_reason_truncation(self, post_infraction_mock, get_active_mock): """Should truncate reason for `ctx.guild.ban`.""" - get_active_mock.return_value = {"foo": "bar"} + get_active_mock.return_value = {"id": 1} post_infraction_mock.return_value = {"foo": "bar"} self.cog.apply_infraction = AsyncMock() -- cgit v1.2.3 From a1b6d147befd4043acdddc00667d3bda94cc76ad Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 11:15:09 +0300 Subject: Infr Tests: Make `get_active_infraction` return `None` --- tests/bot/cogs/moderation/test_infractions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 925439bf3..5548d9f68 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -21,7 +21,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): @patch("bot.cogs.moderation.utils.post_infraction") async def test_apply_ban_reason_truncation(self, post_infraction_mock, get_active_mock): """Should truncate reason for `ctx.guild.ban`.""" - get_active_mock.return_value = {"id": 1} + get_active_mock.return_value = None post_infraction_mock.return_value = {"foo": "bar"} self.cog.apply_infraction = AsyncMock() -- cgit v1.2.3 From 6c5979b88961cb1df3d669e07ae108d12e698119 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 21 May 2020 11:47:39 +0300 Subject: Config: Added new `HelpChannels` config `deleted_idle_minutes` This show how much minutes should this wait before making channel dormant when no messages in channel (original message deleted). --- bot/constants.py | 1 + config-default.yml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/bot/constants.py b/bot/constants.py index fd280e9de..3003c9d36 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -541,6 +541,7 @@ class HelpChannels(metaclass=YAMLGetter): claim_minutes: int cmd_whitelist: List[int] idle_minutes: int + deleted_idle_minutes: int max_available: int max_total_channels: int name_prefix: str diff --git a/config-default.yml b/config-default.yml index 83ea59016..2e8a777ba 100644 --- a/config-default.yml +++ b/config-default.yml @@ -529,6 +529,10 @@ help_channels: # Allowed duration of inactivity before making a channel dormant idle_minutes: 30 + # Allowed duration of inactivity when question message deleted + # and no one other sent before message making channel dormant. + deleted_idle_minutes: 5 + # Maximum number of channels to put in the available category max_available: 2 -- cgit v1.2.3 From 0c84302f7e3475c13924fda33c52e98566114082 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 21 May 2020 12:27:25 +0300 Subject: Help: Implemented faster close when claimant delete msg no more messages - Created function `is_empty` that check is there any message in channel after bot own available message. - `on_message_delete` that reschedule task when message is on correct channel and is empty. - In `move_idle_channel` function, implemented choosing right cooldown, based on is channel empty or not. --- bot/cogs/help_channels.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 1bd1f9d68..4415ce550 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -461,7 +461,11 @@ class HelpChannels(Scheduler, commands.Cog): """ log.trace(f"Handling in-use channel #{channel} ({channel.id}).") - idle_seconds = constants.HelpChannels.idle_minutes * 60 + if not self.is_empty(channel): + idle_seconds = constants.HelpChannels.idle_minutes * 60 + else: + idle_seconds = constants.HelpChannels.deleted_idle_minutes * 60 + time_elapsed = await self.get_idle_time(channel) if time_elapsed is None or time_elapsed >= idle_seconds: @@ -713,6 +717,32 @@ class HelpChannels(Scheduler, commands.Cog): # be put in the queue. await self.move_to_available() + @commands.Cog.listener() + async def on_message_delete(self, msg: discord.Message) -> None: + """Reschedule dormant when help channel is empty.""" + if not self.is_in_category(msg.channel, constants.Categories.help_in_use) or not self.is_empty(msg.channel): + return + + log.info(f"Claimant of #{msg.channel} ({msg.author}) deleted message, channel is empty now. Rescheduling task.") + + # Cancel existing dormant task before scheduling new. + self.cancel_task(msg.channel.id) + + task = TaskData(constants.HelpChannels.deleted_idle_minutes * 60, self.move_idle_channel(msg.channel)) + self.schedule_task(msg.channel.id, task) + + async def is_empty(self, channel: discord.TextChannel) -> bool: + """Check is last message bot sent available message.""" + msg = await self.get_last_message(channel) + if not msg or not msg.author.bot or not msg.embeds: + return False + + embed = msg.embeds[0] + if embed.description == AVAILABLE_MSG: + return True + else: + return False + async def reset_send_permissions(self) -> None: """Reset send permissions in the Available category for claimants.""" log.trace("Resetting send permissions in the Available category.") -- cgit v1.2.3 From cc3591df0f14041be683bb6716d1e427c52aa2d7 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Fri, 22 May 2020 02:34:04 +0200 Subject: Add the REDIS_PASSWORD environment variable In production, we will need this password to make a connection to Redis. --- bot/utils/redis_cache.py | 115 +++++++++++++++++++++++++++++++++++++++ bot/utils/redis_dict.py | 137 ----------------------------------------------- 2 files changed, 115 insertions(+), 137 deletions(-) create mode 100644 bot/utils/redis_cache.py delete mode 100644 bot/utils/redis_dict.py diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py new file mode 100644 index 000000000..d0a7eba4a --- /dev/null +++ b/bot/utils/redis_cache.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +from enum import Enum +from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Union + +from bot.bot import Bot + +ValidRedisKey = Union[str, int, float] +JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] + + +class RedisCache: + """ + A simplified interface for a Redis connection. + + This class must be created as a class attribute in a class. This is because it + uses __set_name__ to create a namespace like MyCog.my_class_attribute which is + used as a hash name when we store stuff in Redis, to prevent collisions. + + The class this object is instantiated in must also contains an attribute with an + instance of Bot. This is because Bot contains our redis_pool, which is how this + class communicates with the Redis server. + + We implement several convenient methods that are fairly similar to have a dict + behaves, and should be familiar to Python users. The biggest difference is that + all the public methods in this class are coroutines. + """ + + _namespaces = [] + + def __init__(self) -> None: + """Raise a NotImplementedError if `__set_name__` hasn't been run.""" + if not self._namespace: + raise NotImplementedError("RedisCache must be a class attribute.") + + def _set_namespace(self, namespace: str) -> None: + """Try to set the namespace, but do not permit collisions.""" + while namespace in self._namespaces: + namespace += "_" + + self._namespaces.append(namespace) + self._namespace = namespace + + def __set_name__(self, owner: object, attribute_name: str) -> None: + """ + Set the namespace to Class.attribute_name. + + Called automatically when this class is constructed inside a class as an attribute. + """ + if not self._has_custom_namespace: + self._set_namespace(f"{owner.__name__}.{attribute_name}") + + def __get__(self, instance: RedisCache, owner: Any) -> RedisCache: + """Fetch the Bot instance, we need it for the redis pool.""" + if self.bot: + return self + + if instance is None: + raise NotImplementedError("You must create an instance of RedisCache to use it.") + + for attribute in vars(instance).values(): + if isinstance(attribute, Bot): + self.bot = attribute + self._redis = self.bot.redis_pool + return self + else: + raise RuntimeError("Cannot initialize a RedisCache without a `Bot` instance.") + + def __repr__(self) -> str: + """Return a beautiful representation of this object instance.""" + return f"RedisCache(namespace={self._namespace!r})" + + async def set(self, key: ValidRedisKey, value: JSONSerializableType) -> None: + """Store an item in the Redis cache.""" + # await self._redis.hset(self._namespace, key, value) + + async def get(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + """Get an item from the Redis cache.""" + # value = await self._redis.hget(self._namespace, key) + + async def delete(self, key: ValidRedisKey) -> None: + """Delete an item from the Redis cache.""" + # await self._redis.hdel(self._namespace, key) + + async def contains(self, key: ValidRedisKey) -> bool: + """Check if a key exists in the Redis cache.""" + # return await self._redis.hexists(self._namespace, key) + + async def items(self) -> AsyncIterator: + """Iterate all the items in the Redis cache.""" + # data = await redis.hgetall(self.get_with_namespace(key)) + # for item in data: + # yield item + + async def length(self) -> int: + """Return the number of items in the Redis cache.""" + # return await self._redis.hlen(self._namespace) + + async def to_dict(self) -> Dict: + """Convert to dict and return.""" + # return dict(self.items()) + + async def clear(self) -> None: + """Deletes the entire hash from the Redis cache.""" + # await self._redis.delete(self._namespace) + + async def pop(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + """Get the item, remove it from the cache, and provide a default if not found.""" + value = await self.get(key, default) + await self.delete(key) + return value + + async def update(self) -> None: + """Update the Redis cache with multiple values.""" + # https://aioredis.readthedocs.io/en/v1.3.0/mixins.html#aioredis.commands.HashCommandsMixin.hmset_dict diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py deleted file mode 100644 index 4a5e34249..000000000 --- a/bot/utils/redis_dict.py +++ /dev/null @@ -1,137 +0,0 @@ -from __future__ import annotations - -import json -from collections.abc import MutableMapping -from enum import Enum -from typing import Dict, List, Optional, Tuple, Union - -import redis as redis_py - -from bot import constants - -ValidRedisKey = Union[str, int, float] -JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] - - -class RedisDict(MutableMapping): - """ - A dictionary interface for a Redis database. - - Objects created by this class should mostly behave like a normal dictionary, - but will store all the data in our Redis database for persistence between restarts. - - Redis is limited to simple types, so to allow you to store collections like lists - and dictionaries, we JSON deserialize every value. That means that it will not be possible - to store complex objects, only stuff like strings, numbers, and collections of strings and numbers. - """ - - _namespaces = [] - _redis = redis_py.Redis( - host=constants.Redis.host, - port=constants.Redis.port, - password=constants.Redis.password, - ) # Can be overridden for testing - - def __init__(self, namespace: Optional[str] = None) -> None: - """Initialize the RedisDict with the right namespace.""" - super().__init__() - self._has_custom_namespace = namespace is not None - - if self._has_custom_namespace: - self._set_namespace(namespace) - else: - self.namespace = "global" - - def _set_namespace(self, namespace: str) -> None: - """Try to set the namespace, but do not permit collisions.""" - while namespace in self._namespaces: - namespace = namespace + "_" - - self._namespaces.append(namespace) - self._namespace = namespace - - def __set_name__(self, owner: object, attribute_name: str) -> None: - """ - Set the namespace to Class.attribute_name. - - Called automatically when this class is constructed inside a class as an attribute, as long as - no custom namespace is provided to the constructor. - """ - if not self._has_custom_namespace: - self._set_namespace(f"{owner.__name__}.{attribute_name}") - - def __repr__(self) -> str: - """Return a beautiful representation of this object instance.""" - return f"RedisDict(namespace={self._namespace!r})" - - def __eq__(self, other: RedisDict) -> bool: - """Check equality between two RedisDicts.""" - return self.items() == other.items() and self._namespace == other._namespace - - def __ne__(self, other: RedisDict) -> bool: - """Check inequality between two RedisDicts.""" - return self.items() != other.items() or self._namespace != other._namespace - - def __setitem__(self, key: ValidRedisKey, value: JSONSerializableType): - """Store an item in the Redis cache.""" - # JSON serialize the value before storing it. - json_value = json.dumps(value) - self._redis.hset(self._namespace, key, json_value) - - def __getitem__(self, key: ValidRedisKey): - """Get an item from the Redis cache.""" - value = self._redis.hget(self._namespace, key) - - if value: - return json.loads(value) - - def __delitem__(self, key: ValidRedisKey): - """Delete an item from the Redis cache.""" - self._redis.hdel(self._namespace, key) - - def __contains__(self, key: ValidRedisKey): - """Check if a key exists in the Redis cache.""" - return self._redis.hexists(self._namespace, key) - - def __iter__(self): - """Iterate all the items in the Redis cache.""" - keys = self._redis.hkeys(self._namespace) - return iter([key.decode('utf-8') for key in keys]) - - def __len__(self): - """Return the number of items in the Redis cache.""" - return self._redis.hlen(self._namespace) - - def copy(self) -> Dict: - """Convert to dict and return.""" - return dict(self.items()) - - def clear(self) -> None: - """Deletes the entire hash from the Redis cache.""" - self._redis.delete(self._namespace) - - def get(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: - """Get the item, but provide a default if not found.""" - if key in self: - return self[key] - else: - return default - - def pop(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: - """Get the item, remove it from the cache, and provide a default if not found.""" - value = self.get(key, default) - del self[key] - return value - - def popitem(self) -> JSONSerializableType: - """Get the last item added to the cache.""" - key = list(self.keys())[-1] - return self.pop(key) - - def setdefault(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: - """Try to get the item. If the item does not exist, set it to `default` and return that.""" - value = self.get(key) - - if value is None: - self[key] = default - return default -- cgit v1.2.3 From 23c2e7a42c13e03a1765e49f5dac3cfd4fed65b7 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Fri, 22 May 2020 02:34:51 +0200 Subject: Replace redis-py with aioredis. --- Pipfile | 2 +- Pipfile.lock | 162 +++++++++++++++++++++++++++++++++++++---------------------- 2 files changed, 104 insertions(+), 60 deletions(-) diff --git a/Pipfile b/Pipfile index 1d6cd7015..cd2f2ad7a 100644 --- a/Pipfile +++ b/Pipfile @@ -23,7 +23,7 @@ colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} statsd = "~=3.3" feedparser = "~=5.2" beautifulsoup4 = "~=4.9" -redis = "~=3.5" +aioredis = "~=1.3.1" [dev-packages] coverage = "~=5.0" diff --git a/Pipfile.lock b/Pipfile.lock index 25383b355..1941f6887 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "49c231092320b48c5a7618bf048a477f7e0ed33dcbfb71c6dc8f18ef819dd935" + "sha256": "c0b3e4d3e2c9ddb6ba28d2c09d521fe90ad4ea3df5c7ea7cd3a8b679fb3f85f9" }, "pipfile-spec": 6, "requires": { @@ -18,11 +18,11 @@ "default": { "aio-pika": { "hashes": [ - "sha256:9e4614636296e0040055bd6b304e97a38cc9796669ef391fc9b36649831d43ee", - "sha256:c9d242b3c7142d64b185feb6c5cce4154962610e89ec2e9b52bd69ef01f89b2f" + "sha256:c4cbbeb85b3c7bf81bc127371846cd949e6231717ce1e6ac7ee1dd5ede21f866", + "sha256:ec7fef24f588d90314873463ab4f2c3debce0bd8830e49e3786586be96bc2e8e" ], "index": "pypi", - "version": "==6.6.0" + "version": "==6.6.1" }, "aiodns": { "hashes": [ @@ -50,6 +50,14 @@ "index": "pypi", "version": "==3.6.2" }, + "aioredis": { + "hashes": [ + "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a", + "sha256:b61808d7e97b7cd5a92ed574937a079c9387fdadd22bfbfa7ad2fd319ecc26e3" + ], + "index": "pypi", + "version": "==1.3.1" + }, "aiormq": { "hashes": [ "sha256:41a9d4eb17db805f30ed172f3f609fe0c2b16657fb15b1b67df19d251dd93c0d", @@ -87,12 +95,12 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:594ca51a10d2b3443cbac41214e12dbb2a1cd57e1a7344659849e2e20ba6a8d8", - "sha256:a4bbe77fd30670455c5296242967a123ec28c37e9702a8a81bd2f20a4baf0368", - "sha256:d4e96ac9b0c3a6d3f0caae2e4124e6055c5dcafde8e2f831ff194c104f0775a0" + "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7", + "sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8", + "sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c" ], "index": "pypi", - "version": "==4.9.0" + "version": "==4.9.1" }, "certifi": { "hashes": [ @@ -205,6 +213,51 @@ "index": "pypi", "version": "==0.18.0" }, + "hiredis": { + "hashes": [ + "sha256:01b577f84c20ecc9c07fc4c184231b08e3c3942de096fa99978e053de231c423", + "sha256:01ff0900134166961c9e339df77c33b72f7edc5cb41739f0babcd9faa345926e", + "sha256:03ed34a13316d0c34213c4fd46e0fa3a5299073f4d4f08e93fed8c2108b399b3", + "sha256:040436e91df5143aff9e0debb49530d0b17a6bd52200ce568621c31ef581b10d", + "sha256:091eb38fbf968d1c5b703e412bbbd25f43a7967d8400842cee33a5a07b33c27b", + "sha256:102f9b9dc6ed57feb3a7c9bdf7e71cb7c278fe8df1edfcfe896bc3e0c2be9447", + "sha256:2b4b392c7e3082860c8371fab3ae762139090f9115819e12d9f56060f9ede05d", + "sha256:2c9cc0b986397b833073f466e6b9e9c70d1d4dc2c2c1b3e9cae3a23102ff296c", + "sha256:2fa65a9df683bca72073cd77709ddeb289ea2b114d3775d225fbbcc5faf808c5", + "sha256:38437a681f17c975fd22349e72c29bc643f8e7eb2d6dc5df419eac59afa4d7ce", + "sha256:3b3428fa3cf1ee178807b52c9bee8950ab94cd4eaa9bfae8c1bbae3c49501d34", + "sha256:3dd8c2fae7f5494978facb0e93297dd627b1a3f536f3b070cf0a7d9157a07dcb", + "sha256:4414a96c212e732723b5c3d7c04d386ebbb2ec359e1de646322cbc3f875cbd0d", + "sha256:48c627581ad4ef60adbac980981407939acf13a0e18f093502c7b542223c4f19", + "sha256:4a60e71625a2d78d8ab84dfb2fa2cfd9458c964b6e6c04fea76d9ade153fb371", + "sha256:585ace09f434e43d8a8dbeb366865b1a044d7c06319b3c7372a0a00e63b860f4", + "sha256:74b364b3f06c9cf0a53f7df611045bc9437ed972a283fa1f0b12537236d23ddc", + "sha256:75c65c3850e89e9daa68d1b9bedd5806f177d60aa5a7b0953b4829481cfc1f72", + "sha256:7f052de8bf744730a9120dbdc67bfeb7605a01f69fb8e7ba5c475af33c24e145", + "sha256:8113a7d5e87ecf57cd4ae263cc9e429adb9a3e59f5a7768da5d3312a8d0a051a", + "sha256:84857ce239eb8ed191ac78e77ff65d52902f00f30f4ee83bf80eb71da73b70e6", + "sha256:8644a48ddc4a40b3e3a6b9443f396c2ee353afb2d45656c4fc68d04a82e8e3f7", + "sha256:936aa565e673536e8a211e43ec43197406f24cd1f290138bd143765079c8ba00", + "sha256:9afeb88c67bbc663b9f27385c496da056d06ad87f55df6e393e1516cfecb0461", + "sha256:9d62cc7880110e4f83b0a51d218f465d3095e2751fbddd34e553dbd106a929ff", + "sha256:a1fadd062fc8d647ff39220c57ea2b48c99bb73f18223828ec97f88fc27e7898", + "sha256:a7754a783b1e5d6f627c19d099b178059c62f782ab62b4d8ba165b9fbc2ee34c", + "sha256:aa59dd63bb3f736de4fc2d080114429d5d369dfb3265f771778e8349d67a97a4", + "sha256:ae2ee0992f8de249715435942137843a93db204dd7db1e7cc9bdc5a8436443e8", + "sha256:b36842d7cf32929d568f37ec5b3173b72b2ec6572dec4d6be6ce774762215aee", + "sha256:bcbf9379c553b5facc6c04c1e5569b44b38ff16bcbf354676287698d61ee0c92", + "sha256:cbccbda6f1c62ab460449d9c85fdf24d0d32a6bf45176581151e53cc26a5d910", + "sha256:d0caf98dfb8af395d6732bd16561c0a2458851bea522e39f12f04802dbf6f502", + "sha256:d6456afeddba036def1a36d8a2758eca53202308d83db20ab5d0b66590919627", + "sha256:dbaef9a21a4f10bc281684ee4124f169e62bb533c2a92b55f8c06f64f9af7b8f", + "sha256:dce84916c09aaece006272b37234ae84a8ed13abb3a4d341a23933b8701abfb5", + "sha256:eb8c9c8b9869539d58d60ff4a28373a22514d40495911451343971cb4835b7a9", + "sha256:efc98b14ee3a8595e40b1425e8d42f5fd26f11a7b215a81ef9259068931754f4", + "sha256:fa2dc05b87d97acc1c6ae63f3e0f39eae5246565232484b08db6bf2dc1580678", + "sha256:fe7d6ce9f6a5fbe24f09d95ea93e9c7271abc4e1565da511e1449b107b4d7848" + ], + "version": "==1.0.1" + }, "humanfriendly": { "hashes": [ "sha256:bf52ec91244819c780341a3438d5d7b09f431d3f113a475147ac9b7b167a3d12", @@ -235,36 +288,36 @@ }, "lxml": { "hashes": [ - "sha256:06d4e0bbb1d62e38ae6118406d7cdb4693a3fa34ee3762238bcb96c9e36a93cd", - "sha256:0701f7965903a1c3f6f09328c1278ac0eee8f56f244e66af79cb224b7ef3801c", - "sha256:1f2c4ec372bf1c4a2c7e4bb20845e8bcf8050365189d86806bad1e3ae473d081", - "sha256:4235bc124fdcf611d02047d7034164897ade13046bda967768836629bc62784f", - "sha256:5828c7f3e615f3975d48f40d4fe66e8a7b25f16b5e5705ffe1d22e43fb1f6261", - "sha256:585c0869f75577ac7a8ff38d08f7aac9033da2c41c11352ebf86a04652758b7a", - "sha256:5d467ce9c5d35b3bcc7172c06320dddb275fea6ac2037f72f0a4d7472035cea9", - "sha256:63dbc21efd7e822c11d5ddbedbbb08cd11a41e0032e382a0fd59b0b08e405a3a", - "sha256:7bc1b221e7867f2e7ff1933165c0cec7153dce93d0cdba6554b42a8beb687bdb", - "sha256:8620ce80f50d023d414183bf90cc2576c2837b88e00bea3f33ad2630133bbb60", - "sha256:8a0ebda56ebca1a83eb2d1ac266649b80af8dd4b4a3502b2c1e09ac2f88fe128", - "sha256:90ed0e36455a81b25b7034038e40880189169c308a3df360861ad74da7b68c1a", - "sha256:95e67224815ef86924fbc2b71a9dbd1f7262384bca4bc4793645794ac4200717", - "sha256:afdb34b715daf814d1abea0317b6d672476b498472f1e5aacbadc34ebbc26e89", - "sha256:b4b2c63cc7963aedd08a5f5a454c9f67251b1ac9e22fd9d72836206c42dc2a72", - "sha256:d068f55bda3c2c3fcaec24bd083d9e2eede32c583faf084d6e4b9daaea77dde8", - "sha256:d5b3c4b7edd2e770375a01139be11307f04341ec709cf724e0f26ebb1eef12c3", - "sha256:deadf4df349d1dcd7b2853a2c8796593cc346600726eff680ed8ed11812382a7", - "sha256:df533af6f88080419c5a604d0d63b2c33b1c0c4409aba7d0cb6de305147ea8c8", - "sha256:e4aa948eb15018a657702fee0b9db47e908491c64d36b4a90f59a64741516e77", - "sha256:e5d842c73e4ef6ed8c1bd77806bf84a7cb535f9c0cf9b2c74d02ebda310070e1", - "sha256:ebec08091a22c2be870890913bdadd86fcd8e9f0f22bcb398abd3af914690c15", - "sha256:edc15fcfd77395e24543be48871c251f38132bb834d9fdfdad756adb6ea37679", - "sha256:f2b74784ed7e0bc2d02bd53e48ad6ba523c9b36c194260b7a5045071abbb1012", - "sha256:fa071559f14bd1e92077b1b5f6c22cf09756c6de7139370249eb372854ce51e6", - "sha256:fd52e796fee7171c4361d441796b64df1acfceb51f29e545e812f16d023c4bbc", - "sha256:fe976a0f1ef09b3638778024ab9fb8cde3118f203364212c198f71341c0715ca" - ], - "index": "pypi", - "version": "==4.5.0" + "sha256:06748c7192eab0f48e3d35a7adae609a329c6257495d5e53878003660dc0fec6", + "sha256:0790ddca3f825dd914978c94c2545dbea5f56f008b050e835403714babe62a5f", + "sha256:1aa7a6197c1cdd65d974f3e4953764eee3d9c7b67e3966616b41fab7f8f516b7", + "sha256:22c6d34fdb0e65d5f782a4d1a1edb52e0a8365858dafb1c08cb1d16546cf0786", + "sha256:2754d4406438c83144f9ffd3628bbe2dcc6d62b20dbc5c1ec4bc4385e5d44b42", + "sha256:27ee0faf8077c7c1a589573b1450743011117f1aa1a91d5ae776bbc5ca6070f2", + "sha256:2b02c106709466a93ed424454ce4c970791c486d5fcdf52b0d822a7e29789626", + "sha256:2d1ddce96cf15f1254a68dba6935e6e0f1fe39247de631c115e84dd404a6f031", + "sha256:4f282737d187ae723b2633856085c31ae5d4d432968b7f3f478a48a54835f5c4", + "sha256:51bb4edeb36d24ec97eb3e6a6007be128b720114f9a875d6b370317d62ac80b9", + "sha256:7eee37c1b9815e6505847aa5e68f192e8a1b730c5c7ead39ff317fde9ce29448", + "sha256:7fd88cb91a470b383aafad554c3fe1ccf6dfb2456ff0e84b95335d582a799804", + "sha256:9144ce36ca0824b29ebc2e02ca186e54040ebb224292072250467190fb613b96", + "sha256:925baf6ff1ef2c45169f548cc85204433e061360bfa7d01e1be7ae38bef73194", + "sha256:a636346c6c0e1092ffc202d97ec1843a75937d8c98aaf6771348ad6422e44bb0", + "sha256:a87dbee7ad9dce3aaefada2081843caf08a44a8f52e03e0a4cc5819f8398f2f4", + "sha256:a9e3b8011388e7e373565daa5e92f6c9cb844790dc18e43073212bb3e76f7007", + "sha256:afb53edf1046599991fb4a7d03e601ab5f5422a5435c47ee6ba91ec3b61416a6", + "sha256:b26719890c79a1dae7d53acac5f089d66fd8cc68a81f4e4bd355e45470dc25e1", + "sha256:b7462cdab6fffcda853338e1741ce99706cdf880d921b5a769202ea7b94e8528", + "sha256:b77975465234ff49fdad871c08aa747aae06f5e5be62866595057c43f8d2f62c", + "sha256:c47a8a5d00060122ca5908909478abce7bbf62d812e3fc35c6c802df8fb01fe7", + "sha256:c79e5debbe092e3c93ca4aee44c9a7631bdd407b2871cb541b979fd350bbbc29", + "sha256:d8d40e0121ca1606aa9e78c28a3a7d88a05c06b3ca61630242cded87d8ce55fa", + "sha256:ee2be8b8f72a2772e72ab926a3bccebf47bb727bda41ae070dc91d1fb759b726", + "sha256:f95d28193c3863132b1f55c1056036bf580b5a488d908f7d22a04ace8935a3a9", + "sha256:fadd2a63a2bfd7fb604508e553d1cf68eca250b2fbdbd81213b5f6f2fbf23529" + ], + "index": "pypi", + "version": "==4.5.1" }, "markdownify": { "hashes": [ @@ -349,10 +402,10 @@ }, "packaging": { "hashes": [ - "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3", - "sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752" + "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", + "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" ], - "version": "==20.3" + "version": "==20.4" }, "pamqp": { "hashes": [ @@ -448,14 +501,6 @@ "index": "pypi", "version": "==5.3.1" }, - "redis": { - "hashes": [ - "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", - "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" - ], - "index": "pypi", - "version": "==3.5.2" - }, "requests": { "hashes": [ "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", @@ -474,10 +519,10 @@ }, "six": { "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "version": "==1.14.0" + "version": "==1.15.0" }, "snowballstemmer": { "hashes": [ @@ -837,15 +882,14 @@ "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" ], - "index": "pypi", "version": "==3.5.2" }, "six": { "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "version": "==1.14.0" + "version": "==1.15.0" }, "snowballstemmer": { "hashes": [ @@ -878,10 +922,10 @@ }, "virtualenv": { "hashes": [ - "sha256:b4c14d4d73a0c23db267095383c4276ef60e161f94fde0427f2f21a0132dde74", - "sha256:fd0e54dec8ac96c1c7c87daba85f0a59a7c37fe38748e154306ca21c73244637" + "sha256:a116629d4e7f4d03433b8afa27f43deba09d48bc48f5ecefa4f015a178efb6cf", + "sha256:a730548b27366c5e6cbdf6f97406d861cccece2e22275e8e1a757aeff5e00c70" ], - "version": "==20.0.20" + "version": "==20.0.21" } } } -- cgit v1.2.3 From 3f596d5245403b75759a9c73029768d9e4510303 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Fri, 22 May 2020 02:38:55 +0200 Subject: Opens a Redis connection in the Bot class. This global connection is the one we will be using in RedisCache to power all our commands. This also ensures that connection is closed when the bot starts its shutdown process. --- bot/bot.py | 17 +++++++++++++++++ bot/utils/__init__.py | 4 ++-- bot/utils/redis_cache.py | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/bot/bot.py b/bot/bot.py index a85a22aa9..f55eec5bb 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -5,6 +5,7 @@ import warnings from typing import Optional import aiohttp +import aioredis import discord from discord.ext import commands from sentry_sdk import push_scope @@ -28,11 +29,13 @@ class Bot(commands.Bot): super().__init__(*args, **kwargs) self.http_session: Optional[aiohttp.ClientSession] = None + self.redis_session: Optional[aioredis.Redis] = None self.api_client = api.APIClient(loop=self.loop) self._connector = None self._resolver = None self._guild_available = asyncio.Event() + self._redis_ready = asyncio.Event() statsd_url = constants.Stats.statsd_host @@ -42,8 +45,18 @@ class Bot(commands.Bot): # will effectively disable stats. statsd_url = "127.0.0.1" + asyncio.create_task(self._create_redis_session()) + self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") + async def _create_redis_session(self) -> None: + """Create the Redis connection pool, and then open the redis event gate.""" + self.redis_session = await aioredis.create_redis_pool( + address=(constants.Redis.host, constants.Redis.port), + password=constants.Redis.password, + ) + self._redis_ready.set() + def add_cog(self, cog: commands.Cog) -> None: """Adds a "cog" to the bot and logs the operation.""" super().add_cog(cog) @@ -78,6 +91,10 @@ class Bot(commands.Bot): if self.stats._transport: self.stats._transport.close() + if self.redis_session: + self.redis_session.close() + await self.redis_session.wait_closed() + async def login(self, *args, **kwargs) -> None: """Re-create the connector and set up sessions before logging into Discord.""" self._recreate() diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 5ce383bf2..c5a12d5e3 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -2,9 +2,9 @@ from abc import ABCMeta from discord.ext.commands import CogMeta -from bot.utils.redis_dict import RedisDict +from bot.utils.redis_cache import RedisCache -__all__ = ['RedisDict', 'CogABCMeta'] +__all__ = ['RedisCache', 'CogABCMeta'] class CogABCMeta(CogMeta, ABCMeta): diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index d0a7eba4a..467f16767 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -61,7 +61,7 @@ class RedisCache: for attribute in vars(instance).values(): if isinstance(attribute, Bot): self.bot = attribute - self._redis = self.bot.redis_pool + self._redis = self.bot.redis_session return self else: raise RuntimeError("Cannot initialize a RedisCache without a `Bot` instance.") -- cgit v1.2.3 From e23aa887959059e17fc21dcab9c83db20dc987f5 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 21 May 2020 20:29:44 -0700 Subject: Token remover: decode ID using URL-safe base64 Though I've not seen an ID with neither + and \ nor - and _, given that the timestamp uses URL-safe encoding, the ID probably does too. --- bot/cogs/token_remover.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index cae482e6e..5b4598959 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -150,7 +150,7 @@ class TokenRemover(Cog): b64_content = utils.pad_base64(b64_content) try: - decoded_bytes: bytes = base64.b64decode(b64_content) + decoded_bytes = base64.urlsafe_b64decode(b64_content) string = decoded_bytes.decode('utf-8') # isdigit on its own would match a lot of other Unicode characters, hence the isascii. -- cgit v1.2.3 From 95ef2dc01143902289c9aacde7969fb5c9e1a85c Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 21 May 2020 21:34:10 -0700 Subject: Token remover: match only base64 in regex Making the regex more accurate reduces false positives at an earlier stage. There's no benefit to matching non-base64 as that would just be weeded out as invalid at a later stage anyway when it tries to decode it. --- bot/cogs/token_remover.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index 5b4598959..fa0647828 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -29,13 +29,12 @@ DELETION_MESSAGE_TEMPLATE = ( ) DISCORD_EPOCH = 1_420_070_400_000 TOKEN_EPOCH = 1_293_840_000 -TOKEN_RE = re.compile( - r"[^\s\.()\"']+" # Matches token part 1: The user ID string, encoded as base64 - r"\." # Matches a literal dot between the token parts - r"[^\s\.()\"']+" # Matches token part 2: The creation timestamp, as an integer - r"\." # Matches a literal dot between the token parts - r"[^\s\.()\"']+" # Matches token part 3: The HMAC, unused by us, but check that it isn't empty -) + +# Three parts delimited by dots: user ID, creation timestamp, HMAC. +# The HMAC isn't parsed further, but it's in the regex to ensure it at least exists in the string. +# Each part only matches base64 URL-safe characters. +# Padding has never been observed, but the padding character '=' is matched just in case. +TOKEN_RE = re.compile(r"[\w-=]+\.[\w-=]+\.[\w-=]+", re.ASCII) class TokenRemover(Cog): -- cgit v1.2.3 From a46eff8d976cf65155f27ed75f49bd3e58155c84 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 22 May 2020 08:12:08 +0300 Subject: Help: Fix docstrings of `is_empty` and `on_message_delete` Co-authored-by: Mark --- bot/cogs/help_channels.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 4415ce550..ed1f7c55e 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -719,7 +719,11 @@ class HelpChannels(Scheduler, commands.Cog): @commands.Cog.listener() async def on_message_delete(self, msg: discord.Message) -> None: - """Reschedule dormant when help channel is empty.""" + """ + Reschedule an in-use channel to become dormant sooner if the channel is empty. + + The new time for the dormant task is configured with `HelpChannels.deleted_idle_minutes`. + """ if not self.is_in_category(msg.channel, constants.Categories.help_in_use) or not self.is_empty(msg.channel): return @@ -732,7 +736,7 @@ class HelpChannels(Scheduler, commands.Cog): self.schedule_task(msg.channel.id, task) async def is_empty(self, channel: discord.TextChannel) -> bool: - """Check is last message bot sent available message.""" + """Return True if the most recent message in `channel` is the bot's `AVAILABLE_MSG`.""" msg = await self.get_last_message(channel) if not msg or not msg.author.bot or not msg.embeds: return False -- cgit v1.2.3 From 841ce9ba155d2aea3011500f5129d6b3dd309b99 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 22 May 2020 08:29:41 +0300 Subject: Help: Create `embed_description_match` - Created function `embed_description_match`. - Implemented this to `is_empty` - Implemented this to `is_dormant_message` --- bot/cogs/help_channels.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index ed1f7c55e..554fdc55e 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -440,11 +440,18 @@ class HelpChannels(Scheduler, commands.Cog): def is_dormant_message(self, message: t.Optional[discord.Message]) -> bool: """Return True if the contents of the `message` match `DORMANT_MSG`.""" - if not message or not message.embeds: + if not message: + return False + + return self.embed_description_match(message, DORMANT_MSG) + + def embed_description_match(self, message: discord.Message, text: str) -> bool: + """Return `True` if `message` embed description match with `text`.""" + if not message.embeds: return False embed = message.embeds[0] - return message.author == self.bot.user and embed.description.strip() == DORMANT_MSG.strip() + return message.author == self.bot.user and embed.description.strip() == text.strip() @staticmethod def is_in_category(channel: discord.TextChannel, category_id: int) -> bool: @@ -722,7 +729,7 @@ class HelpChannels(Scheduler, commands.Cog): """ Reschedule an in-use channel to become dormant sooner if the channel is empty. - The new time for the dormant task is configured with `HelpChannels.deleted_idle_minutes`. + The new time for the dormant task is configured with `HelpChannels.deleted_idle_minutes`. """ if not self.is_in_category(msg.channel, constants.Categories.help_in_use) or not self.is_empty(msg.channel): return @@ -738,14 +745,10 @@ class HelpChannels(Scheduler, commands.Cog): async def is_empty(self, channel: discord.TextChannel) -> bool: """Return True if the most recent message in `channel` is the bot's `AVAILABLE_MSG`.""" msg = await self.get_last_message(channel) - if not msg or not msg.author.bot or not msg.embeds: + if not msg: return False - embed = msg.embeds[0] - if embed.description == AVAILABLE_MSG: - return True - else: - return False + return self.embed_description_match(msg, AVAILABLE_MSG) async def reset_send_permissions(self) -> None: """Reset send permissions in the Available category for claimants.""" -- cgit v1.2.3 From b3619949a17ba40a3b1f6364cf83464275717283 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 22 May 2020 08:35:03 +0300 Subject: Eval Stats: Replaced `elif` with `else` on icon check Co-authored-by: Mark --- bot/cogs/snekbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index efff6d815..e2e55e7ca 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -208,7 +208,7 @@ class Snekbox(Cog): # Collect stats of eval fails + successes if icon == ":x:": self.bot.stats.incr("snekbox.python.fail") - elif icon in (":warning:", ":white_check_mark:"): + else: self.bot.stats.incr("snekbox.python.success") response = await ctx.send(msg) -- cgit v1.2.3 From d3550ce1138e1b00e64ba355a09a08b480b077e6 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 11:11:43 -0700 Subject: HelpChannels: fix `is_empty` not being awaited --- bot/cogs/help_channels.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 554fdc55e..2aec22be4 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -468,7 +468,7 @@ class HelpChannels(Scheduler, commands.Cog): """ log.trace(f"Handling in-use channel #{channel} ({channel.id}).") - if not self.is_empty(channel): + if not await self.is_empty(channel): idle_seconds = constants.HelpChannels.idle_minutes * 60 else: idle_seconds = constants.HelpChannels.deleted_idle_minutes * 60 @@ -731,7 +731,10 @@ class HelpChannels(Scheduler, commands.Cog): The new time for the dormant task is configured with `HelpChannels.deleted_idle_minutes`. """ - if not self.is_in_category(msg.channel, constants.Categories.help_in_use) or not self.is_empty(msg.channel): + if not self.is_in_category(msg.channel, constants.Categories.help_in_use): + return + + if not await self.is_empty(msg.channel): return log.info(f"Claimant of #{msg.channel} ({msg.author}) deleted message, channel is empty now. Rescheduling task.") -- cgit v1.2.3 From 8deeeca83c6c2c3de3b856ea8d6f94f8b5db3526 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 11:16:21 -0700 Subject: HelpChannels: rename `embed_description_match` --- bot/cogs/help_channels.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 2aec22be4..b9b577256 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -443,15 +443,15 @@ class HelpChannels(Scheduler, commands.Cog): if not message: return False - return self.embed_description_match(message, DORMANT_MSG) + return self.match_bot_embed(message, DORMANT_MSG) - def embed_description_match(self, message: discord.Message, text: str) -> bool: - """Return `True` if `message` embed description match with `text`.""" + def match_bot_embed(self, message: discord.Message, description: str) -> bool: + """Return `True` if the bot's `message`'s embed description matches `description`.""" if not message.embeds: return False embed = message.embeds[0] - return message.author == self.bot.user and embed.description.strip() == text.strip() + return message.author == self.bot.user and embed.description.strip() == description.strip() @staticmethod def is_in_category(channel: discord.TextChannel, category_id: int) -> bool: @@ -751,7 +751,7 @@ class HelpChannels(Scheduler, commands.Cog): if not msg: return False - return self.embed_description_match(msg, AVAILABLE_MSG) + return self.match_bot_embed(msg, AVAILABLE_MSG) async def reset_send_permissions(self) -> None: """Reset send permissions in the Available category for claimants.""" -- cgit v1.2.3 From e8266b8e1029f31dea3ad6ecbe36b7df56b8acdc Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 11:20:46 -0700 Subject: HelpChannels: move message None check inside `match_bot_embed` It was being done repeatedly outside the function so let's move it in to reduce redundancy. --- bot/cogs/help_channels.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index b9b577256..07acff34d 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -440,14 +440,11 @@ class HelpChannels(Scheduler, commands.Cog): def is_dormant_message(self, message: t.Optional[discord.Message]) -> bool: """Return True if the contents of the `message` match `DORMANT_MSG`.""" - if not message: - return False - return self.match_bot_embed(message, DORMANT_MSG) - def match_bot_embed(self, message: discord.Message, description: str) -> bool: + def match_bot_embed(self, message: t.Optional[discord.Message], description: str) -> bool: """Return `True` if the bot's `message`'s embed description matches `description`.""" - if not message.embeds: + if not message or not message.embeds: return False embed = message.embeds[0] @@ -748,9 +745,6 @@ class HelpChannels(Scheduler, commands.Cog): async def is_empty(self, channel: discord.TextChannel) -> bool: """Return True if the most recent message in `channel` is the bot's `AVAILABLE_MSG`.""" msg = await self.get_last_message(channel) - if not msg: - return False - return self.match_bot_embed(msg, AVAILABLE_MSG) async def reset_send_permissions(self) -> None: -- cgit v1.2.3 From 278ae309be27058920424c4049272bd5171bc158 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 11:22:27 -0700 Subject: HelpChannels: remove `is_dormant_message` At this point, it's just a thin wrapper to call another function. It's redundant. --- bot/cogs/help_channels.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 07acff34d..f0e6746f0 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -438,10 +438,6 @@ class HelpChannels(Scheduler, commands.Cog): """Return True if `member` has the 'Help Cooldown' role.""" return any(constants.Roles.help_cooldown == role.id for role in member.roles) - def is_dormant_message(self, message: t.Optional[discord.Message]) -> bool: - """Return True if the contents of the `message` match `DORMANT_MSG`.""" - return self.match_bot_embed(message, DORMANT_MSG) - def match_bot_embed(self, message: t.Optional[discord.Message], description: str) -> bool: """Return `True` if the bot's `message`'s embed description matches `description`.""" if not message or not message.embeds: @@ -822,7 +818,7 @@ class HelpChannels(Scheduler, commands.Cog): embed = discord.Embed(description=AVAILABLE_MSG) msg = await self.get_last_message(channel) - if self.is_dormant_message(msg): + if self.match_bot_embed(msg, DORMANT_MSG): log.trace(f"Found dormant message {msg.id} in {channel_info}; editing it.") await msg.edit(embed=embed) else: -- cgit v1.2.3 From 57fe4bf893e94289b5b6f7158ff2d6b92b1e3fae Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Fri, 22 May 2020 22:56:25 +0200 Subject: Set up async testbed --- bot/bot.py | 3 +- bot/utils/redis_cache.py | 13 ++- tests/bot/utils/test_redis_cache.py | 128 ++++++++++++++++++++++++ tests/bot/utils/test_redis_dict.py | 189 ------------------------------------ 4 files changed, 135 insertions(+), 198 deletions(-) create mode 100644 tests/bot/utils/test_redis_cache.py delete mode 100644 tests/bot/utils/test_redis_dict.py diff --git a/bot/bot.py b/bot/bot.py index f55eec5bb..8a3805989 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -45,8 +45,7 @@ class Bot(commands.Bot): # will effectively disable stats. statsd_url = "127.0.0.1" - asyncio.create_task(self._create_redis_session()) - + self.loop.create_task(self._create_redis_session()) self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") async def _create_redis_session(self) -> None: diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 467f16767..483bbc2cd 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -30,8 +30,8 @@ class RedisCache: def __init__(self) -> None: """Raise a NotImplementedError if `__set_name__` hasn't been run.""" - if not self._namespace: - raise NotImplementedError("RedisCache must be a class attribute.") + self._namespace = None + self.bot = None def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" @@ -47,8 +47,7 @@ class RedisCache: Called automatically when this class is constructed inside a class as an attribute. """ - if not self._has_custom_namespace: - self._set_namespace(f"{owner.__name__}.{attribute_name}") + self._set_namespace(f"{owner.__name__}.{attribute_name}") def __get__(self, instance: RedisCache, owner: Any) -> RedisCache: """Fetch the Bot instance, we need it for the redis pool.""" @@ -106,9 +105,9 @@ class RedisCache: async def pop(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: """Get the item, remove it from the cache, and provide a default if not found.""" - value = await self.get(key, default) - await self.delete(key) - return value + # value = await self.get(key, default) + # await self.delete(key) + # return value async def update(self) -> None: """Update the Redis cache with multiple values.""" diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py new file mode 100644 index 000000000..f6344803f --- /dev/null +++ b/tests/bot/utils/test_redis_cache.py @@ -0,0 +1,128 @@ +import asyncio +import unittest +from unittest.mock import MagicMock + +import fakeredis.aioredis + +from bot.bot import Bot +from bot.utils import RedisCache + + +class RedisCacheTests(unittest.IsolatedAsyncioTestCase): + """Tests the RedisDict class from utils.redis_dict.py.""" + + redis = RedisCache() + + async def asyncSetUp(self): # noqa: N802 - this special method can't be all lowercase + """Sets up the objects that only have to be initialized once.""" + self.bot = MagicMock( + spec=Bot, + redis_session=await fakeredis.aioredis.create_redis_pool(), + _redis_ready=asyncio.Event(), + ) + self.bot._redis_ready.set() + + def test_class_attribute_namespace(self): + """Test that RedisDict creates a namespace automatically for class attributes.""" + self.assertEqual(self.redis._namespace, "RedisCacheTests.redis") + # Test that errors are raised when this isn't true. + + # def test_set_get_item(self): + # """Test that users can set and get items from the RedisDict.""" + # self.redis['favorite_fruit'] = 'melon' + # self.redis['favorite_number'] = 86 + # self.assertEqual(self.redis['favorite_fruit'], 'melon') + # self.assertEqual(self.redis['favorite_number'], 86) + # + # def test_set_item_types(self): + # """Test that setitem rejects keys and values that are not strings, ints or floats.""" + # fruits = ["lemon", "melon", "apple"] + # + # with self.assertRaises(DataError): + # self.redis[fruits] = "nice" + # + # def test_contains(self): + # """Test that we can reliably use the `in` operator with our RedisDict.""" + # self.redis['favorite_country'] = "Burkina Faso" + # + # self.assertIn('favorite_country', self.redis) + # self.assertNotIn('favorite_dentist', self.redis) + # + # def test_items(self): + # """Test that the RedisDict can be iterated.""" + # self.redis.clear() + # test_cases = ( + # ('favorite_turtle', 'Donatello'), + # ('second_favorite_turtle', 'Leonardo'), + # ('third_favorite_turtle', 'Raphael'), + # ) + # for key, value in test_cases: + # self.redis[key] = value + # + # # Test regular iteration + # for test_case, key in zip(test_cases, self.redis): + # value = test_case[1] + # self.assertEqual(self.redis[key], value) + # + # # Test .items iteration + # for key, value in self.redis.items(): + # self.assertEqual(self.redis[key], value) + # + # # Test .keys iteration + # for test_case, key in zip(test_cases, self.redis.keys()): + # value = test_case[1] + # self.assertEqual(self.redis[key], value) + # + # def test_length(self): + # """Test that we can get the correct len() from the RedisDict.""" + # self.redis.clear() + # self.redis['one'] = 1 + # self.redis['two'] = 2 + # self.redis['three'] = 3 + # self.assertEqual(len(self.redis), 3) + # + # self.redis['four'] = 4 + # self.assertEqual(len(self.redis), 4) + # + # def test_to_dict(self): + # """Test that the .copy method returns a workable dictionary copy.""" + # copy = self.redis.copy() + # local_copy = dict(self.redis.items()) + # self.assertIs(type(copy), dict) + # self.assertEqual(copy, local_copy) + # + # def test_clear(self): + # """Test that the .clear method removes the entire hash.""" + # self.redis.clear() + # self.redis['teddy'] = "with me" + # self.redis['in my dreams'] = "you have a weird hat" + # self.assertEqual(len(self.redis), 2) + # + # self.redis.clear() + # self.assertEqual(len(self.redis), 0) + # + # def test_pop(self): + # """Test that we can .pop an item from the RedisDict.""" + # self.redis.clear() + # self.redis['john'] = 'was afraid' + # + # self.assertEqual(self.redis.pop('john'), 'was afraid') + # self.assertEqual(self.redis.pop('pete', 'breakneck'), 'breakneck') + # self.assertEqual(len(self.redis), 0) + # + # def test_update(self): + # """Test that we can .update the RedisDict with multiple items.""" + # self.redis.clear() + # self.redis["reckfried"] = "lona" + # self.redis["bel air"] = "prince" + # self.redis.update({ + # "reckfried": "jona", + # "mega": "hungry, though", + # }) + # + # result = { + # "reckfried": "jona", + # "bel air": "prince", + # "mega": "hungry, though", + # } + # self.assertEqual(self.redis.copy(), result) diff --git a/tests/bot/utils/test_redis_dict.py b/tests/bot/utils/test_redis_dict.py deleted file mode 100644 index f422887ce..000000000 --- a/tests/bot/utils/test_redis_dict.py +++ /dev/null @@ -1,189 +0,0 @@ -import unittest - -import fakeredis -from redis import DataError - -from bot.utils import RedisDict - -redis_server = fakeredis.FakeServer() -RedisDict._redis = fakeredis.FakeStrictRedis(server=redis_server) - - -class RedisDictTests(unittest.TestCase): - """Tests the RedisDict class from utils.redis_dict.py.""" - - redis = RedisDict() - - def test_class_attribute_namespace(self): - """Test that RedisDict creates a namespace automatically for class attributes.""" - self.assertEqual(self.redis._namespace, "RedisDictTests.redis") - - def test_custom_namespace(self): - """Test that users can set a custom namespaces which never collide.""" - test_cases = ( - (RedisDict("firedog")._namespace, "firedog"), - (RedisDict("firedog")._namespace, "firedog_"), - (RedisDict("firedog")._namespace, "firedog__"), - ) - - for test_case, result in test_cases: - self.assertEqual(test_case, result) - - def test_custom_namespace_takes_precedence(self): - """Test that custom namespaces take precedence over class attribute ones.""" - class LemonJuice: - citrus = RedisDict("citrus") - watercat = RedisDict() - - test_class = LemonJuice() - self.assertEqual(test_class.citrus._namespace, "citrus") - self.assertEqual(test_class.watercat._namespace, "LemonJuice.watercat") - - def test_set_get_item(self): - """Test that users can set and get items from the RedisDict.""" - self.redis['favorite_fruit'] = 'melon' - self.redis['favorite_number'] = 86 - self.assertEqual(self.redis['favorite_fruit'], 'melon') - self.assertEqual(self.redis['favorite_number'], 86) - - def test_set_item_value_types(self): - """Test that setitem rejects values that are not JSON serializable.""" - with self.assertRaises(TypeError): - self.redis['favorite_thing'] = object - self.redis['favorite_stuff'] = RedisDict - - def test_set_item_key_types(self): - """Test that setitem rejects keys that are not strings, ints or floats.""" - fruits = ["lemon", "melon", "apple"] - - with self.assertRaises(DataError): - self.redis[fruits] = "nice" - - def test_get_method(self): - """Test that the .get method works like in a dict.""" - self.redis['favorite_movie'] = 'Code Jam Highlights' - - self.assertEqual(self.redis.get('favorite_movie'), 'Code Jam Highlights') - self.assertEqual(self.redis.get('favorite_youtuber', 'pydis'), 'pydis') - self.assertIsNone(self.redis.get('favorite_dog')) - - def test_membership(self): - """Test that we can reliably use the `in` operator with our RedisDict.""" - self.redis['favorite_country'] = "Burkina Faso" - - self.assertIn('favorite_country', self.redis) - self.assertNotIn('favorite_dentist', self.redis) - - def test_del_item(self): - """Test that users can delete items from the RedisDict.""" - self.redis['favorite_band'] = "Radiohead" - self.assertIn('favorite_band', self.redis) - - del self.redis['favorite_band'] - self.assertNotIn('favorite_band', self.redis) - - def test_iter(self): - """Test that the RedisDict can be iterated.""" - self.redis.clear() - test_cases = ( - ('favorite_turtle', 'Donatello'), - ('second_favorite_turtle', 'Leonardo'), - ('third_favorite_turtle', 'Raphael'), - ) - for key, value in test_cases: - self.redis[key] = value - - # Test regular iteration - for test_case, key in zip(test_cases, self.redis): - value = test_case[1] - self.assertEqual(self.redis[key], value) - - # Test .items iteration - for key, value in self.redis.items(): - self.assertEqual(self.redis[key], value) - - # Test .keys iteration - for test_case, key in zip(test_cases, self.redis.keys()): - value = test_case[1] - self.assertEqual(self.redis[key], value) - - def test_len(self): - """Test that we can get the correct len() from the RedisDict.""" - self.redis.clear() - self.redis['one'] = 1 - self.redis['two'] = 2 - self.redis['three'] = 3 - self.assertEqual(len(self.redis), 3) - - self.redis['four'] = 4 - self.assertEqual(len(self.redis), 4) - - def test_copy(self): - """Test that the .copy method returns a workable dictionary copy.""" - copy = self.redis.copy() - local_copy = dict(self.redis.items()) - self.assertIs(type(copy), dict) - self.assertEqual(copy, local_copy) - - def test_clear(self): - """Test that the .clear method removes the entire hash.""" - self.redis.clear() - self.redis['teddy'] = "with me" - self.redis['in my dreams'] = "you have a weird hat" - self.assertEqual(len(self.redis), 2) - - self.redis.clear() - self.assertEqual(len(self.redis), 0) - - def test_pop(self): - """Test that we can .pop an item from the RedisDict.""" - self.redis.clear() - self.redis['john'] = 'was afraid' - - self.assertEqual(self.redis.pop('john'), 'was afraid') - self.assertEqual(self.redis.pop('pete', 'breakneck'), 'breakneck') - self.assertEqual(len(self.redis), 0) - - def test_popitem(self): - """Test that we can .popitem an item from the RedisDict.""" - self.redis.clear() - self.redis['john'] = 'the revalator' - self.redis['teddy'] = 'big bear' - - self.assertEqual(len(self.redis), 2) - self.assertEqual(self.redis.popitem(), 'big bear') - self.assertEqual(len(self.redis), 1) - - def test_setdefault(self): - """Test that we can .setdefault an item from the RedisDict.""" - self.redis.clear() - self.redis.setdefault('john', 'is yellow and weak') - self.assertEqual(self.redis['john'], 'is yellow and weak') - - with self.assertRaises(TypeError): - self.redis.setdefault('geisha', object) - - def test_update(self): - """Test that we can .update the RedisDict with multiple items.""" - self.redis.clear() - self.redis["reckfried"] = "lona" - self.redis["bel air"] = "prince" - self.redis.update({ - "reckfried": "jona", - "mega": "hungry, though", - }) - - result = { - "reckfried": "jona", - "bel air": "prince", - "mega": "hungry, though", - } - self.assertEqual(self.redis.copy(), result) - - def test_equals(self): - """Test that RedisDicts can be compared with == and !=.""" - new_redis_dict = RedisDict("firedog_the_sequel") - new_new_redis_dict = new_redis_dict - - self.assertEqual(new_redis_dict, new_new_redis_dict) - self.assertNotEqual(new_redis_dict, self.redis) -- cgit v1.2.3 From fd6f3d30b4c67f9a81346bb142d4696948fa2812 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 15:40:50 -0700 Subject: Fix assertion for `create_task` in duck pond tests The assertion wasn't using the assertion method. Furthermore, it was testing a non-existent function `create_loop` rather than `create_task`. --- tests/bot/cogs/test_duck_pond.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_duck_pond.py b/tests/bot/cogs/test_duck_pond.py index 7e6bfc748..a8c0107c6 100644 --- a/tests/bot/cogs/test_duck_pond.py +++ b/tests/bot/cogs/test_duck_pond.py @@ -45,7 +45,7 @@ class DuckPondTests(base.LoggingTestsMixin, unittest.IsolatedAsyncioTestCase): self.assertEqual(cog.bot, bot) self.assertEqual(cog.webhook_id, constants.Webhooks.duck_pond) - bot.loop.create_loop.called_once_with(cog.fetch_webhook()) + bot.loop.create_task.assert_called_once_with(cog.fetch_webhook()) def test_fetch_webhook_succeeds_without_connectivity_issues(self): """The `fetch_webhook` method waits until `READY` event and sets the `webhook` attribute.""" -- cgit v1.2.3 From 45e6f8dba869a367b01d99a596bd3355802d1fbe Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 15:44:04 -0700 Subject: Improve aiohttp context manager mocking in snekbox tests I'm not sure how it even managed to work before. It was calling the `post` coroutine (without specifying a URL) and then changing `__aenter__`. Now, a separate mock is created for the context manager and the `post` simply returns that mocked context manager. --- tests/bot/cogs/test_snekbox.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/bot/cogs/test_snekbox.py b/tests/bot/cogs/test_snekbox.py index 1dec0ccaf..84b273a7d 100644 --- a/tests/bot/cogs/test_snekbox.py +++ b/tests/bot/cogs/test_snekbox.py @@ -21,7 +21,10 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): """Post the eval code to the URLs.snekbox_eval_api endpoint.""" resp = MagicMock() resp.json = AsyncMock(return_value="return") - self.bot.http_session.post().__aenter__.return_value = resp + + context_manager = MagicMock() + context_manager.__aenter__.return_value = resp + self.bot.http_session.post.return_value = context_manager self.assertEqual(await self.cog.post_eval("import random"), "return") self.bot.http_session.post.assert_called_with( @@ -41,7 +44,10 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): key = "MarkDiamond" resp = MagicMock() resp.json = AsyncMock(return_value={"key": key}) - self.bot.http_session.post().__aenter__.return_value = resp + + context_manager = MagicMock() + context_manager.__aenter__.return_value = resp + self.bot.http_session.post.return_value = context_manager self.assertEqual( await self.cog.upload_output("My awesome output"), @@ -57,7 +63,10 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): """Output upload gracefully fallback if the upload fail.""" resp = MagicMock() resp.json = AsyncMock(side_effect=Exception) - self.bot.http_session.post().__aenter__.return_value = resp + + context_manager = MagicMock() + context_manager.__aenter__.return_value = resp + self.bot.http_session.post.return_value = context_manager log = logging.getLogger("bot.cogs.snekbox") with self.assertLogs(logger=log, level='ERROR'): -- cgit v1.2.3 From 6aed2f6b69b79b5a7e5f327819d026e7a63a7dab Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 16:15:23 -0700 Subject: Fix unawaited coro warning when instantiating Bot for MockBot's spec The fix is to mock the loop and pass it to the Bot. It will then set it as `self.loop` rather than trying to get an event loop from asyncio. The `create_task` patch has been moved to this loop mock rather than being done in MockBot to ensure that it applies to anything calling it when instantiating the Bot. --- tests/helpers.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 2b79a6c2a..2efeff7db 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -4,6 +4,7 @@ import collections import itertools import logging import unittest.mock +from asyncio import AbstractEventLoop from typing import Iterable, Optional import discord @@ -264,10 +265,16 @@ class MockAPIClient(CustomMockMixin, unittest.mock.MagicMock): spec_set = APIClient -# Create a Bot instance to get a realistic MagicMock of `discord.ext.commands.Bot` -bot_instance = Bot(command_prefix=unittest.mock.MagicMock()) -bot_instance.http_session = None -bot_instance.api_client = None +def _get_mock_loop() -> unittest.mock.Mock: + """Return a mocked asyncio.AbstractEventLoop.""" + loop = unittest.mock.create_autospec(spec=AbstractEventLoop, spec_set=True) + + # Since calling `create_task` on our MockBot does not actually schedule the coroutine object + # as a task in the asyncio loop, this `side_effect` calls `close()` on the coroutine object + # to prevent "has not been awaited"-warnings. + loop.create_task.side_effect = lambda coroutine: coroutine.close() + + return loop class MockBot(CustomMockMixin, unittest.mock.MagicMock): @@ -277,17 +284,14 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): Instances of this class will follow the specifications of `discord.ext.commands.Bot` instances. For more information, see the `MockGuild` docstring. """ - spec_set = bot_instance + spec_set = Bot(command_prefix=unittest.mock.MagicMock(), loop=_get_mock_loop()) additional_spec_asyncs = ("wait_for",) def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - self.api_client = MockAPIClient() - # Since calling `create_task` on our MockBot does not actually schedule the coroutine object - # as a task in the asyncio loop, this `side_effect` calls `close()` on the coroutine object - # to prevent "has not been awaited"-warnings. - self.loop.create_task.side_effect = lambda coroutine: coroutine.close() + self.loop = _get_mock_loop() + self.api_client = MockAPIClient(loop=self.loop) # Create a TextChannel instance to get a realistic MagicMock of `discord.TextChannel` -- cgit v1.2.3 From 1ad7833d800918efca06e5d6b2fbafdb0d757009 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 16:23:12 -0700 Subject: Properly mock the redis pool in MockBot Because some of the redis pool/connection methods return futures rather than being coroutines, the redis pool had to be mocked using the CustomMockMixin so it could take advantage of `additional_spec_asyncs` to use AsyncMocks for these future-returning methods. --- tests/bot/utils/test_redis_cache.py | 12 +++--------- tests/helpers.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index f6344803f..991225481 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -1,11 +1,9 @@ -import asyncio import unittest -from unittest.mock import MagicMock import fakeredis.aioredis -from bot.bot import Bot from bot.utils import RedisCache +from tests import helpers class RedisCacheTests(unittest.IsolatedAsyncioTestCase): @@ -15,12 +13,8 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): # noqa: N802 - this special method can't be all lowercase """Sets up the objects that only have to be initialized once.""" - self.bot = MagicMock( - spec=Bot, - redis_session=await fakeredis.aioredis.create_redis_pool(), - _redis_ready=asyncio.Event(), - ) - self.bot._redis_ready.set() + self.bot = helpers.MockBot() + self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() def test_class_attribute_namespace(self): """Test that RedisDict creates a namespace automatically for class attributes.""" diff --git a/tests/helpers.py b/tests/helpers.py index 2efeff7db..33d4f787c 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -7,6 +7,7 @@ import unittest.mock from asyncio import AbstractEventLoop from typing import Iterable, Optional +import aioredis.abc import discord from discord.ext.commands import Context @@ -265,6 +266,17 @@ class MockAPIClient(CustomMockMixin, unittest.mock.MagicMock): spec_set = APIClient +class MockRedisPool(CustomMockMixin, unittest.mock.MagicMock): + """ + A MagicMock subclass to mock an aioredis connection pool. + + Instances of this class will follow the specifications of `aioredis.abc.AbcPool` instances. + For more information, see the `MockGuild` docstring. + """ + spec_set = aioredis.abc.AbcPool + additional_spec_asyncs = ("execute", "execute_pubsub") + + def _get_mock_loop() -> unittest.mock.Mock: """Return a mocked asyncio.AbstractEventLoop.""" loop = unittest.mock.create_autospec(spec=AbstractEventLoop, spec_set=True) @@ -293,6 +305,10 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): self.loop = _get_mock_loop() self.api_client = MockAPIClient(loop=self.loop) + # fakeredis can't be used cause it'd require awaiting a coroutine to create the pool, + # which cannot be done here in __init__. + self.redis_session = MockRedisPool() + # Create a TextChannel instance to get a realistic MagicMock of `discord.TextChannel` channel_data = { -- cgit v1.2.3 From d8f1634ab68b2cd480d57c8b9da8834866b5c9cc Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 16:25:10 -0700 Subject: Use autospecced mocks in MockBot for the stats and aiohttp session This will help catch anything that tries to get/set an attribute/method which doesn't exist. It'll also catch missing/too many parameters being passed to methods. --- tests/helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/helpers.py b/tests/helpers.py index 33d4f787c..d226be3f0 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -9,9 +9,11 @@ from typing import Iterable, Optional import aioredis.abc import discord +from aiohttp import ClientSession from discord.ext.commands import Context from bot.api import APIClient +from bot.async_stats import AsyncStatsClient from bot.bot import Bot @@ -304,6 +306,8 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): self.loop = _get_mock_loop() self.api_client = MockAPIClient(loop=self.loop) + self.http_session = unittest.mock.create_autospec(spec=ClientSession, spec_set=True) + self.stats = unittest.mock.create_autospec(spec=AsyncStatsClient, spec_set=True) # fakeredis can't be used cause it'd require awaiting a coroutine to create the pool, # which cannot be done here in __init__. -- cgit v1.2.3 From eb63fb02a49bf1979afd04a1350304edf00d3a56 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 02:06:27 +0200 Subject: Finish .set and .get, and add tests. The .set and .get will accept ints, floats, and strings. These will be converted into "typestrings", which is basically just a simple format that's been invented for this object. For example, an int looks like `b"i|2423"`. Note how it is still stored as a bytestring (like everything in Redis), but because of this prefix we are able to coerce it into the type we want on the way out of the db. --- bot/utils/redis_cache.py | 72 ++++++++++++++++++++++++++++++------- tests/bot/utils/test_redis_cache.py | 36 +++++++++++++------ tests/helpers.py | 2 +- 3 files changed, 85 insertions(+), 25 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 483bbc2cd..24f2f2e03 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -1,12 +1,10 @@ from __future__ import annotations -from enum import Enum -from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Union +from typing import Any, AsyncIterator, Dict, Optional, Union from bot.bot import Bot -ValidRedisKey = Union[str, int, float] -JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] +ValidRedisType = Union[str, int, float] class RedisCache: @@ -41,7 +39,39 @@ class RedisCache: self._namespaces.append(namespace) self._namespace = namespace - def __set_name__(self, owner: object, attribute_name: str) -> None: + @staticmethod + def _to_typestring(value: ValidRedisType) -> str: + """Turn a valid Redis type into a typestring.""" + if isinstance(value, float): + return f"f|{value}" + elif isinstance(value, int): + return f"i|{value}" + elif isinstance(value, str): + return f"s|{value}" + + @staticmethod + def _from_typestring(value: str) -> ValidRedisType: + """Turn a valid Redis type into a typestring.""" + if value.startswith("f|"): + return float(value[2:]) + if value.startswith("i|"): + return int(value[2:]) + if value.startswith("s|"): + return value[2:] + + async def _validate_cache(self) -> None: + """Validate that the RedisCache is ready to be used.""" + if self.bot is None: + raise RuntimeError("Critical error: RedisCache has no `Bot` instance.") + + if self._namespace is None: + raise RuntimeError( + "Critical error: RedisCache has no namespace. " + "Did you initialize this object as a class attribute?" + ) + await self.bot._redis_ready.wait() + + def __set_name__(self, owner: Any, attribute_name: str) -> None: """ Set the namespace to Class.attribute_name. @@ -54,8 +84,11 @@ class RedisCache: if self.bot: return self + if self._namespace is None: + raise RuntimeError("RedisCache must be a class attribute.") + if instance is None: - raise NotImplementedError("You must create an instance of RedisCache to use it.") + raise RuntimeError("You must create an instance of RedisCache to use it.") for attribute in vars(instance).values(): if isinstance(attribute, Bot): @@ -69,19 +102,32 @@ class RedisCache: """Return a beautiful representation of this object instance.""" return f"RedisCache(namespace={self._namespace!r})" - async def set(self, key: ValidRedisKey, value: JSONSerializableType) -> None: + async def set(self, key: ValidRedisType, value: ValidRedisType) -> None: """Store an item in the Redis cache.""" - # await self._redis.hset(self._namespace, key, value) + await self._validate_cache() + + # Convert to a typestring and then set it + key = self._to_typestring(key) + value = self._to_typestring(value) + await self._redis.hset(self._namespace, key, value) - async def get(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + async def get(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: """Get an item from the Redis cache.""" - # value = await self._redis.hget(self._namespace, key) + await self._validate_cache() + key = self._to_typestring(key) + value = await self._redis.hget(self._namespace, key) + + if value is None: + return default + else: + value = self._from_typestring(value.decode("utf-8")) + return value - async def delete(self, key: ValidRedisKey) -> None: + async def delete(self, key: ValidRedisType) -> None: """Delete an item from the Redis cache.""" # await self._redis.hdel(self._namespace, key) - async def contains(self, key: ValidRedisKey) -> bool: + async def contains(self, key: ValidRedisType) -> bool: """Check if a key exists in the Redis cache.""" # return await self._redis.hexists(self._namespace, key) @@ -103,7 +149,7 @@ class RedisCache: """Deletes the entire hash from the Redis cache.""" # await self._redis.delete(self._namespace) - async def pop(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + async def pop(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: """Get the item, remove it from the cache, and provide a default if not found.""" # value = await self.get(key, default) # await self.delete(key) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 991225481..ad38bfde0 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -7,27 +7,41 @@ from tests import helpers class RedisCacheTests(unittest.IsolatedAsyncioTestCase): - """Tests the RedisDict class from utils.redis_dict.py.""" + """Tests the RedisCache class from utils.redis_dict.py.""" redis = RedisCache() - async def asyncSetUp(self): # noqa: N802 - this special method can't be all lowercase + async def asyncSetUp(self): # noqa: N802 """Sets up the objects that only have to be initialized once.""" self.bot = helpers.MockBot() self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() - def test_class_attribute_namespace(self): + async def test_class_attribute_namespace(self): """Test that RedisDict creates a namespace automatically for class attributes.""" self.assertEqual(self.redis._namespace, "RedisCacheTests.redis") - # Test that errors are raised when this isn't true. - # def test_set_get_item(self): - # """Test that users can set and get items from the RedisDict.""" - # self.redis['favorite_fruit'] = 'melon' - # self.redis['favorite_number'] = 86 - # self.assertEqual(self.redis['favorite_fruit'], 'melon') - # self.assertEqual(self.redis['favorite_number'], 86) - # + # Test that errors are raised when not assigned as a class attribute + bad_cache = RedisCache() + + with self.assertRaises(RuntimeError): + await bad_cache.set("test", "me_up_deadman") + + async def test_set_get_item(self): + """Test that users can set and get items from the RedisDict.""" + test_cases = ( + ('favorite_fruit', 'melon'), + ('favorite_number', 86), + ('favorite_fraction', 86.54) + ) + + # Test that we can get and set different types. + for test in test_cases: + await self.redis.set(*test) + self.assertEqual(await self.redis.get(test[0]), test[1]) + + # Test that .get allows a default value + self.assertEqual(await self.redis.get('favorite_nothing', "bearclaw"), "bearclaw") + # def test_set_item_types(self): # """Test that setitem rejects keys and values that are not strings, ints or floats.""" # fruits = ["lemon", "melon", "apple"] diff --git a/tests/helpers.py b/tests/helpers.py index d226be3f0..2b176db79 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -299,7 +299,7 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): For more information, see the `MockGuild` docstring. """ spec_set = Bot(command_prefix=unittest.mock.MagicMock(), loop=_get_mock_loop()) - additional_spec_asyncs = ("wait_for",) + additional_spec_asyncs = ("wait_for", "_redis_ready") def __init__(self, **kwargs) -> None: super().__init__(**kwargs) -- cgit v1.2.3 From c8a9a7713c4394556faadb432d1ed3b7ba5c103a Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 11:35:19 +0200 Subject: Finish asyncifying RedisCache methods - All methods will now do a validation check - Complete interface spec added to class: - .update - .clear - .pop - .to_dict - .length - .contains - .delete - .get - .set --- bot/utils/redis_cache.py | 50 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 24f2f2e03..bd14fc239 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -50,8 +50,10 @@ class RedisCache: return f"s|{value}" @staticmethod - def _from_typestring(value: str) -> ValidRedisType: - """Turn a valid Redis type into a typestring.""" + def _from_typestring(value: Union[bytes, str]) -> ValidRedisType: + """Turn a typestring into a valid Redis type.""" + if isinstance(value, bytes): + value = value.decode('utf-8') if value.startswith("f|"): return float(value[2:]) if value.startswith("i|"): @@ -59,6 +61,14 @@ class RedisCache: if value.startswith("s|"): return value[2:] + def _dict_from_typestring(self, dictionary: Dict) -> Dict: + """Turns all contents of a dict into valid Redis types.""" + return {self._from_typestring(key): self._from_typestring(value) for key, value in dictionary.items()} + + def _dict_to_typestring(self, dictionary: Dict) -> Dict: + """Turns all contents of a dict into typestrings.""" + return {self._to_typestring(key): self._to_typestring(value) for key, value in dictionary.items()} + async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" if self.bot is None: @@ -120,41 +130,49 @@ class RedisCache: if value is None: return default else: - value = self._from_typestring(value.decode("utf-8")) + value = self._from_typestring(value) return value async def delete(self, key: ValidRedisType) -> None: """Delete an item from the Redis cache.""" - # await self._redis.hdel(self._namespace, key) + await self._validate_cache() + key = self._to_typestring(key) + return await self._redis.hdel(self._namespace, key) async def contains(self, key: ValidRedisType) -> bool: """Check if a key exists in the Redis cache.""" - # return await self._redis.hexists(self._namespace, key) + await self._validate_cache() + key = self._to_typestring(key) + return await self._redis.hexists(self._namespace, key) async def items(self) -> AsyncIterator: """Iterate all the items in the Redis cache.""" - # data = await redis.hgetall(self.get_with_namespace(key)) - # for item in data: - # yield item + await self._validate_cache() + data = await self._redis.hgetall(self._namespace) # Get all the keys + for key, value in self._dict_from_typestring(data).items(): + yield key, value async def length(self) -> int: """Return the number of items in the Redis cache.""" - # return await self._redis.hlen(self._namespace) + await self._validate_cache() + return await self._redis.hlen(self._namespace) async def to_dict(self) -> Dict: """Convert to dict and return.""" - # return dict(self.items()) + return {key: value async for key, value in self.items()} async def clear(self) -> None: """Deletes the entire hash from the Redis cache.""" - # await self._redis.delete(self._namespace) + await self._validate_cache() + await self._redis.delete(self._namespace) async def pop(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: """Get the item, remove it from the cache, and provide a default if not found.""" - # value = await self.get(key, default) - # await self.delete(key) - # return value + value = await self.get(key, default) + await self.delete(key) + return value - async def update(self) -> None: + async def update(self, items: Dict) -> None: """Update the Redis cache with multiple values.""" - # https://aioredis.readthedocs.io/en/v1.3.0/mixins.html#aioredis.commands.HashCommandsMixin.hmset_dict + await self._validate_cache() + await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) -- cgit v1.2.3 From 387bf5c6b6a21e25c4fc690fb992b6b3e4c165a6 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 11:36:12 +0200 Subject: Complete asyncified test suite for RedisCache This commit just alters existing code to work with the new interface, and with async. All tests are passing successfully. --- tests/bot/utils/test_redis_cache.py | 206 ++++++++++++++++++++---------------- 1 file changed, 112 insertions(+), 94 deletions(-) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index ad38bfde0..d257e91d9 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -16,16 +16,24 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): self.bot = helpers.MockBot() self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() - async def test_class_attribute_namespace(self): + def test_class_attribute_namespace(self): """Test that RedisDict creates a namespace automatically for class attributes.""" self.assertEqual(self.redis._namespace, "RedisCacheTests.redis") - # Test that errors are raised when not assigned as a class attribute + async def test_class_attribute_required(self): + """Test that errors are raised when not assigned as a class attribute.""" bad_cache = RedisCache() + self.assertIs(bad_cache._namespace, None) with self.assertRaises(RuntimeError): await bad_cache.set("test", "me_up_deadman") + def test_namespace_collision(self): + """Test that we prevent colliding namespaces.""" + bad_cache = RedisCache() + bad_cache._set_namespace("RedisCacheTests.redis") + self.assertEqual(bad_cache._namespace, "RedisCacheTests.redis_") + async def test_set_get_item(self): """Test that users can set and get items from the RedisDict.""" test_cases = ( @@ -42,95 +50,105 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): # Test that .get allows a default value self.assertEqual(await self.redis.get('favorite_nothing', "bearclaw"), "bearclaw") - # def test_set_item_types(self): - # """Test that setitem rejects keys and values that are not strings, ints or floats.""" - # fruits = ["lemon", "melon", "apple"] - # - # with self.assertRaises(DataError): - # self.redis[fruits] = "nice" - # - # def test_contains(self): - # """Test that we can reliably use the `in` operator with our RedisDict.""" - # self.redis['favorite_country'] = "Burkina Faso" - # - # self.assertIn('favorite_country', self.redis) - # self.assertNotIn('favorite_dentist', self.redis) - # - # def test_items(self): - # """Test that the RedisDict can be iterated.""" - # self.redis.clear() - # test_cases = ( - # ('favorite_turtle', 'Donatello'), - # ('second_favorite_turtle', 'Leonardo'), - # ('third_favorite_turtle', 'Raphael'), - # ) - # for key, value in test_cases: - # self.redis[key] = value - # - # # Test regular iteration - # for test_case, key in zip(test_cases, self.redis): - # value = test_case[1] - # self.assertEqual(self.redis[key], value) - # - # # Test .items iteration - # for key, value in self.redis.items(): - # self.assertEqual(self.redis[key], value) - # - # # Test .keys iteration - # for test_case, key in zip(test_cases, self.redis.keys()): - # value = test_case[1] - # self.assertEqual(self.redis[key], value) - # - # def test_length(self): - # """Test that we can get the correct len() from the RedisDict.""" - # self.redis.clear() - # self.redis['one'] = 1 - # self.redis['two'] = 2 - # self.redis['three'] = 3 - # self.assertEqual(len(self.redis), 3) - # - # self.redis['four'] = 4 - # self.assertEqual(len(self.redis), 4) - # - # def test_to_dict(self): - # """Test that the .copy method returns a workable dictionary copy.""" - # copy = self.redis.copy() - # local_copy = dict(self.redis.items()) - # self.assertIs(type(copy), dict) - # self.assertEqual(copy, local_copy) - # - # def test_clear(self): - # """Test that the .clear method removes the entire hash.""" - # self.redis.clear() - # self.redis['teddy'] = "with me" - # self.redis['in my dreams'] = "you have a weird hat" - # self.assertEqual(len(self.redis), 2) - # - # self.redis.clear() - # self.assertEqual(len(self.redis), 0) - # - # def test_pop(self): - # """Test that we can .pop an item from the RedisDict.""" - # self.redis.clear() - # self.redis['john'] = 'was afraid' - # - # self.assertEqual(self.redis.pop('john'), 'was afraid') - # self.assertEqual(self.redis.pop('pete', 'breakneck'), 'breakneck') - # self.assertEqual(len(self.redis), 0) - # - # def test_update(self): - # """Test that we can .update the RedisDict with multiple items.""" - # self.redis.clear() - # self.redis["reckfried"] = "lona" - # self.redis["bel air"] = "prince" - # self.redis.update({ - # "reckfried": "jona", - # "mega": "hungry, though", - # }) - # - # result = { - # "reckfried": "jona", - # "bel air": "prince", - # "mega": "hungry, though", - # } - # self.assertEqual(self.redis.copy(), result) + async def test_set_item_type(self): + """Test that .set rejects keys and values that are not strings, ints or floats.""" + fruits = ["lemon", "melon", "apple"] + + with self.assertRaises(TypeError): + await self.redis.set(fruits, "nice") + + async def test_delete_item(self): + """Test that .delete allows us to delete stuff from the RedisCache.""" + # Add an item and verify that it gets added + await self.redis.set("internet", "firetruck") + self.assertEqual(await self.redis.get("internet"), "firetruck") + + # Delete that item and verify that it gets deleted + await self.redis.delete("internet") + self.assertIs(await self.redis.get("internet"), None) + + async def test_contains(self): + """Test that we can check membership with .contains.""" + await self.redis.set('favorite_country', "Burkina Faso") + + self.assertIs(await self.redis.contains('favorite_country'), True) + self.assertIs(await self.redis.contains('favorite_dentist'), False) + + async def test_items(self): + """Test that the RedisDict can be iterated.""" + await self.redis.clear() + + # Set up our test cases in the Redis cache + test_cases = [ + ('favorite_turtle', 'Donatello'), + ('second_favorite_turtle', 'Leonardo'), + ('third_favorite_turtle', 'Raphael'), + ] + for key, value in test_cases: + await self.redis.set(key, value) + + # Consume the AsyncIterator into a regular list, easier to compare that way. + redis_items = [item async for item in self.redis.items()] + + # These sequences are probably in the same order now, but probably + # isn't good enough for tests. Let's not rely on .hgetall always + # returning things in sequence, and just sort both lists to be safe. + redis_items = sorted(redis_items) + test_cases = sorted(test_cases) + + # If these are equal now, everything works fine. + self.assertSequenceEqual(test_cases, redis_items) + + async def test_length(self): + """Test that we can get the correct .length from the RedisDict.""" + await self.redis.clear() + await self.redis.set('one', 1) + await self.redis.set('two', 2) + await self.redis.set('three', 3) + self.assertEqual(await self.redis.length(), 3) + + await self.redis.set('four', 4) + self.assertEqual(await self.redis.length(), 4) + + async def test_to_dict(self): + """Test that the .copy method returns a workable dictionary copy.""" + copy = await self.redis.to_dict() + local_copy = {key: value async for key, value in self.redis.items()} + self.assertIs(type(copy), dict) + self.assertDictEqual(copy, local_copy) + + async def test_clear(self): + """Test that the .clear method removes the entire hash.""" + await self.redis.clear() + await self.redis.set('teddy', 'with me') + await self.redis.set('in my dreams', 'you have a weird hat') + self.assertEqual(await self.redis.length(), 2) + + await self.redis.clear() + self.assertEqual(await self.redis.length(), 0) + + async def test_pop(self): + """Test that we can .pop an item from the RedisDict.""" + await self.redis.clear() + await self.redis.set('john', 'was afraid') + + self.assertEqual(await self.redis.pop('john'), 'was afraid') + self.assertEqual(await self.redis.pop('pete', 'breakneck'), 'breakneck') + self.assertEqual(await self.redis.length(), 0) + + async def test_update(self): + """Test that we can .update the RedisDict with multiple items.""" + await self.redis.clear() + await self.redis.set("reckfried", "lona") + await self.redis.set("bel air", "prince") + await self.redis.update({ + "reckfried": "jona", + "mega": "hungry, though", + }) + + result = { + "reckfried": "jona", + "bel air": "prince", + "mega": "hungry, though", + } + self.assertDictEqual(await self.redis.to_dict(), result) -- cgit v1.2.3 From 5bd8e13088822cfa4b189a30a7d745de61984dc7 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 11:45:01 +0200 Subject: Better docstring for RedisCache --- bot/utils/redis_cache.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index bd14fc239..26a100ef0 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -21,7 +21,37 @@ class RedisCache: We implement several convenient methods that are fairly similar to have a dict behaves, and should be familiar to Python users. The biggest difference is that - all the public methods in this class are coroutines. + all the public methods in this class are coroutines, and must be awaited. + + Because of limitations in Redis, this cache will only accept strings, integers and + floats both for keys and values. + + Simple example for how to use this: + + class SomeCog(Cog): + # To initialize a valid RedisCache, just add it as a class attribute here. + # Do not add it to the __init__ method or anywhere else, it MUST be a class + # attribute. Do not pass any parameters. + cache = RedisCache() + + async def my_method(self): + # Now we can store some stuff in the cache just by doing this. + # This data will persist through restarts! + await self.cache.set("key", "value") + + # To get the data, simply do this. + value = await self.cache.get("key") + + # Other methods work more or less like a dictionary. + # Checking if something is in the cache + await self.cache.contains("key") + + # iterating the cache + async for key, value in self.cache.items(): + print(value) + + # We can even iterate in a comprehension! + consumed = [value async for key, value in self.cache.items()] """ _namespaces = [] -- cgit v1.2.3 From db75440a273277111e7140b1c226630b865d154b Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 11:55:10 +0200 Subject: Better docstring for RedisCache.contains --- bot/utils/redis_cache.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 26a100ef0..2b60ae0c3 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -170,7 +170,11 @@ class RedisCache: return await self._redis.hdel(self._namespace, key) async def contains(self, key: ValidRedisType) -> bool: - """Check if a key exists in the Redis cache.""" + """ + Check if a key exists in the Redis cache. + + Return True if the key exists, otherwise False. + """ await self._validate_cache() key = self._to_typestring(key) return await self._redis.hexists(self._namespace, key) -- cgit v1.2.3 From fc1999ea80df2ebc904260ff0e6f56d9b36bc6c5 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 12:17:39 +0200 Subject: Unbreak the error_handler --- bot/cogs/error_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index e635bd46f..77d16c051 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -167,7 +167,7 @@ class ErrorHandler(Cog): self.bot.stats.incr("errors.missing_required_argument") elif isinstance(e, errors.TooManyArguments): await ctx.send("Too many arguments provided.") - await ctx.invoke(*help_command) + await prepared_help_command self.bot.stats.incr("errors.too_many_arguments") elif isinstance(e, errors.BadArgument): await ctx.send(f"Bad argument: {e}\n") -- cgit v1.2.3 From 49492ffd5e5c87d347048dc370085be12215ed7f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 14:12:13 +0200 Subject: Moving the Redis session creation to Bot._recreate --- bot/bot.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bot/bot.py b/bot/bot.py index 8a3805989..224f5f4e4 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -45,7 +45,6 @@ class Bot(commands.Bot): # will effectively disable stats. statsd_url = "127.0.0.1" - self.loop.create_task(self._create_redis_session()) self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") async def _create_redis_session(self) -> None: @@ -91,6 +90,7 @@ class Bot(commands.Bot): self.stats._transport.close() if self.redis_session: + self._redis_ready.clear() self.redis_session.close() await self.redis_session.wait_closed() @@ -101,7 +101,7 @@ class Bot(commands.Bot): await super().login(*args, **kwargs) def _recreate(self) -> None: - """Re-create the connector, aiohttp session, and the APIClient.""" + """Re-create the connector, aiohttp session, the APIClient and the Redis session.""" # Use asyncio for DNS resolution instead of threads so threads aren't spammed. # Doesn't seem to have any state with regards to being closed, so no need to worry? self._resolver = aiohttp.AsyncResolver() @@ -112,6 +112,9 @@ class Bot(commands.Bot): "The previous connector was not closed; it will remain open and be overwritten" ) + # Create the redis session + self.loop.create_task(self._create_redis_session()) + # Use AF_INET as its socket family to prevent HTTPS related problems both locally # and in production. self._connector = aiohttp.TCPConnector( -- cgit v1.2.3 From 489f9405a77eb88baa0d77a88fd04bf13cbbde1f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 14:21:39 +0200 Subject: CI needs REDIS_PASSWORD to pass tests --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d56675029..4500cb6e8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,6 +22,7 @@ jobs: REDDIT_CLIENT_ID: spam REDDIT_SECRET: ham WOLFRAM_API_KEY: baz + REDIS_PASSWORD: '' steps: - task: UsePythonVersion@0 -- cgit v1.2.3 From 1ea471865fa68a001e25980d50e71333752add6d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 14:27:11 +0200 Subject: Update exception message This was incorrectly suggesting the user needed to create an instance of RedisCache, when in fact it is the parent that needs to be instantiated. Co-authored-by: Sebastiaan Zeeff <33516116+SebastiaanZ@users.noreply.github.com> --- bot/utils/redis_cache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 2b60ae0c3..f9d9e571f 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -128,7 +128,10 @@ class RedisCache: raise RuntimeError("RedisCache must be a class attribute.") if instance is None: - raise RuntimeError("You must create an instance of RedisCache to use it.") + raise RuntimeError( + "You must access the RedisCache instance through the cog instance " + "before accessing it using the cog's class object." + ) for attribute in vars(instance).values(): if isinstance(attribute, Bot): -- cgit v1.2.3 From aa0bb028ed889d93376981213673053a540e137c Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 14:30:56 +0200 Subject: Fix typo in test_to_dict docstring --- tests/bot/utils/test_redis_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index d257e91d9..2ce57499a 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -111,7 +111,7 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(await self.redis.length(), 4) async def test_to_dict(self): - """Test that the .copy method returns a workable dictionary copy.""" + """Test that the .to_dict method returns a workable dictionary copy.""" copy = await self.redis.to_dict() local_copy = {key: value async for key, value in self.redis.items()} self.assertIs(type(copy), dict) -- cgit v1.2.3 From 2fb86258471626863c2214cabc2529e78c77729a Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 14:41:16 +0200 Subject: Don't rely on HDEL ignoring keys for .pop Previously we would try to .delete keys that did not exist if a default was provided when calling .pop. This is okay to do (because HDEL will just ignore any attempts to delete non-existing calls), but it does add an additional pointless API call to Redis, so I've added some validation as a small optimization. This also adds a few additional lines of documentation as requested by @SebastiaanZ in their review. --- bot/utils/redis_cache.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index f9d9e571f..6831be157 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -35,6 +35,14 @@ class RedisCache: cache = RedisCache() async def my_method(self): + + # Now we're ready to use the RedisCache. + # One thing to note here is that this will not work unless + # we access self.cache through an _instance_ of this class. + # + # For example, attempting to use SomeCog.cache will _not_ work, + # you _must_ instantiate the class first and use that instance. + # # Now we can store some stuff in the cache just by doing this. # This data will persist through restarts! await self.cache.set("key", "value") @@ -167,7 +175,13 @@ class RedisCache: return value async def delete(self, key: ValidRedisType) -> None: - """Delete an item from the Redis cache.""" + """ + Delete an item from the Redis cache. + + If we try to delete a key that does not exist, it will simply be ignored. + + See https://redis.io/commands/hdel for more info on how this works. + """ await self._validate_cache() key = self._to_typestring(key) return await self._redis.hdel(self._namespace, key) @@ -206,7 +220,12 @@ class RedisCache: async def pop(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: """Get the item, remove it from the cache, and provide a default if not found.""" value = await self.get(key, default) - await self.delete(key) + + # No need to try to delete something that doesn't exist, + # that's just a superfluous API call. + if value != default: + await self.delete(key) + return value async def update(self, items: Dict) -> None: -- cgit v1.2.3 From 5120717a47c07812d1631cf0905ff3062e139487 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 15:17:57 +0200 Subject: DRY approach to typestring prefix resolution Thanks to @kwzrd for this idea, basically we're making a constant with the typestring prefixes and iterating that in all our converters. These converter functions will also now raise TypeErrors if we try to convert something that isn't in this constants list. I've also added a new test that tests this functionality. --- bot/utils/redis_cache.py | 54 ++++++++++++++++++++++++++----------- tests/bot/utils/test_redis_cache.py | 21 +++++++++++++++ 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 6831be157..1ec1b9fea 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -4,6 +4,11 @@ from typing import Any, AsyncIterator, Dict, Optional, Union from bot.bot import Bot +TYPESTRING_PREFIXES = ( + ("f|", float), + ("i|", int), + ("s|", str), +) ValidRedisType = Union[str, int, float] @@ -78,26 +83,45 @@ class RedisCache: self._namespace = namespace @staticmethod - def _to_typestring(value: ValidRedisType) -> str: - """Turn a valid Redis type into a typestring.""" - if isinstance(value, float): - return f"f|{value}" - elif isinstance(value, int): - return f"i|{value}" - elif isinstance(value, str): - return f"s|{value}" + def _valid_typestring_types() -> str: + """ + Creates a nice, readable list of valid types for typestrings, useful for error messages. + + This will be dynamically updated if we change the TYPESTRING_PREFIXES constant up top. + """ + valid_types = ", ".join([str(_type).split("'")[1] for _, _type in TYPESTRING_PREFIXES]) + valid_types = ", and ".join(valid_types.rsplit(", ", 1)) + return valid_types @staticmethod - def _from_typestring(value: Union[bytes, str]) -> ValidRedisType: + def _valid_typestring_prefixes() -> str: + """ + Creates a nice, readable list of valid prefixes for typestrings, useful for error messages. + + This will be dynamically updated if we change the TYPESTRING_PREFIXES constant up top. + """ + valid_prefixes = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_PREFIXES]) + valid_prefixes = ", and ".join(valid_prefixes.rsplit(", ", 1)) + return valid_prefixes + + def _to_typestring(self, value: ValidRedisType) -> str: + """Turn a valid Redis type into a typestring.""" + for prefix, _type in TYPESTRING_PREFIXES: + if isinstance(value, _type): + return f"{prefix}{value}" + raise TypeError(f"RedisCache._from_typestring only supports the types {self._valid_typestring_types()}.") + + def _from_typestring(self, value: Union[bytes, str]) -> ValidRedisType: """Turn a typestring into a valid Redis type.""" + # Stuff that comes out of Redis will be bytestrings, so let's decode those. if isinstance(value, bytes): value = value.decode('utf-8') - if value.startswith("f|"): - return float(value[2:]) - if value.startswith("i|"): - return int(value[2:]) - if value.startswith("s|"): - return value[2:] + + # Now we convert our unicode string back into the type it originally was. + for prefix, _type in TYPESTRING_PREFIXES: + if value.startswith(prefix): + return _type(value[2:]) + raise TypeError(f"RedisCache._to_typestring only supports the prefixes {self._valid_typestring_prefixes()}.") def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 2ce57499a..150195726 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -152,3 +152,24 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): "mega": "hungry, though", } self.assertDictEqual(await self.redis.to_dict(), result) + + def test_typestring_conversion(self): + """Test the typestring-related helper functions.""" + conversion_tests = ( + (12, "i|12"), + (12.4, "f|12.4"), + ("cowabunga", "s|cowabunga"), + ) + + # Test conversion to typestring + for _input, expected in conversion_tests: + self.assertEqual(self.redis._to_typestring(_input), expected) + + # Test conversion from typestrings + for _input, expected in conversion_tests: + self.assertEqual(self.redis._from_typestring(expected), _input) + + # Test that exceptions are raised on invalid input + with self.assertRaises(TypeError): + self.redis._to_typestring(["internet"]) + self.redis._from_typestring("o|firedog") -- cgit v1.2.3 From a52a13020f3468c671cb549052a9c8e303ae9d8c Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 23 May 2020 10:27:32 -0700 Subject: Remove redis session mock from MockBot It's not feasible to mock it because all the commands return futures rather than being coroutines, so they cannot automatically be turned into AsyncMocks. Furthermore, no code should ever use the redis session directly besides RedisCache. Since the tests for RedisCache already use fakeredis, there's no use in trying to mock redis in MockBot. --- tests/helpers.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 2b176db79..5ad826156 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -7,7 +7,6 @@ import unittest.mock from asyncio import AbstractEventLoop from typing import Iterable, Optional -import aioredis.abc import discord from aiohttp import ClientSession from discord.ext.commands import Context @@ -268,17 +267,6 @@ class MockAPIClient(CustomMockMixin, unittest.mock.MagicMock): spec_set = APIClient -class MockRedisPool(CustomMockMixin, unittest.mock.MagicMock): - """ - A MagicMock subclass to mock an aioredis connection pool. - - Instances of this class will follow the specifications of `aioredis.abc.AbcPool` instances. - For more information, see the `MockGuild` docstring. - """ - spec_set = aioredis.abc.AbcPool - additional_spec_asyncs = ("execute", "execute_pubsub") - - def _get_mock_loop() -> unittest.mock.Mock: """Return a mocked asyncio.AbstractEventLoop.""" loop = unittest.mock.create_autospec(spec=AbstractEventLoop, spec_set=True) @@ -309,10 +297,6 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): self.http_session = unittest.mock.create_autospec(spec=ClientSession, spec_set=True) self.stats = unittest.mock.create_autospec(spec=AsyncStatsClient, spec_set=True) - # fakeredis can't be used cause it'd require awaiting a coroutine to create the pool, - # which cannot be done here in __init__. - self.redis_session = MockRedisPool() - # Create a TextChannel instance to get a realistic MagicMock of `discord.TextChannel` channel_data = { -- cgit v1.2.3 From 923d03a8040251ae7766b9655a3d0ff9f8413c8b Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 23 May 2020 11:14:13 -0700 Subject: Show a warning when redis pool isn't closed --- bot/bot.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bot/bot.py b/bot/bot.py index 224f5f4e4..bf7f9c9df 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -112,6 +112,11 @@ class Bot(commands.Bot): "The previous connector was not closed; it will remain open and be overwritten" ) + if self.redis_session and not self.redis_session.closed: + log.warning( + "The previous redis pool was not closed; it will remain open and be overwritten" + ) + # Create the redis session self.loop.create_task(self._create_redis_session()) -- cgit v1.2.3 From 2c7ff94c956691dafa35c92dd0baa95a60aafacf Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 23 May 2020 18:02:39 -0700 Subject: Token remover: escape dashes in regex They need to be escaped when they're in a character set. By default, they are interpreted as part of the character range syntax. --- bot/cogs/token_remover.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index fa0647828..f23eba89b 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -34,7 +34,7 @@ TOKEN_EPOCH = 1_293_840_000 # The HMAC isn't parsed further, but it's in the regex to ensure it at least exists in the string. # Each part only matches base64 URL-safe characters. # Padding has never been observed, but the padding character '=' is matched just in case. -TOKEN_RE = re.compile(r"[\w-=]+\.[\w-=]+\.[\w-=]+", re.ASCII) +TOKEN_RE = re.compile(r"[\w\-=]+\.[\w\-=]+\.[\w\-=]+", re.ASCII) class TokenRemover(Cog): -- cgit v1.2.3 From 3f8dce7502e3afb2d119979cfc455efcde7ad9db Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 10:18:39 +0200 Subject: use __name__ for type list Instead of relying on __str__ representation, we'll use the __name__ dunder. Co-authored-by: Mark --- bot/utils/redis_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 1ec1b9fea..fadbca673 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -89,7 +89,7 @@ class RedisCache: This will be dynamically updated if we change the TYPESTRING_PREFIXES constant up top. """ - valid_types = ", ".join([str(_type).split("'")[1] for _, _type in TYPESTRING_PREFIXES]) + valid_types = ", ".join(str(_type.__name__) for _, _type in TYPESTRING_PREFIXES) valid_types = ", and ".join(valid_types.rsplit(", ", 1)) return valid_types -- cgit v1.2.3 From ed12a2fa303b7faebeb773dac096bd2b0b8ec23d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 10:25:17 +0200 Subject: len(prefix) instead of hardcoding 2 Co-authored-by: Mark --- bot/utils/redis_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index fadbca673..13e88e8e1 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -120,7 +120,7 @@ class RedisCache: # Now we convert our unicode string back into the type it originally was. for prefix, _type in TYPESTRING_PREFIXES: if value.startswith(prefix): - return _type(value[2:]) + return _type(value[len(prefix):]) raise TypeError(f"RedisCache._to_typestring only supports the prefixes {self._valid_typestring_prefixes()}.") def _dict_from_typestring(self, dictionary: Dict) -> Dict: -- cgit v1.2.3 From 98d8bb3e841ba52fe036b36492029a9fdeb36518 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 10:53:46 +0200 Subject: Refactor the nice prefix/type strings to constants It's leaner to just move that code out of the class and up to the module level as constants. This commit also renames ValidRedisType to RedisType. --- bot/utils/redis_cache.py | 50 ++++++++++++++++++------------------------------ 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 13e88e8e1..d5563c079 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -4,12 +4,20 @@ from typing import Any, AsyncIterator, Dict, Optional, Union from bot.bot import Bot +RedisType = Union[str, int, float] TYPESTRING_PREFIXES = ( ("f|", float), ("i|", int), ("s|", str), ) -ValidRedisType = Union[str, int, float] + +# Makes a nice list like "float, int, and str" +NICE_TYPE_LIST = ", ".join(str(_type.__name__) for _, _type in TYPESTRING_PREFIXES) +NICE_TYPE_LIST = ", and ".join(NICE_TYPE_LIST.rsplit(", ", 1)) + +# Makes a list like "'f|', 'i|', and 's|'" +NICE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_PREFIXES]) +NICE_PREFIX_LIST = ", and ".join(NICE_PREFIX_LIST.rsplit(", ", 1)) class RedisCache: @@ -83,35 +91,15 @@ class RedisCache: self._namespace = namespace @staticmethod - def _valid_typestring_types() -> str: - """ - Creates a nice, readable list of valid types for typestrings, useful for error messages. - - This will be dynamically updated if we change the TYPESTRING_PREFIXES constant up top. - """ - valid_types = ", ".join(str(_type.__name__) for _, _type in TYPESTRING_PREFIXES) - valid_types = ", and ".join(valid_types.rsplit(", ", 1)) - return valid_types - - @staticmethod - def _valid_typestring_prefixes() -> str: - """ - Creates a nice, readable list of valid prefixes for typestrings, useful for error messages. - - This will be dynamically updated if we change the TYPESTRING_PREFIXES constant up top. - """ - valid_prefixes = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_PREFIXES]) - valid_prefixes = ", and ".join(valid_prefixes.rsplit(", ", 1)) - return valid_prefixes - - def _to_typestring(self, value: ValidRedisType) -> str: + def _to_typestring(value: RedisType) -> str: """Turn a valid Redis type into a typestring.""" for prefix, _type in TYPESTRING_PREFIXES: if isinstance(value, _type): return f"{prefix}{value}" - raise TypeError(f"RedisCache._from_typestring only supports the types {self._valid_typestring_types()}.") + raise TypeError(f"RedisCache._from_typestring only supports the types {NICE_TYPE_LIST}.") - def _from_typestring(self, value: Union[bytes, str]) -> ValidRedisType: + @staticmethod + def _from_typestring(value: Union[bytes, str]) -> RedisType: """Turn a typestring into a valid Redis type.""" # Stuff that comes out of Redis will be bytestrings, so let's decode those. if isinstance(value, bytes): @@ -121,7 +109,7 @@ class RedisCache: for prefix, _type in TYPESTRING_PREFIXES: if value.startswith(prefix): return _type(value[len(prefix):]) - raise TypeError(f"RedisCache._to_typestring only supports the prefixes {self._valid_typestring_prefixes()}.") + raise TypeError(f"RedisCache._to_typestring only supports the prefixes {NICE_PREFIX_LIST}.") def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" @@ -177,7 +165,7 @@ class RedisCache: """Return a beautiful representation of this object instance.""" return f"RedisCache(namespace={self._namespace!r})" - async def set(self, key: ValidRedisType, value: ValidRedisType) -> None: + async def set(self, key: RedisType, value: RedisType) -> None: """Store an item in the Redis cache.""" await self._validate_cache() @@ -186,7 +174,7 @@ class RedisCache: value = self._to_typestring(value) await self._redis.hset(self._namespace, key, value) - async def get(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: + async def get(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: """Get an item from the Redis cache.""" await self._validate_cache() key = self._to_typestring(key) @@ -198,7 +186,7 @@ class RedisCache: value = self._from_typestring(value) return value - async def delete(self, key: ValidRedisType) -> None: + async def delete(self, key: RedisType) -> None: """ Delete an item from the Redis cache. @@ -210,7 +198,7 @@ class RedisCache: key = self._to_typestring(key) return await self._redis.hdel(self._namespace, key) - async def contains(self, key: ValidRedisType) -> bool: + async def contains(self, key: RedisType) -> bool: """ Check if a key exists in the Redis cache. @@ -241,7 +229,7 @@ class RedisCache: await self._validate_cache() await self._redis.delete(self._namespace) - async def pop(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: + async def pop(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: """Get the item, remove it from the cache, and provide a default if not found.""" value = await self.get(key, default) -- cgit v1.2.3 From 1d05a4d409cd0652cec36128114739ede2f529cf Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 10:54:34 +0200 Subject: Improves various docstrings and comments. Thanks to @MarkKoz for suggesting most of these in their code review. --- bot/utils/redis_cache.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index d5563c079..e4dce7526 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -24,14 +24,6 @@ class RedisCache: """ A simplified interface for a Redis connection. - This class must be created as a class attribute in a class. This is because it - uses __set_name__ to create a namespace like MyCog.my_class_attribute which is - used as a hash name when we store stuff in Redis, to prevent collisions. - - The class this object is instantiated in must also contains an attribute with an - instance of Bot. This is because Bot contains our redis_pool, which is how this - class communicates with the Redis server. - We implement several convenient methods that are fairly similar to have a dict behaves, and should be familiar to Python users. The biggest difference is that all the public methods in this class are coroutines, and must be awaited. @@ -39,6 +31,10 @@ class RedisCache: Because of limitations in Redis, this cache will only accept strings, integers and floats both for keys and values. + Please note that this class MUST be created as a class attribute, and that that class + must also contain an attribute with an instance of our Bot. See `__get__` and `__set_name__` + for more information about how this works. + Simple example for how to use this: class SomeCog(Cog): @@ -78,12 +74,18 @@ class RedisCache: _namespaces = [] def __init__(self) -> None: - """Raise a NotImplementedError if `__set_name__` hasn't been run.""" + """Initialize the RedisCache.""" self._namespace = None self.bot = None def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" + # We need a unique namespace, to prevent collisions. This loop + # will try appending underscores to the end of the namespace until + # it finds one that is unique. + # + # For example, if `john` and `john_` are both taken, the namespace will + # be `john__` at the end of this loop. while namespace in self._namespaces: namespace += "_" @@ -136,11 +138,26 @@ class RedisCache: Set the namespace to Class.attribute_name. Called automatically when this class is constructed inside a class as an attribute. + + This class MUST be created as a class attribute in a class, otherwise it will raise + exceptions whenever a method is used. This is because it uses this method to create + a namespace like `MyCog.my_class_attribute` which is used as a hash name when we store + stuff in Redis, to prevent collisions. """ self._set_namespace(f"{owner.__name__}.{attribute_name}") def __get__(self, instance: RedisCache, owner: Any) -> RedisCache: - """Fetch the Bot instance, we need it for the redis pool.""" + """ + This is called if the RedisCache is a class attribute, and is accessed. + + The class this object is instantiated in must contain an attribute with an + instance of Bot. This is because Bot contains our redis_session, which is + the mechanism by which we will communicate with the Redis server. + + Any attempt to use RedisCache in a class that does not have a Bot instance + will fail. It is mostly intended to be used inside of a Cog, although theoretically + it should work in any class that has a Bot instance. + """ if self.bot: return self -- cgit v1.2.3 From f05fefb4a51c6653f7f93805489838259782c376 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 11:26:04 +0200 Subject: Better RuntimeErrors. We provide suggestions for how to solve these problems now. --- bot/utils/redis_cache.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index e4dce7526..558ab33a7 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -124,7 +124,13 @@ class RedisCache: async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" if self.bot is None: - raise RuntimeError("Critical error: RedisCache has no `Bot` instance.") + raise RuntimeError( + "Critical error: RedisCache has no `Bot` instance. " + "This happens when the class RedisCache was created in doesn't " + "have a Bot instance. Please make sure that you're instantiating " + "the RedisCache inside a class that has a Bot instance " + "class attribute." + ) if self._namespace is None: raise RuntimeError( @@ -176,7 +182,13 @@ class RedisCache: self._redis = self.bot.redis_session return self else: - raise RuntimeError("Cannot initialize a RedisCache without a `Bot` instance.") + raise RuntimeError( + "Critical error: RedisCache has no `Bot` instance. " + "This happens when the class RedisCache was created in doesn't " + "have a Bot instance. Please make sure that you're instantiating " + "the RedisCache inside a class that has a Bot instance " + "class attribute." + ) def __repr__(self) -> str: """Return a beautiful representation of this object instance.""" -- cgit v1.2.3 From b2009d5304beba4829b7727ca154bb6a0d1cd50a Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 11:42:33 +0200 Subject: Make .items return ItemsView instead of AsyncIter There really was no compelling reason why this method should return an AsyncIterator or than that `async for items in cache.items()` has nice readability, but there were a few concerns. One is a concern about race conditions raised by @SebastiaanZ, and @MarkKoz raised a concern that it was misleading to have an AsyncIterator that only "pretended" to be lazy. To address these concerns, I've refactored it to return a regular ItemsView instead. I also improved the docstring, and fixed the relevant tests. --- bot/utils/redis_cache.py | 28 +++++++++++++++++++++------- tests/bot/utils/test_redis_cache.py | 4 ++-- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 558ab33a7..fb9a534bd 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, AsyncIterator, Dict, Optional, Union +from typing import Any, Dict, ItemsView, Optional, Union from bot.bot import Bot @@ -237,12 +237,26 @@ class RedisCache: key = self._to_typestring(key) return await self._redis.hexists(self._namespace, key) - async def items(self) -> AsyncIterator: - """Iterate all the items in the Redis cache.""" + async def items(self) -> ItemsView: + """ + Fetch all the key/value pairs in the cache. + + Returns a normal ItemsView, like you would get from dict.items(). + + Keep in mind that these items are just a _copy_ of the data in the + RedisCache - any changes you make to them will not be reflected + into the RedisCache itself. If you want to change these, you need + to make a .set call. + + Example: + items = await my_cache.items() + for key, value in items: + # Iterate like a normal dictionary + """ await self._validate_cache() - data = await self._redis.hgetall(self._namespace) # Get all the keys - for key, value in self._dict_from_typestring(data).items(): - yield key, value + return self._dict_from_typestring( + await self._redis.hgetall(self._namespace) + ).items() async def length(self) -> int: """Return the number of items in the Redis cache.""" @@ -251,7 +265,7 @@ class RedisCache: async def to_dict(self) -> Dict: """Convert to dict and return.""" - return {key: value async for key, value in self.items()} + return {key: value for key, value in await self.items()} async def clear(self) -> None: """Deletes the entire hash from the Redis cache.""" diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 150195726..6e12002ed 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -88,7 +88,7 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): await self.redis.set(key, value) # Consume the AsyncIterator into a regular list, easier to compare that way. - redis_items = [item async for item in self.redis.items()] + redis_items = [item for item in await self.redis.items()] # These sequences are probably in the same order now, but probably # isn't good enough for tests. Let's not rely on .hgetall always @@ -113,7 +113,7 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_to_dict(self): """Test that the .to_dict method returns a workable dictionary copy.""" copy = await self.redis.to_dict() - local_copy = {key: value async for key, value in self.redis.items()} + local_copy = {key: value for key, value in await self.redis.items()} self.assertIs(type(copy), dict) self.assertDictEqual(copy, local_copy) -- cgit v1.2.3 From 01bedcadf762262eef0a2b406faf66cdc16a5c85 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 13:04:41 +0200 Subject: Add .increment and .decrement methods. Sometimes, we just want to store a counter in the cache. In this case, it is convenient to have a single method that will allow us to increment or decrement this counter. These methods allow you to decrement or increment floats and integers by an specified amount. By default, it'll increment or decrement by 1. Since this involves several API requests, we create an asyncio.Lock so that we don't end up with race conditions. --- bot/utils/redis_cache.py | 35 +++++++++++++++++++++++++++++++++++ tests/bot/utils/test_redis_cache.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index fb9a534bd..290fae1a0 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio from typing import Any, Dict, ItemsView, Optional, Union from bot.bot import Bot @@ -77,6 +78,7 @@ class RedisCache: """Initialize the RedisCache.""" self._namespace = None self.bot = None + self.increment_lock = asyncio.Lock() def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" @@ -287,3 +289,36 @@ class RedisCache: """Update the Redis cache with multiple values.""" await self._validate_cache() await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) + + async def increment(self, key: RedisType, amount: Optional[int, float] = 1) -> None: + """ + Increment the value by `amount`. + + This works for both floats and ints, but will raise a TypeError + if you try to do it for any other type of value. + + This also supports negative amounts, although it would provide better + readability to use .decrement() for that. + """ + # Since this has several API calls, we need a lock to prevent race conditions + async with self.increment_lock: + value = await self.get(key) + + # Can't increment a non-existing value + if value is None: + raise RuntimeError("The provided key does not exist!") + + # If it does exist, and it's an int or a float, increment and set it. + if isinstance(value, int) or isinstance(value, float): + value += amount + await self.set(key, value) + else: + raise TypeError("You may only increment or decrement values that are integers or floats.") + + async def decrement(self, key: RedisType, amount: Optional[int, float] = 1) -> None: + """ + Decrement the value by `amount`. + + Basically just does the opposite of .increment. + """ + await self.increment(key, -amount) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 6e12002ed..dbbaef018 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -173,3 +173,37 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): with self.assertRaises(TypeError): self.redis._to_typestring(["internet"]) self.redis._from_typestring("o|firedog") + + async def test_increment_decrement(self): + """Test .increment and .decrement methods.""" + await self.redis.set("entropic", 5) + await self.redis.set("disentropic", 12.5) + + # Test default increment + await self.redis.increment("entropic") + self.assertEqual(await self.redis.get("entropic"), 6) + + # Test default decrement + await self.redis.decrement("entropic") + self.assertEqual(await self.redis.get("entropic"), 5) + + # Test float increment with float + await self.redis.increment("disentropic", 2.0) + self.assertEqual(await self.redis.get("disentropic"), 14.5) + + # Test float increment with int + await self.redis.increment("disentropic", 2) + self.assertEqual(await self.redis.get("disentropic"), 16.5) + + # Test negative increments, because why not. + await self.redis.increment("entropic", -5) + self.assertEqual(await self.redis.get("entropic"), 0) + + # Negative decrements? Sure. + await self.redis.decrement("entropic", -5) + self.assertEqual(await self.redis.get("entropic"), 5) + + # What about if we use a negative float to decrement an int? + # This should convert the type into a float. + await self.redis.decrement("entropic", -2.5) + self.assertEqual(await self.redis.get("entropic"), 7.5) -- cgit v1.2.3 From f80ce10aee4a46ab4fc4a2d249fe182fb812a826 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 13:13:03 +0200 Subject: Rename Bot._redis_ready to Bot.redis_ready It's a public attribute, we're accessing it from RedisCache. --- bot/bot.py | 6 +++--- bot/utils/redis_cache.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/bot.py b/bot/bot.py index bf7f9c9df..0d423201b 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -30,12 +30,12 @@ class Bot(commands.Bot): self.http_session: Optional[aiohttp.ClientSession] = None self.redis_session: Optional[aioredis.Redis] = None + self.redis_ready = asyncio.Event() self.api_client = api.APIClient(loop=self.loop) self._connector = None self._resolver = None self._guild_available = asyncio.Event() - self._redis_ready = asyncio.Event() statsd_url = constants.Stats.statsd_host @@ -53,7 +53,7 @@ class Bot(commands.Bot): address=(constants.Redis.host, constants.Redis.port), password=constants.Redis.password, ) - self._redis_ready.set() + self.redis_ready.set() def add_cog(self, cog: commands.Cog) -> None: """Adds a "cog" to the bot and logs the operation.""" @@ -90,7 +90,7 @@ class Bot(commands.Bot): self.stats._transport.close() if self.redis_session: - self._redis_ready.clear() + self.redis_ready.clear() self.redis_session.close() await self.redis_session.wait_closed() diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 290fae1a0..bd885c22c 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -139,7 +139,7 @@ class RedisCache: "Critical error: RedisCache has no namespace. " "Did you initialize this object as a class attribute?" ) - await self.bot._redis_ready.wait() + await self.bot.redis_ready.wait() def __set_name__(self, owner: Any, attribute_name: str) -> None: """ -- cgit v1.2.3 From 361b740f27a579a085e93ebfdd06df10f386cca1 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 13:44:18 +0200 Subject: Add logging to the RedisCache. Mostly trace and exception logging. --- bot/utils/redis_cache.py | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index bd885c22c..5fc34d464 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -1,10 +1,13 @@ from __future__ import annotations import asyncio +import logging from typing import Any, Dict, ItemsView, Optional, Union from bot.bot import Bot +log = logging.getLogger(__name__) + RedisType = Union[str, int, float] TYPESTRING_PREFIXES = ( ("f|", float), @@ -91,6 +94,7 @@ class RedisCache: while namespace in self._namespaces: namespace += "_" + log.trace(f"RedisCache setting namespace to {self._namespace}") self._namespaces.append(namespace) self._namespace = namespace @@ -126,6 +130,7 @@ class RedisCache: async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" if self.bot is None: + log.exception("Attempt to use RedisCache with no `Bot` instance.") raise RuntimeError( "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " @@ -135,6 +140,7 @@ class RedisCache: ) if self._namespace is None: + log.exception("Attempt to use RedisCache with no namespace.") raise RuntimeError( "Critical error: RedisCache has no namespace. " "Did you initialize this object as a class attribute?" @@ -170,9 +176,14 @@ class RedisCache: return self if self._namespace is None: + log.exception("RedisCache must be a class attribute.") raise RuntimeError("RedisCache must be a class attribute.") if instance is None: + log.exception( + "Attempt to access RedisCache instance through the cog's class object " + "before accessing it through the cog instance." + ) raise RuntimeError( "You must access the RedisCache instance through the cog instance " "before accessing it using the cog's class object." @@ -184,6 +195,7 @@ class RedisCache: self._redis = self.bot.redis_session return self else: + log.exception("Attempt to use RedisCache with no `Bot` instance.") raise RuntimeError( "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " @@ -203,18 +215,24 @@ class RedisCache: # Convert to a typestring and then set it key = self._to_typestring(key) value = self._to_typestring(value) + + log.trace(f"Setting {key} to {value}.") await self._redis.hset(self._namespace, key, value) async def get(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: """Get an item from the Redis cache.""" await self._validate_cache() key = self._to_typestring(key) + + log.trace(f"Attempting to retrieve {key}.") value = await self._redis.hget(self._namespace, key) if value is None: + log.trace(f"Value not found, returning default value {default}") return default else: value = self._from_typestring(value) + log.trace(f"Value found, returning value {value}") return value async def delete(self, key: RedisType) -> None: @@ -227,6 +245,8 @@ class RedisCache: """ await self._validate_cache() key = self._to_typestring(key) + + log.trace(f"Attempting to delete {key}.") return await self._redis.hdel(self._namespace, key) async def contains(self, key: RedisType) -> bool: @@ -237,7 +257,10 @@ class RedisCache: """ await self._validate_cache() key = self._to_typestring(key) - return await self._redis.hexists(self._namespace, key) + exists = await self._redis.hexists(self._namespace, key) + + log.trace(f"Testing if {key} exists in the RedisCache - Result is {exists}") + return exists async def items(self) -> ItemsView: """ @@ -256,14 +279,19 @@ class RedisCache: # Iterate like a normal dictionary """ await self._validate_cache() - return self._dict_from_typestring( + items = self._dict_from_typestring( await self._redis.hgetall(self._namespace) ).items() + log.trace(f"Retrieving all key/value pairs from cache, total of {len(items)} items.") + return items + async def length(self) -> int: """Return the number of items in the Redis cache.""" await self._validate_cache() - return await self._redis.hlen(self._namespace) + number_of_items = await self._redis.hlen(self._namespace) + log.trace(f"Returning length. Result is {number_of_items}.") + return number_of_items async def to_dict(self) -> Dict: """Convert to dict and return.""" @@ -272,15 +300,18 @@ class RedisCache: async def clear(self) -> None: """Deletes the entire hash from the Redis cache.""" await self._validate_cache() + log.trace("Clearing the cache of all key/value pairs.") await self._redis.delete(self._namespace) async def pop(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: """Get the item, remove it from the cache, and provide a default if not found.""" + log.trace(f"Attempting to pop {key}.") value = await self.get(key, default) # No need to try to delete something that doesn't exist, # that's just a superfluous API call. if value != default: + log.trace(f"Key {key} exists, deleting it from the cache.") await self.delete(key) return value @@ -288,6 +319,7 @@ class RedisCache: async def update(self, items: Dict) -> None: """Update the Redis cache with multiple values.""" await self._validate_cache() + log.trace(f"Updating the cache with the following items:\n{items}") await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) async def increment(self, key: RedisType, amount: Optional[int, float] = 1) -> None: @@ -300,12 +332,15 @@ class RedisCache: This also supports negative amounts, although it would provide better readability to use .decrement() for that. """ + log.trace(f"Attempting to increment/decrement the value with the key {key} by {amount}.") + # Since this has several API calls, we need a lock to prevent race conditions async with self.increment_lock: value = await self.get(key) # Can't increment a non-existing value if value is None: + log.exception("Attempt to increment/decrement value for non-existent key.") raise RuntimeError("The provided key does not exist!") # If it does exist, and it's an int or a float, increment and set it. @@ -313,6 +348,7 @@ class RedisCache: value += amount await self.set(key, value) else: + log.exception("Attempt to increment/decrement non-numerical value.") raise TypeError("You may only increment or decrement values that are integers or floats.") async def decrement(self, key: RedisType, amount: Optional[int, float] = 1) -> None: -- cgit v1.2.3 From c5e6e8f796265ee6faebdd3d02c839972cd028a9 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 13:49:20 +0200 Subject: MockBot needs to be aware of redis_ready Forgot to update the additional_spec_asyncs when changing the name of this Bot attribute to be public. --- tests/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers.py b/tests/helpers.py index 5ad826156..13283339b 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -287,7 +287,7 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): For more information, see the `MockGuild` docstring. """ spec_set = Bot(command_prefix=unittest.mock.MagicMock(), loop=_get_mock_loop()) - additional_spec_asyncs = ("wait_for", "_redis_ready") + additional_spec_asyncs = ("wait_for", "redis_ready") def __init__(self, **kwargs) -> None: super().__init__(**kwargs) -- cgit v1.2.3 From 66d273f0b8f1de6850760f4d561db446c027fdfe Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 15:21:35 +0200 Subject: Add an option to use fakeredis in Bot. Without this option, all contributors would need to set up a Redis server in order to run the bot. But with use_fakeredis set to True, this is no longer necessary because it will just set up a fakeredis redis pool instead of trying to contact an actual server. This is more than good enough for most local testing purposes, since data persistence across restarts isn't really relevant for them. This also means we need to move fakeredis into our real dependency list instead of having it as a dev dependency, so there's a minor change for that as well. I also made a small kaizen change to sort all the dependencies in the Pipfile alphabetically. --- Pipfile | 28 +++++++++++++------------- Pipfile.lock | 58 +++++++++++++++++++++++++++--------------------------- bot/bot.py | 26 +++++++++++++++++++----- bot/constants.py | 1 + config-default.yml | 1 + 5 files changed, 66 insertions(+), 48 deletions(-) diff --git a/Pipfile b/Pipfile index cd2f2ad7a..b42ca6d58 100644 --- a/Pipfile +++ b/Pipfile @@ -4,30 +4,30 @@ verify_ssl = true name = "pypi" [packages] -discord.py = "~=1.3.2" +aio-pika = "~=6.1" aiodns = "~=2.0" aiohttp = "~=3.5" -sphinx = "~=2.2" -markdownify = "~=0.4" -lxml = "~=4.4" -pyyaml = "~=5.1" +aioredis = "~=1.3.1" +beautifulsoup4 = "~=4.9" +colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} +coloredlogs = "~=14.0" +deepdiff = "~=4.0" +discord.py = "~=1.3.2" +fakeredis = "~=1.4" +feedparser = "~=5.2" fuzzywuzzy = "~=0.17" -aio-pika = "~=6.1" +lxml = "~=4.4" +markdownify = "~=0.4" +more_itertools = "~=8.2" python-dateutil = "~=2.8" -deepdiff = "~=4.0" +pyyaml = "~=5.1" requests = "~=2.22" -more_itertools = "~=8.2" sentry-sdk = "~=0.14" -coloredlogs = "~=14.0" -colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} +sphinx = "~=2.2" statsd = "~=3.3" -feedparser = "~=5.2" -beautifulsoup4 = "~=4.9" -aioredis = "~=1.3.1" [dev-packages] coverage = "~=5.0" -fakeredis = "~=1.4" flake8 = "~=3.7" flake8-annotations = "~=2.0" flake8-bugbear = "~=20.1" diff --git a/Pipfile.lock b/Pipfile.lock index 1941f6887..0e591710c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c0b3e4d3e2c9ddb6ba28d2c09d521fe90ad4ea3df5c7ea7cd3a8b679fb3f85f9" + "sha256": "0297accc3d614d3da8080b89d56ef7fe489c28a0ada8102df396a604af7ee330" }, "pipfile-spec": 6, "requires": { @@ -196,6 +196,14 @@ ], "version": "==0.16" }, + "fakeredis": { + "hashes": [ + "sha256:4d170886865a91dbc8b7f8cbd4e5d488f4c5f2f25dfae127f001617bbe9e8f97", + "sha256:647b2593d349d9d4e566c8dadb2e4c71ba35be5bdc4f1f7ac2d565a12a965053" + ], + "index": "pypi", + "version": "==1.4.1" + }, "feedparser": { "hashes": [ "sha256:bd030652c2d08532c034c27fcd7c85868e7fa3cb2b17f230a44a6bbc92519bf9", @@ -501,6 +509,13 @@ "index": "pypi", "version": "==5.3.1" }, + "redis": { + "hashes": [ + "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", + "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" + ], + "version": "==3.5.2" + }, "requests": { "hashes": [ "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", @@ -531,6 +546,13 @@ ], "version": "==2.0.0" }, + "sortedcontainers": { + "hashes": [ + "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", + "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" + ], + "version": "==2.1.0" + }, "soupsieve": { "hashes": [ "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", @@ -718,14 +740,6 @@ ], "version": "==0.3.0" }, - "fakeredis": { - "hashes": [ - "sha256:4d170886865a91dbc8b7f8cbd4e5d488f4c5f2f25dfae127f001617bbe9e8f97", - "sha256:647b2593d349d9d4e566c8dadb2e4c71ba35be5bdc4f1f7ac2d565a12a965053" - ], - "index": "pypi", - "version": "==1.4.1" - }, "filelock": { "hashes": [ "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", @@ -735,11 +749,11 @@ }, "flake8": { "hashes": [ - "sha256:6c1193b0c3f853ef763969238f6c81e9e63ace9d024518edc020d5f1d6d93195", - "sha256:ea6623797bf9a52f4c9577d780da0bb17d65f870213f7b5bcc9fca82540c31d5" + "sha256:c69ac1668e434d37a2d2880b3ca9aafd54b3a10a3ac1ab101d22f29e29cf8634", + "sha256:ccaa799ef9893cebe69fdfefed76865aeaefbb94cb8545617b2298786a4de9a5" ], "index": "pypi", - "version": "==3.8.1" + "version": "==3.8.2" }, "flake8-annotations": { "hashes": [ @@ -805,10 +819,10 @@ }, "identify": { "hashes": [ - "sha256:23c18d97bb50e05be1a54917ee45cc61d57cb96aedc06aabb2b02331edf0dbf0", - "sha256:88ed90632023e52a6495749c6732e61e08ec9f4f04e95484a5c37b9caf40283c" + "sha256:0f3c3aac62b51b86fea6ff52fe8ff9e06f57f10411502443809064d23e16f1c2", + "sha256:f9ad3d41f01e98eb066b6e05c5b184fd1e925fadec48eb165b4e01c72a1ef3a7" ], - "version": "==1.4.15" + "version": "==1.4.16" }, "mccabe": { "hashes": [ @@ -877,13 +891,6 @@ "index": "pypi", "version": "==5.3.1" }, - "redis": { - "hashes": [ - "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", - "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" - ], - "version": "==3.5.2" - }, "six": { "hashes": [ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", @@ -898,13 +905,6 @@ ], "version": "==2.0.0" }, - "sortedcontainers": { - "hashes": [ - "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", - "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" - ], - "version": "==2.1.0" - }, "toml": { "hashes": [ "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", diff --git a/bot/bot.py b/bot/bot.py index 0d423201b..f1365d532 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -7,6 +7,7 @@ from typing import Optional import aiohttp import aioredis import discord +import fakeredis.aioredis from discord.ext import commands from sentry_sdk import push_scope @@ -48,11 +49,26 @@ class Bot(commands.Bot): self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") async def _create_redis_session(self) -> None: - """Create the Redis connection pool, and then open the redis event gate.""" - self.redis_session = await aioredis.create_redis_pool( - address=(constants.Redis.host, constants.Redis.port), - password=constants.Redis.password, - ) + """ + Create the Redis connection pool, and then open the redis event gate. + + If constants.Redis.use_fakeredis is True, we'll set up a fake redis pool instead + of attempting to communicate with a real Redis server. This is useful because it + means contributors don't necessarily need to get Redis running locally just + to run the bot. + + The fakeredis cache won't have persistence across restarts, but that + usually won't matter for local bot testing. + """ + if constants.Redis.use_fakeredis: + log.info("Using fakeredis instead of communicating with a real Redis server.") + self.redis_session = await fakeredis.aioredis.create_redis_pool() + else: + self.redis_session = await aioredis.create_redis_pool( + address=(constants.Redis.host, constants.Redis.port), + password=constants.Redis.password, + ) + self.redis_ready.set() def add_cog(self, cog: commands.Cog) -> None: diff --git a/bot/constants.py b/bot/constants.py index 5d854dd7a..75d394b6a 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -208,6 +208,7 @@ class Redis(metaclass=YAMLGetter): host: str port: int password: str + use_fakeredis: bool # If this is True, Bot will use fakeredis.aioredis class Filter(metaclass=YAMLGetter): diff --git a/config-default.yml b/config-default.yml index 5be393463..cee955f20 100644 --- a/config-default.yml +++ b/config-default.yml @@ -7,6 +7,7 @@ bot: host: "redis" port: 6379 password: !ENV "REDIS_PASSWORD" + use_fakeredis: false stats: statsd_host: "graphite" -- cgit v1.2.3 From ad8b1fa455e141074daec5047682e82ed96db1f5 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 19:09:45 +0200 Subject: Improve error and error testing for increment Changed a RuntimeError to a KeyError (thanks @MarkKoz), and also added some tests to ensure that the right errors are raised whenever this method is used incorrectly. --- bot/utils/redis_cache.py | 2 +- tests/bot/utils/test_redis_cache.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 5fc34d464..b91d663f3 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -341,7 +341,7 @@ class RedisCache: # Can't increment a non-existing value if value is None: log.exception("Attempt to increment/decrement value for non-existent key.") - raise RuntimeError("The provided key does not exist!") + raise KeyError("The provided key does not exist!") # If it does exist, and it's an int or a float, increment and set it. if isinstance(value, int) or isinstance(value, float): diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index dbbaef018..7405487ed 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -207,3 +207,11 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): # This should convert the type into a float. await self.redis.decrement("entropic", -2.5) self.assertEqual(await self.redis.get("entropic"), 7.5) + + # Let's test that they raise the right errors + with self.assertRaises(KeyError): + await self.redis.increment("doesn't_exist!") + + await self.redis.set("stringthing", "stringthing") + with self.assertRaises(TypeError): + await self.redis.increment("stringthing") -- cgit v1.2.3 From 185c9e84b5fde13ab21de614564eee94963d05b5 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Sun, 24 May 2020 22:46:48 +0100 Subject: Add discord.gift to URL blacklist, closes #958 --- config-default.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config-default.yml b/config-default.yml index 2e98186f1..9a28be700 100644 --- a/config-default.yml +++ b/config-default.yml @@ -318,6 +318,7 @@ filter: - poweredbysecurity.online - ssteam.site - steamwalletgift.com + - discord.gift word_watchlist: - goo+ks* -- cgit v1.2.3 From 856cecbd2354d4cbdbace5a39b7eb9e3d3bf23c7 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 24 May 2020 19:29:13 -0700 Subject: Add support for Union type annotations for constants Note that `Optional[x]` is just an alias for `Union[None, x]` so this effectively supports `Optional` too. This was especially troublesome because the redis password must be unset/None in order to avoid authentication, but the test would complain that `None` isn't a `str`. Setting to an empty string would pass the test but then make redis authenticate and fail. --- bot/constants.py | 14 +++++++------- tests/bot/test_constants.py | 17 ++++++++++++----- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/bot/constants.py b/bot/constants.py index 75d394b6a..145ae54db 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -15,7 +15,7 @@ import os from collections.abc import Mapping from enum import Enum from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional import yaml @@ -198,7 +198,7 @@ class Bot(metaclass=YAMLGetter): prefix: str token: str - sentry_dsn: str + sentry_dsn: Optional[str] class Redis(metaclass=YAMLGetter): @@ -207,7 +207,7 @@ class Redis(metaclass=YAMLGetter): host: str port: int - password: str + password: Optional[str] use_fakeredis: bool # If this is True, Bot will use fakeredis.aioredis @@ -459,7 +459,7 @@ class Guild(metaclass=YAMLGetter): class Keys(metaclass=YAMLGetter): section = "keys" - site_api: str + site_api: Optional[str] class URLs(metaclass=YAMLGetter): @@ -502,8 +502,8 @@ class Reddit(metaclass=YAMLGetter): section = "reddit" subreddits: list - client_id: str - secret: str + client_id: Optional[str] + secret: Optional[str] class Wolfram(metaclass=YAMLGetter): @@ -511,7 +511,7 @@ class Wolfram(metaclass=YAMLGetter): user_limit_day: int guild_limit_day: int - key: str + key: Optional[str] class AntiSpam(metaclass=YAMLGetter): diff --git a/tests/bot/test_constants.py b/tests/bot/test_constants.py index dae7c066c..db9a9bcb0 100644 --- a/tests/bot/test_constants.py +++ b/tests/bot/test_constants.py @@ -1,4 +1,5 @@ import inspect +import typing import unittest from bot import constants @@ -8,7 +9,7 @@ class ConstantsTests(unittest.TestCase): """Tests for our constants.""" def test_section_configuration_matches_type_specification(self): - """The section annotations should match the actual types of the sections.""" + """"The section annotations should match the actual types of the sections.""" sections = ( cls @@ -19,8 +20,14 @@ class ConstantsTests(unittest.TestCase): for name, annotation in section.__annotations__.items(): with self.subTest(section=section, name=name, annotation=annotation): value = getattr(section, name) + annotation_args = typing.get_args(annotation) - if getattr(annotation, '_name', None) in ('Dict', 'List'): - self.skipTest("Cannot validate containers yet.") - - self.assertIsInstance(value, annotation) + if not annotation_args: + self.assertIsInstance(value, annotation) + else: + origin = typing.get_origin(annotation) + if origin is typing.Union: + is_instance = any(isinstance(value, arg) for arg in annotation_args) + self.assertTrue(is_instance) + else: + self.skipTest(f"Validating type {annotation} is unsupported.") -- cgit v1.2.3 From 0ede719d7beb36f476ac26f948ab940882978476 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 25 May 2020 20:44:35 +0200 Subject: AntiMalware tests - Switched from monkeypatch to unittest.patch --- tests/bot/cogs/test_antimalware.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index fab063201..f219fc1ba 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,5 +1,5 @@ import unittest -from unittest.mock import AsyncMock, Mock +from unittest.mock import AsyncMock, Mock, patch from discord import NotFound @@ -10,6 +10,7 @@ from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole MODULE = "bot.cogs.antimalware" +@patch(f"{MODULE}.AntiMalwareConfig.whitelist", new=[".first", ".second", ".third"]) class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): """Test the AntiMalware cog.""" @@ -18,7 +19,6 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.bot = MockBot() self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() - AntiMalwareConfig.whitelist = [".first", ".second", ".third"] async def test_message_with_allowed_attachment(self): """Messages with allowed extensions should not be deleted""" -- cgit v1.2.3 From 9b9aa9b2adbdcd0e0b8c4f4ad38f112a9566fa2f Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 12:03:09 -0700 Subject: Support validating collection types for constants This is a simple validation that only check the type of the collection. It does not validate the types inside the collection because that has proven to be quite complex. --- tests/bot/test_constants.py | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/tests/bot/test_constants.py b/tests/bot/test_constants.py index db9a9bcb0..2937b6189 100644 --- a/tests/bot/test_constants.py +++ b/tests/bot/test_constants.py @@ -5,6 +5,31 @@ import unittest from bot import constants +def is_annotation_instance(value: typing.Any, annotation: typing.Any) -> bool: + """ + Return True if `value` is an instance of the type represented by `annotation`. + + This doesn't account for things like Unions or checking for homogenous types in collections. + """ + origin = typing.get_origin(annotation) + + # This is done in case a bare e.g. `typing.List` is used. + # In such case, for the assertion to pass, the type needs to be normalised to e.g. `list`. + # `get_origin()` does this normalisation for us. + type_ = annotation if origin is None else origin + + return isinstance(value, type_) + + +def is_any_instance(value: typing.Any, types: typing.Collection) -> bool: + """Return True if `value` is an instance of any type in `types`.""" + for type_ in types: + if is_annotation_instance(value, type_): + return True + + return False + + class ConstantsTests(unittest.TestCase): """Tests for our constants.""" @@ -20,14 +45,13 @@ class ConstantsTests(unittest.TestCase): for name, annotation in section.__annotations__.items(): with self.subTest(section=section, name=name, annotation=annotation): value = getattr(section, name) + origin = typing.get_origin(annotation) annotation_args = typing.get_args(annotation) + failure_msg = f"{value} is not an instance of {annotation}" - if not annotation_args: - self.assertIsInstance(value, annotation) + if origin is typing.Union: + is_instance = is_any_instance(value, annotation_args) + self.assertTrue(is_instance, failure_msg) else: - origin = typing.get_origin(annotation) - if origin is typing.Union: - is_instance = any(isinstance(value, arg) for arg in annotation_args) - self.assertTrue(is_instance) - else: - self.skipTest(f"Validating type {annotation} is unsupported.") + is_instance = is_annotation_instance(value, annotation) + self.assertTrue(is_instance, failure_msg) -- cgit v1.2.3 From 87d42add019e8ef1bad5d9593f6ed5a803e4d153 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 12:04:50 -0700 Subject: Improve output of section name in config validation subtests --- tests/bot/test_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/test_constants.py b/tests/bot/test_constants.py index 2937b6189..f10d6fbe8 100644 --- a/tests/bot/test_constants.py +++ b/tests/bot/test_constants.py @@ -43,7 +43,7 @@ class ConstantsTests(unittest.TestCase): ) for section in sections: for name, annotation in section.__annotations__.items(): - with self.subTest(section=section, name=name, annotation=annotation): + with self.subTest(section=section.__name__, name=name, annotation=annotation): value = getattr(section, name) origin = typing.get_origin(annotation) annotation_args = typing.get_args(annotation) -- cgit v1.2.3 From 8b5c1aabf58eb3c794cc61173bd7500a696a8376 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 12:12:24 -0700 Subject: Expose the redis port to the host Useful for those that run redis with docker-compose but not the bot. The bot on the host won't have access to the Docker network in such case so the port must be exposed. --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 1bcf1008e..9884e35f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,8 @@ services: redis: image: redis:5.0.9 + ports: + - "127.0.0.1:6379:6379" web: image: pythondiscord/site:latest -- cgit v1.2.3 From 6cedfdc0b24ea44b86fca039c9d7335072abede6 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Tue, 26 May 2020 02:21:58 +0100 Subject: [stats] Do not report modmail channels to stats --- bot/cogs/stats.py | 8 +++++++- bot/constants.py | 2 ++ config-default.yml | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py index 9baf222e2..14409ecb0 100644 --- a/bot/cogs/stats.py +++ b/bot/cogs/stats.py @@ -6,7 +6,7 @@ from discord.ext.commands import Cog, Context from discord.ext.tasks import loop from bot.bot import Bot -from bot.constants import Channels, Guild, Stats as StatConf +from bot.constants import Categories, Channels, Guild, Stats as StatConf CHANNEL_NAME_OVERRIDES = { @@ -36,6 +36,12 @@ class Stats(Cog): if message.guild.id != Guild.id: return + if message.channel.category.id == Categories.modmail: + if message.channel.id != Channels.incidents: + # Do not report modmail channels to stats, there are too many + # of them for interesting statistics to be drawn out of this. + return + reformatted_name = message.channel.name.replace('-', '_') if CHANNEL_NAME_OVERRIDES.get(message.channel.id): diff --git a/bot/constants.py b/bot/constants.py index 3003c9d36..39de2ee41 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -365,6 +365,7 @@ class Categories(metaclass=YAMLGetter): help_available: int help_in_use: int help_dormant: int + modmail: int class Channels(metaclass=YAMLGetter): @@ -384,6 +385,7 @@ class Channels(metaclass=YAMLGetter): esoteric: int helpers: int how_to_get_help: int + incidents: int message_log: int meta: int mod_alerts: int diff --git a/config-default.yml b/config-default.yml index 9a28be700..c7d25894c 100644 --- a/config-default.yml +++ b/config-default.yml @@ -118,6 +118,7 @@ guild: help_available: 691405807388196926 help_in_use: 696958401460043776 help_dormant: 691405908919451718 + modmail: 714494672835444826 channels: announcements: 354619224620138496 @@ -164,6 +165,7 @@ guild: mod_spam: &MOD_SPAM 620607373828030464 organisation: &ORGANISATION 551789653284356126 staff_lounge: &STAFF_LOUNGE 464905259261755392 + incidents: 714214212200562749 # Voice admins_voice: &ADMINS_VOICE 500734494840717332 -- cgit v1.2.3 From 161bf818ed0f1690c63f4f54cc9549e298e3e45c Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 19:45:04 -0700 Subject: Token remover: use regex groups and pass the token as a NamedTuple It felt redundant to be splitting the token in two different functions when regex could take care of this from the outset. ' A NamedTuple was created to house the token. This is nicer than passing an re.Match object, because it's clearer which attributes are available. Even if the regex used named groups, it wouldn't be as obvious which group names exist. Without the split, `is_maybe_token` is dwindled down to a redundant function. Therefore, it's been removed. --- bot/cogs/token_remover.py | 47 ++++++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index f23eba89b..e5d0ae838 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -34,7 +34,15 @@ TOKEN_EPOCH = 1_293_840_000 # The HMAC isn't parsed further, but it's in the regex to ensure it at least exists in the string. # Each part only matches base64 URL-safe characters. # Padding has never been observed, but the padding character '=' is matched just in case. -TOKEN_RE = re.compile(r"[\w\-=]+\.[\w\-=]+\.[\w\-=]+", re.ASCII) +TOKEN_RE = re.compile(r"([\w\-=]+)\.([\w\-=]+)\.([\w\-=]+)", re.ASCII) + + +class Token(t.NamedTuple): + """A Discord Bot token.""" + + user_id: str + timestamp: str + hmac: str class TokenRemover(Cog): @@ -68,7 +76,7 @@ class TokenRemover(Cog): """ await self.on_message(after) - async def take_action(self, msg: Message, found_token: str) -> None: + async def take_action(self, msg: Message, found_token: Token) -> None: """Remove the `msg` containing the `found_token` and send a mod log message.""" self.mod_log.ignore(Event.message_delete, msg.id) await self.delete_message(msg) @@ -95,20 +103,19 @@ class TokenRemover(Cog): await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention)) @staticmethod - def format_log_message(msg: Message, found_token: str) -> str: - """Return the log message to send for `found_token` being censored in `msg`.""" - user_id, creation_timestamp, hmac = found_token.split('.') + def format_log_message(msg: Message, token: Token) -> str: + """Return the log message to send for `token` being censored in `msg`.""" return LOG_MESSAGE.format( author=msg.author, author_id=msg.author.id, channel=msg.channel.mention, - user_id=user_id, - timestamp=creation_timestamp, - hmac='x' * len(hmac), + user_id=token.user_id, + timestamp=token.timestamp, + hmac='x' * len(token.hmac), ) @classmethod - def find_token_in_message(cls, msg: Message) -> t.Optional[str]: + def find_token_in_message(cls, msg: Message) -> t.Optional[Token]: """Return a seemingly valid token found in `msg` or `None` if no token is found.""" if msg.author.bot: return @@ -116,29 +123,15 @@ class TokenRemover(Cog): # Use findall rather than search to guard against method calls prematurely returning the # token check (e.g. `message.channel.send` also matches our token pattern) maybe_matches = TOKEN_RE.findall(msg.content) - for substr in maybe_matches: - if cls.is_maybe_token(substr): + for match_groups in maybe_matches: + token = Token(*match_groups) + if cls.is_valid_user_id(token.user_id) and cls.is_valid_timestamp(token.timestamp): # Short-circuit on first match - return substr + return token # No matching substring return - @classmethod - def is_maybe_token(cls, test_str: str) -> bool: - """Check the provided string to see if it is a seemingly valid token.""" - try: - user_id, creation_timestamp, hmac = test_str.split('.') - except ValueError: - log.debug(f"Invalid token format in '{test_str}': does not have all 3 parts.") - return False - - if cls.is_valid_user_id(user_id) and cls.is_valid_timestamp(creation_timestamp): - return True - else: - log.debug(f"Invalid user ID or timestamp in '{test_str}'.") - return False - @staticmethod def is_valid_user_id(b64_content: str) -> bool: """ -- cgit v1.2.3 From bfe79efdfe699bf7289cba9db95d5637a7fb965a Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 19:46:51 -0700 Subject: Token remover: use finditer instead of findall It makes more sense to use the lazy function when the loop is already short-circuiting on the first valid token it finds. --- bot/cogs/token_remover.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index e5d0ae838..8913ca64d 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -120,11 +120,10 @@ class TokenRemover(Cog): if msg.author.bot: return - # Use findall rather than search to guard against method calls prematurely returning the + # Use finditer rather than search to guard against method calls prematurely returning the # token check (e.g. `message.channel.send` also matches our token pattern) - maybe_matches = TOKEN_RE.findall(msg.content) - for match_groups in maybe_matches: - token = Token(*match_groups) + for match in TOKEN_RE.finditer(msg.content): + token = Token(*match.groups()) if cls.is_valid_user_id(token.user_id) and cls.is_valid_timestamp(token.timestamp): # Short-circuit on first match return token -- cgit v1.2.3 From 5386eda1731bb8eae287c20ed70a76399db2ae0e Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 19:55:23 -0700 Subject: Token remover: specify Discord epoch in seconds The timestamp in the token is in seconds and is being compared against the epoch. To make life easier, they should use the same unit. Previously, the epoch was in milliseconds. --- bot/cogs/token_remover.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index 8913ca64d..46329e207 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -27,7 +27,7 @@ DELETION_MESSAGE_TEMPLATE = ( "Feel free to re-post it with the token removed. " "If you believe this was a mistake, please let us know!" ) -DISCORD_EPOCH = 1_420_070_400_000 +DISCORD_EPOCH = 1_420_070_400 TOKEN_EPOCH = 1_293_840_000 # Three parts delimited by dots: user ID, creation timestamp, HMAC. -- cgit v1.2.3 From 47886501fb7d030f1cb91c69413058e3ffcb76bf Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 20:47:32 -0700 Subject: Test token regex won't match non-base64 characters --- tests/bot/cogs/test_token_remover.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 8e743a715..dbea5ad1b 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -144,10 +144,9 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): "x..z", " . . ", "\n.\n.\n", - "'.'.'", - '"."."', - "(.(.(", - ").).)" + "hellö.world.bye", + "base64.nötbåse64.morebase64", + "19jd3J.dfkm3d.€víł§tüff", ) for token in tokens: -- cgit v1.2.3 From e76099d48b9a895c48e58c5f5489886f4191eeb6 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 20:50:30 -0700 Subject: Add more valid tokens to test the regex with --- tests/bot/cogs/test_token_remover.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index dbea5ad1b..6a280f358 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -156,10 +156,12 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): def test_regex_valid_tokens(self): """Messages that look like tokens should be matched.""" - # Don't worry, the token's been invalidated. + # Don't worry, these tokens have been invalidated. tokens = ( - "x1.y2.z_3", - "NDcyMjY1OTQzMDYyNDEzMzMy.Xrim9Q.Ysnu2wacjaKs7qnoo46S8Dm2us8" + "NDcyMjY1OTQzMDYy_DEzMz-y.XsyRkw.VXmErH7j511turNpfURmb0rVNm8", + "NDcyMjY1OTQzMDYyNDEzMzMy.Xrim9Q.Ysnu2wacjaKs7qnoo46S8Dm2us8", + "NDc1MDczNjI5Mzk5NTQ3OTA0.XsyR-w.sJf6omBPORBPju3WJEIAcwW9Zds", + "NDY3MjIzMjMwNjUwNzc3NjQx.XsySD_.s45jqDV_Iisn-symw0yDRrk_jf4", ) for token in tokens: -- cgit v1.2.3 From a8a216d0803b67a330ae092a17bea563f5012275 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 21:02:24 -0700 Subject: Fix valid token regex test It was broken due to the addition of groups. Rather than returning the full match, `findall` returns groups if any exist. The test was comparing a tuple of groups to the token string, which was of course failing. Now `fullmatch` is used cause it's simpler - just check for `None` and don't worry about iterating matches to search. --- tests/bot/cogs/test_token_remover.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 6a280f358..518bf91ca 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -166,8 +166,8 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): for token in tokens: with self.subTest(token=token): - results = token_remover.TOKEN_RE.findall(token) - self.assertIn(token, results) + results = token_remover.TOKEN_RE.fullmatch(token) + self.assertIsNotNone(results, f"{token} was not matched by the regex") def test_regex_matches_multiple_valid(self): """Should support multiple matches in the middle of a string.""" -- cgit v1.2.3 From 19cc849d4c70bc3e792460ad712aa308fa500462 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 21:07:21 -0700 Subject: Fix multiple match text for token regex It has to account for the addition of groups. It's easiest to compare the entire string so `finditer` is used to return re.Match objects; the tuples of `findall` would be cumbersome. Also threw in a change to use `assertCountEqual` cause the order doesn't really matter. --- tests/bot/cogs/test_token_remover.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 518bf91ca..2ecfae2bd 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -174,8 +174,9 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): tokens = ["x.y.z", "a.b.c"] message = f"garbage {tokens[0]} hello {tokens[1]} world" - results = token_remover.TOKEN_RE.findall(message) - self.assertEqual(tokens, results) + results = token_remover.TOKEN_RE.finditer(message) + results = [match[0] for match in results] + self.assertCountEqual(tokens, results) @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") def test_is_maybe_token_missing_part_returns_false(self, valid_user, valid_time): -- cgit v1.2.3 From 300f8c093edea03855d94be179c64c328ec842ac Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 21:09:04 -0700 Subject: Use real token values for testing multiple matches in regex --- tests/bot/cogs/test_token_remover.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 2ecfae2bd..971bc93fc 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -171,12 +171,13 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): def test_regex_matches_multiple_valid(self): """Should support multiple matches in the middle of a string.""" - tokens = ["x.y.z", "a.b.c"] - message = f"garbage {tokens[0]} hello {tokens[1]} world" + token_1 = "NDY3MjIzMjMwNjUwNzc3NjQx.XsyWGg.uFNEQPCc4ePwGh7egG8UicQssz8" + token_2 = "NDcyMjY1OTQzMDYyNDEzMzMy.XsyWMw.l8XPnDqb0lp-EiQ2g_0xVFT1pyc" + message = f"garbage {token_1} hello {token_2} world" results = token_remover.TOKEN_RE.finditer(message) results = [match[0] for match in results] - self.assertCountEqual(tokens, results) + self.assertCountEqual((token_1, token_2), results) @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") def test_is_maybe_token_missing_part_returns_false(self, valid_user, valid_time): -- cgit v1.2.3 From 9b882c768344cc866d366dc595fbfc19bc2cb6de Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:38:00 +0200 Subject: Turn log.exception into log.error Also, refactor error messages to be consistent and DRY throughout the file. --- bot/utils/redis_cache.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index b91d663f3..da78f1431 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -130,21 +130,24 @@ class RedisCache: async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" if self.bot is None: - log.exception("Attempt to use RedisCache with no `Bot` instance.") - raise RuntimeError( + error_message = ( "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " "have a Bot instance. Please make sure that you're instantiating " "the RedisCache inside a class that has a Bot instance " "class attribute." ) + log.error(error_message) + raise RuntimeError(error_message) if self._namespace is None: - log.exception("Attempt to use RedisCache with no namespace.") - raise RuntimeError( + error_message = ( "Critical error: RedisCache has no namespace. " "Did you initialize this object as a class attribute?" ) + log.error(error_message) + raise RuntimeError(error_message) + await self.bot.redis_ready.wait() def __set_name__(self, owner: Any, attribute_name: str) -> None: @@ -176,18 +179,17 @@ class RedisCache: return self if self._namespace is None: - log.exception("RedisCache must be a class attribute.") - raise RuntimeError("RedisCache must be a class attribute.") + error_message = "RedisCache must be a class attribute." + log.error(error_message) + raise RuntimeError(error_message) if instance is None: - log.exception( - "Attempt to access RedisCache instance through the cog's class object " - "before accessing it through the cog instance." - ) - raise RuntimeError( + error_message = ( "You must access the RedisCache instance through the cog instance " "before accessing it using the cog's class object." ) + log.error(error_message) + raise RuntimeError(error_message) for attribute in vars(instance).values(): if isinstance(attribute, Bot): @@ -195,14 +197,15 @@ class RedisCache: self._redis = self.bot.redis_session return self else: - log.exception("Attempt to use RedisCache with no `Bot` instance.") - raise RuntimeError( + error_message = ( "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " "have a Bot instance. Please make sure that you're instantiating " "the RedisCache inside a class that has a Bot instance " "class attribute." ) + log.error(error_message) + raise RuntimeError(error_message) def __repr__(self) -> str: """Return a beautiful representation of this object instance.""" @@ -340,16 +343,18 @@ class RedisCache: # Can't increment a non-existing value if value is None: - log.exception("Attempt to increment/decrement value for non-existent key.") - raise KeyError("The provided key does not exist!") + error_message = "The provided key does not exist!" + log.error(error_message) + raise KeyError(error_message) # If it does exist, and it's an int or a float, increment and set it. if isinstance(value, int) or isinstance(value, float): value += amount await self.set(key, value) else: - log.exception("Attempt to increment/decrement non-numerical value.") - raise TypeError("You may only increment or decrement values that are integers or floats.") + error_message = "You may only increment or decrement values that are integers or floats." + log.error(error_message) + raise TypeError(error_message) async def decrement(self, key: RedisType, amount: Optional[int, float] = 1) -> None: """ -- cgit v1.2.3 From 723c1d3337b0a59401f1d3fc50a123f0314a5d3e Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:47:16 +0200 Subject: Fix edge case where pop might not delete. If you passed a key for a value that was the same as your optional, it would just return it but not delete it. This edge case isn't worth it, so I'm just removing that condition and letting the extra API call fly. --- bot/utils/redis_cache.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index da78f1431..dd20b5842 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -222,7 +222,7 @@ class RedisCache: log.trace(f"Setting {key} to {value}.") await self._redis.hset(self._namespace, key, value) - async def get(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: + async def get(self, key: RedisType, default: Optional[RedisType] = None) -> Optional[RedisType]: """Get an item from the Redis cache.""" await self._validate_cache() key = self._to_typestring(key) @@ -311,11 +311,11 @@ class RedisCache: log.trace(f"Attempting to pop {key}.") value = await self.get(key, default) - # No need to try to delete something that doesn't exist, - # that's just a superfluous API call. - if value != default: - log.trace(f"Key {key} exists, deleting it from the cache.") - await self.delete(key) + log.trace( + f"Attempting to delete item with key '{key}' from the cache. " + "If this key doesn't exist, nothing will happen." + ) + await self.delete(key) return value -- cgit v1.2.3 From b630aceb1adb624f45465aeb698e844f9ea340c8 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:51:53 +0200 Subject: Add better docstring for RedisCache.update --- bot/utils/redis_cache.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index dd20b5842..b77ec47a2 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -319,8 +319,18 @@ class RedisCache: return value - async def update(self, items: Dict) -> None: - """Update the Redis cache with multiple values.""" + async def update(self, items: Dict[RedisType, RedisType]) -> None: + """ + Update the Redis cache with multiple values. + + This works exactly like dict.update from a normal dictionary. You pass + a dictionary with one or more key/value pairs into this method. If the keys + do not exist in the RedisCache, they are created. If they do exist, the values + are updated with the new ones from `items`. + + Please note that both the keys and the values in the `items` dictionary + must consist of valid RedisTypes - ints, floats, or strings. + """ await self._validate_cache() log.trace(f"Updating the cache with the following items:\n{items}") await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) -- cgit v1.2.3 From 19206734df651c7d65e5114715db6db9253cb7d6 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:52:52 +0200 Subject: Make self.increment_lock private. --- bot/utils/redis_cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index b77ec47a2..89af225e2 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -81,7 +81,7 @@ class RedisCache: """Initialize the RedisCache.""" self._namespace = None self.bot = None - self.increment_lock = asyncio.Lock() + self._increment_lock = asyncio.Lock() def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" @@ -348,7 +348,7 @@ class RedisCache: log.trace(f"Attempting to increment/decrement the value with the key {key} by {amount}.") # Since this has several API calls, we need a lock to prevent race conditions - async with self.increment_lock: + async with self._increment_lock: value = await self.get(key) # Can't increment a non-existing value -- cgit v1.2.3 From 46a377deef15545d1b860e283d8d0f8291298cee Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:55:12 +0200 Subject: Improve some docstrings for RedisCache. Thanks @MarkKoz! --- bot/utils/redis_cache.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 89af225e2..a1196fcb5 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -134,8 +134,7 @@ class RedisCache: "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " "have a Bot instance. Please make sure that you're instantiating " - "the RedisCache inside a class that has a Bot instance " - "class attribute." + "the RedisCache inside a class that has a Bot instance attribute." ) log.error(error_message) raise RuntimeError(error_message) @@ -143,7 +142,7 @@ class RedisCache: if self._namespace is None: error_message = ( "Critical error: RedisCache has no namespace. " - "Did you initialize this object as a class attribute?" + "This object must be initialized as a class attribute." ) log.error(error_message) raise RuntimeError(error_message) @@ -201,8 +200,7 @@ class RedisCache: "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " "have a Bot instance. Please make sure that you're instantiating " - "the RedisCache inside a class that has a Bot instance " - "class attribute." + "the RedisCache inside a class that has a Bot instance attribute." ) log.error(error_message) raise RuntimeError(error_message) -- cgit v1.2.3 From ec8205cfd7adb5e40aabd52e497e4e387b932211 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:56:30 +0200 Subject: Swap the order for the validate_cache checks. --- bot/utils/redis_cache.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index a1196fcb5..895a12da4 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -129,20 +129,20 @@ class RedisCache: async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" - if self.bot is None: + if self._namespace is None: error_message = ( - "Critical error: RedisCache has no `Bot` instance. " - "This happens when the class RedisCache was created in doesn't " - "have a Bot instance. Please make sure that you're instantiating " - "the RedisCache inside a class that has a Bot instance attribute." + "Critical error: RedisCache has no namespace. " + "This object must be initialized as a class attribute." ) log.error(error_message) raise RuntimeError(error_message) - if self._namespace is None: + if self.bot is None: error_message = ( - "Critical error: RedisCache has no namespace. " - "This object must be initialized as a class attribute." + "Critical error: RedisCache has no `Bot` instance. " + "This happens when the class RedisCache was created in doesn't " + "have a Bot instance. Please make sure that you're instantiating " + "the RedisCache inside a class that has a Bot instance attribute." ) log.error(error_message) raise RuntimeError(error_message) -- cgit v1.2.3 From 1ab34dd48fce2de70db1fb2dd6da06f752460829 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 19:06:57 +0200 Subject: Add a test for RuntimeErrors. This just tests that the various RuntimeErrors are reachable - that includes the error about not having a bot instance, the one about not being a class attribute, and the one about not having instantiated the class. This test addresses a concern raised by @MarkKoz in a review. I've decided not to test that actual contents of these RuntimeErrors, because I believe that sort of testing is a bit too brittle. It shouldn't break a test just to change the content of an error string. --- tests/bot/utils/test_redis_cache.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 7405487ed..1b05ae350 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -215,3 +215,25 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): await self.redis.set("stringthing", "stringthing") with self.assertRaises(TypeError): await self.redis.increment("stringthing") + + async def test_exceptions_raised(self): + """Testing that the various RuntimeErrors are reachable.""" + class MyCog: + cache = RedisCache() + + def __init__(self): + self.other_cache = RedisCache() + + cog = MyCog() + + # Raises "No Bot instance" + with self.assertRaises(RuntimeError): + await cog.cache.get("john") + + # Raises "RedisCache has no namespace" + with self.assertRaises(RuntimeError): + await cog.other_cache.get("was") + + # Raises "You must access the RedisCache instance through the cog instance" + with self.assertRaises(RuntimeError): + await MyCog.cache.get("afraid") -- cgit v1.2.3 From 63a922e629af76692c4af3902a33942b13f784b6 Mon Sep 17 00:00:00 2001 From: Dennis Pham Date: Tue, 26 May 2020 17:11:49 -0400 Subject: Add /r/FlutterDev to the guild invite whitelist --- config-default.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config-default.yml b/config-default.yml index c7d25894c..7edfb131f 100644 --- a/config-default.yml +++ b/config-default.yml @@ -290,6 +290,7 @@ filter: - 81384788765712384 # Discord API - 613425648685547541 # Discord Developers - 185590609631903755 # Blender Hub + - 420324994703163402 # /r/FlutterDev domain_blacklist: - pornhub.com -- cgit v1.2.3 From d9190d997538f49c0a1b53d63a15bada3c89297f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 07:32:16 +0200 Subject: Refactor the in_whitelist deco to a check. We're moving the actual predicate into the `utils.checks` folder, just like we're doing with most of the other decorators. This is to allow us the flexibility to use it as a pure check, not only as a decorator. This commit doesn't actually change any functionality, just moves it around. --- bot/decorators.py | 54 +++-------------------------- bot/utils/checks.py | 81 ++++++++++++++++++++++++++++++++++++++++++-- tests/bot/test_decorators.py | 4 +-- 3 files changed, 86 insertions(+), 53 deletions(-) diff --git a/bot/decorators.py b/bot/decorators.py index 306f0830c..1e77afe60 100644 --- a/bot/decorators.py +++ b/bot/decorators.py @@ -9,37 +9,20 @@ from weakref import WeakValueDictionary from discord import Colour, Embed, Member from discord.errors import NotFound from discord.ext import commands -from discord.ext.commands import CheckFailure, Cog, Context +from discord.ext.commands import Cog, Context from bot.constants import Channels, ERROR_REPLIES, RedirectOutput -from bot.utils.checks import with_role_check, without_role_check +from bot.utils.checks import in_whitelist_check, with_role_check, without_role_check log = logging.getLogger(__name__) -class InWhitelistCheckFailure(CheckFailure): - """Raised when the `in_whitelist` check fails.""" - - def __init__(self, redirect_channel: Optional[int]) -> None: - self.redirect_channel = redirect_channel - - if redirect_channel: - redirect_message = f" here. Please use the <#{redirect_channel}> channel instead" - else: - redirect_message = "" - - error_message = f"You are not allowed to use that command{redirect_message}." - - super().__init__(error_message) - - def in_whitelist( *, channels: Container[int] = (), categories: Container[int] = (), roles: Container[int] = (), redirect: Optional[int] = Channels.bot_commands, - ) -> Callable: """ Check if a command was issued in a whitelisted context. @@ -54,36 +37,9 @@ def in_whitelist( redirected to the `redirect` channel that was passed (default: #bot-commands) or simply told that they're not allowed to use this particular command (if `None` was passed). """ - if redirect and redirect not in channels: - # It does not make sense for the channel whitelist to not contain the redirection - # channel (if applicable). That's why we add the redirection channel to the `channels` - # container if it's not already in it. As we allow any container type to be passed, - # we first create a tuple in order to safely add the redirection channel. - # - # Note: It's possible for the redirect channel to be in a whitelisted category, but - # there's no easy way to check that and as a channel can easily be moved in and out of - # categories, it's probably not wise to rely on its category in any case. - channels = tuple(channels) + (redirect,) - def predicate(ctx: Context) -> bool: - """Check if a command was issued in a whitelisted context.""" - if channels and ctx.channel.id in channels: - log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted channel.") - return True - - # Only check the category id if we have a category whitelist and the channel has a `category_id` - if categories and hasattr(ctx.channel, "category_id") and ctx.channel.category_id in categories: - log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted category.") - return True - - # Only check the roles whitelist if we have one and ensure the author's roles attribute returns - # an iterable to prevent breakage in DM channels (for if we ever decide to enable commands there). - if roles and any(r.id in roles for r in getattr(ctx.author, "roles", ())): - log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they have a whitelisted role.") - return True - - log.trace(f"{ctx.author} may not use the `{ctx.command.name}` command within this context.") - raise InWhitelistCheckFailure(redirect) + """Check if command was issued in a whitelisted context.""" + return in_whitelist_check(ctx, channels, categories, roles, redirect) return commands.check(predicate) @@ -121,7 +77,7 @@ def locked() -> Callable: embed = Embed() embed.colour = Colour.red() - log.debug(f"User tried to invoke a locked command.") + log.debug("User tried to invoke a locked command.") embed.description = ( "You're already using this command. Please wait until it is done before you use it again." ) diff --git a/bot/utils/checks.py b/bot/utils/checks.py index db56c347c..63568b29e 100644 --- a/bot/utils/checks.py +++ b/bot/utils/checks.py @@ -1,12 +1,89 @@ import datetime import logging -from typing import Callable, Iterable +from typing import Callable, Container, Iterable, Optional -from discord.ext.commands import BucketType, Cog, Command, CommandOnCooldown, Context, Cooldown, CooldownMapping +from discord.ext.commands import ( + BucketType, + CheckFailure, + Cog, + Command, + CommandOnCooldown, + Context, + Cooldown, + CooldownMapping, +) + +from bot import constants log = logging.getLogger(__name__) +class InWhitelistCheckFailure(CheckFailure): + """Raised when the `in_whitelist` check fails.""" + + def __init__(self, redirect_channel: Optional[int]) -> None: + self.redirect_channel = redirect_channel + + if redirect_channel: + redirect_message = f" here. Please use the <#{redirect_channel}> channel instead" + else: + redirect_message = "" + + error_message = f"You are not allowed to use that command{redirect_message}." + + super().__init__(error_message) + + +def in_whitelist_check( + ctx: Context, + channels: Container[int] = (), + categories: Container[int] = (), + roles: Container[int] = (), + redirect: Optional[int] = constants.Channels.bot_commands, +) -> bool: + """ + Check if a command was issued in a whitelisted context. + + The whitelists that can be provided are: + + - `channels`: a container with channel ids for whitelisted channels + - `categories`: a container with category ids for whitelisted categories + - `roles`: a container with with role ids for whitelisted roles + + If the command was invoked in a context that was not whitelisted, the member is either + redirected to the `redirect` channel that was passed (default: #bot-commands) or simply + told that they're not allowed to use this particular command (if `None` was passed). + """ + if redirect and redirect not in channels: + # It does not make sense for the channel whitelist to not contain the redirection + # channel (if applicable). That's why we add the redirection channel to the `channels` + # container if it's not already in it. As we allow any container type to be passed, + # we first create a tuple in order to safely add the redirection channel. + # + # Note: It's possible for the redirect channel to be in a whitelisted category, but + # there's no easy way to check that and as a channel can easily be moved in and out of + # categories, it's probably not wise to rely on its category in any case. + channels = tuple(channels) + (redirect,) + + if channels and ctx.channel.id in channels: + log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted channel.") + return True + + # Only check the category id if we have a category whitelist and the channel has a `category_id` + if categories and hasattr(ctx.channel, "category_id") and ctx.channel.category_id in categories: + log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted category.") + return True + + # Only check the roles whitelist if we have one and ensure the author's roles attribute returns + # an iterable to prevent breakage in DM channels (for if we ever decide to enable commands there). + if roles and any(r.id in roles for r in getattr(ctx.author, "roles", ())): + log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they have a whitelisted role.") + return True + + log.trace(f"{ctx.author} may not use the `{ctx.command.name}` command within this context.") + raise InWhitelistCheckFailure(redirect) + + def with_role_check(ctx: Context, *role_ids: int) -> bool: """Returns True if the user has any one of the roles in role_ids.""" if not ctx.guild: # Return False in a DM diff --git a/tests/bot/test_decorators.py b/tests/bot/test_decorators.py index a17dd3e16..3d450caa0 100644 --- a/tests/bot/test_decorators.py +++ b/tests/bot/test_decorators.py @@ -3,10 +3,10 @@ import unittest import unittest.mock from bot import constants -from bot.decorators import InWhitelistCheckFailure, in_whitelist +from bot.decorators import in_whitelist +from bot.utils.checks import InWhitelistCheckFailure from tests import helpers - InWhitelistTestCase = collections.namedtuple("WhitelistedContextTestCase", ("kwargs", "ctx", "description")) -- cgit v1.2.3 From b51c206e51d0906f326da1e504162920cd2d443d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 07:35:56 +0200 Subject: Allow infraction management in modmail category --- bot/cogs/moderation/management.py | 20 ++++++++++++-------- bot/constants.py | 5 +---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index edfdfd9e2..56f7c390c 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -12,7 +12,7 @@ from bot.bot import Bot from bot.converters import Expiry, InfractionSearchQuery, allowed_strings, proxy_user from bot.pagination import LinePaginator from bot.utils import time -from bot.utils.checks import in_channel_check, with_role_check +from bot.utils.checks import in_whitelist_check, with_role_check from . import utils from .infractions import Infractions from .modlog import ModLog @@ -49,8 +49,8 @@ class ModManagement(commands.Cog): async def infraction_edit( self, ctx: Context, - infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], - duration: t.Union[Expiry, allowed_strings("p", "permanent"), None], + infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], # noqa: F821 + duration: t.Union[Expiry, allowed_strings("p", "permanent"), None], # noqa: F821 *, reason: str = None ) -> None: @@ -83,14 +83,14 @@ class ModManagement(commands.Cog): "actor__id": ctx.author.id, "ordering": "-inserted_at" } - infractions = await self.bot.api_client.get(f"bot/infractions", params=params) + infractions = await self.bot.api_client.get("bot/infractions", params=params) if infractions: old_infraction = infractions[0] infraction_id = old_infraction["id"] else: await ctx.send( - f":x: Couldn't find most recent infraction; you have never given an infraction." + ":x: Couldn't find most recent infraction; you have never given an infraction." ) return else: @@ -224,7 +224,7 @@ class ModManagement(commands.Cog): ) -> None: """Send a paginated embed of infractions for the specified user.""" if not infractions: - await ctx.send(f":warning: No infractions could be found for that query.") + await ctx.send(":warning: No infractions could be found for that query.") return lines = tuple( @@ -283,10 +283,14 @@ class ModManagement(commands.Cog): # This cannot be static (must have a __func__ attribute). def cog_check(self, ctx: Context) -> bool: - """Only allow moderators from moderator channels to invoke the commands in this cog.""" + """Only allow moderators inside moderator channels to invoke the commands in this cog.""" checks = [ with_role_check(ctx, *constants.MODERATION_ROLES), - in_channel_check(ctx, *constants.MODERATION_CHANNELS) + in_whitelist_check( + ctx, + channels=constants.MODERATION_CHANNELS, + categories=[constants.Categories.modmail], + ) ] return all(checks) diff --git a/bot/constants.py b/bot/constants.py index 39de2ee41..2ce5355be 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -612,13 +612,10 @@ PROJECT_ROOT = os.path.abspath(os.path.join(BOT_DIR, os.pardir)) MODERATION_ROLES = Guild.moderation_roles STAFF_ROLES = Guild.staff_roles -# Roles combinations +# Channel combinations STAFF_CHANNELS = Guild.staff_channels - -# Default Channel combinations MODERATION_CHANNELS = Guild.moderation_channels - # Bot replies NEGATIVE_REPLIES = [ "Noooooo!!", -- cgit v1.2.3 From d310f42080278b35914bf5785fa322b97627c45f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 07:42:08 +0200 Subject: Find + change all InWhitelistCheckFailure imports --- bot/cogs/error_handler.py | 6 +++--- bot/cogs/information.py | 4 ++-- bot/cogs/verification.py | 4 ++-- tests/bot/cogs/test_information.py | 3 +-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index 23d1eed82..5de961116 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -9,7 +9,7 @@ from bot.api import ResponseCodeError from bot.bot import Bot from bot.constants import Channels from bot.converters import TagNameConverter -from bot.decorators import InWhitelistCheckFailure +from bot.utils.checks import InWhitelistCheckFailure log = logging.getLogger(__name__) @@ -166,7 +166,7 @@ class ErrorHandler(Cog): await prepared_help_command self.bot.stats.incr("errors.missing_required_argument") elif isinstance(e, errors.TooManyArguments): - await ctx.send(f"Too many arguments provided.") + await ctx.send("Too many arguments provided.") await prepared_help_command self.bot.stats.incr("errors.too_many_arguments") elif isinstance(e, errors.BadArgument): @@ -206,7 +206,7 @@ class ErrorHandler(Cog): if isinstance(e, bot_missing_errors): ctx.bot.stats.incr("errors.bot_permission_error") await ctx.send( - f"Sorry, it looks like I don't have the permissions or roles I need to do that." + "Sorry, it looks like I don't have the permissions or roles I need to do that." ) elif isinstance(e, (InWhitelistCheckFailure, errors.NoPrivateMessage)): ctx.bot.stats.incr("errors.wrong_channel_or_dm_error") diff --git a/bot/cogs/information.py b/bot/cogs/information.py index ef2f308ca..f0eb3a1ea 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -12,9 +12,9 @@ from discord.utils import escape_markdown from bot import constants from bot.bot import Bot -from bot.decorators import InWhitelistCheckFailure, in_whitelist, with_role +from bot.decorators import in_whitelist, with_role from bot.pagination import LinePaginator -from bot.utils.checks import cooldown_with_role_bypass, with_role_check +from bot.utils.checks import InWhitelistCheckFailure, cooldown_with_role_bypass, with_role_check from bot.utils.time import time_since log = logging.getLogger(__name__) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 77e8b5706..99be3cdaa 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -9,8 +9,8 @@ from discord.ext.commands import Cog, Context, command from bot import constants from bot.bot import Bot from bot.cogs.moderation import ModLog -from bot.decorators import InWhitelistCheckFailure, in_whitelist, without_role -from bot.utils.checks import without_role_check +from bot.decorators import in_whitelist, without_role +from bot.utils.checks import InWhitelistCheckFailure, without_role_check log = logging.getLogger(__name__) diff --git a/tests/bot/cogs/test_information.py b/tests/bot/cogs/test_information.py index b5f928dd6..aca6b594f 100644 --- a/tests/bot/cogs/test_information.py +++ b/tests/bot/cogs/test_information.py @@ -7,10 +7,9 @@ import discord from bot import constants from bot.cogs import information -from bot.decorators import InWhitelistCheckFailure +from bot.utils.checks import InWhitelistCheckFailure from tests import helpers - COG_PATH = "bot.cogs.information.Information" -- cgit v1.2.3 From 75622622696beee8299c24e9ddbc36f5eb4f104f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 07:46:04 +0200 Subject: No redirect for mod management. --- bot/cogs/moderation/management.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index 56f7c390c..c7c19e89d 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -290,6 +290,7 @@ class ModManagement(commands.Cog): ctx, channels=constants.MODERATION_CHANNELS, categories=[constants.Categories.modmail], + redirect=None, ) ] return all(checks) -- cgit v1.2.3 From c3cbc842dce1c26f09d774b7ca85eff613765480 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 07:51:20 +0200 Subject: Allow some commands to fail checks silently. For example, we don't want the mod commands to produce any kind of error message when run by ordinary users in regular channels - these should have the perception of being invisible and unavailable. --- bot/cogs/moderation/management.py | 1 + bot/decorators.py | 3 ++- bot/utils/checks.py | 7 ++++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index c7c19e89d..c39c7f3bc 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -291,6 +291,7 @@ class ModManagement(commands.Cog): channels=constants.MODERATION_CHANNELS, categories=[constants.Categories.modmail], redirect=None, + fail_silently=True, ) ] return all(checks) diff --git a/bot/decorators.py b/bot/decorators.py index 1e77afe60..500197c89 100644 --- a/bot/decorators.py +++ b/bot/decorators.py @@ -23,6 +23,7 @@ def in_whitelist( categories: Container[int] = (), roles: Container[int] = (), redirect: Optional[int] = Channels.bot_commands, + fail_silently: bool = False, ) -> Callable: """ Check if a command was issued in a whitelisted context. @@ -39,7 +40,7 @@ def in_whitelist( """ def predicate(ctx: Context) -> bool: """Check if command was issued in a whitelisted context.""" - return in_whitelist_check(ctx, channels, categories, roles, redirect) + return in_whitelist_check(ctx, channels, categories, roles, redirect, fail_silently) return commands.check(predicate) diff --git a/bot/utils/checks.py b/bot/utils/checks.py index 63568b29e..d5ebe4ec9 100644 --- a/bot/utils/checks.py +++ b/bot/utils/checks.py @@ -40,6 +40,7 @@ def in_whitelist_check( categories: Container[int] = (), roles: Container[int] = (), redirect: Optional[int] = constants.Channels.bot_commands, + fail_silently: bool = False, ) -> bool: """ Check if a command was issued in a whitelisted context. @@ -81,7 +82,11 @@ def in_whitelist_check( return True log.trace(f"{ctx.author} may not use the `{ctx.command.name}` command within this context.") - raise InWhitelistCheckFailure(redirect) + + # Some commands are secret, and should produce no feedback at all. + if not fail_silently: + raise InWhitelistCheckFailure(redirect) + return False def with_role_check(ctx: Context, *role_ids: int) -> bool: -- cgit v1.2.3 From 72304495f43e91eb62bb47657bc3ce4858639939 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 09:29:14 +0200 Subject: Remove all sending of avatar_hash. This is a companion commit to this PR: https://github.com/python-discord/site/pull/356 This PR must be merged before this commit. --- bot/cogs/moderation/utils.py | 1 - bot/cogs/sync/cog.py | 4 +--- bot/cogs/sync/syncers.py | 3 +-- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py index e4e0f1ec2..c99847329 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/cogs/moderation/utils.py @@ -41,7 +41,6 @@ async def post_user(ctx: Context, user: UserSnowflake) -> t.Optional[dict]: log.debug("The user being added to the DB is not a Member or User object.") payload = { - 'avatar_hash': getattr(user, 'avatar', 0), 'discriminator': int(getattr(user, 'discriminator', 0)), 'id': user.id, 'in_guild': False, diff --git a/bot/cogs/sync/cog.py b/bot/cogs/sync/cog.py index 5708be3f4..7cc3726b2 100644 --- a/bot/cogs/sync/cog.py +++ b/bot/cogs/sync/cog.py @@ -94,7 +94,6 @@ class Sync(Cog): the database, the user is added. """ packed = { - 'avatar_hash': member.avatar, 'discriminator': int(member.discriminator), 'id': member.id, 'in_guild': True, @@ -135,12 +134,11 @@ class Sync(Cog): @Cog.listener() async def on_user_update(self, before: User, after: User) -> None: """Update the user information in the database if a relevant change is detected.""" - attrs = ("name", "discriminator", "avatar") + attrs = ("name", "discriminator") if any(getattr(before, attr) != getattr(after, attr) for attr in attrs): updated_information = { "name": after.name, "discriminator": int(after.discriminator), - "avatar_hash": after.avatar, } await self.patch_user(after.id, updated_information=updated_information) diff --git a/bot/cogs/sync/syncers.py b/bot/cogs/sync/syncers.py index e55bf27fd..536455668 100644 --- a/bot/cogs/sync/syncers.py +++ b/bot/cogs/sync/syncers.py @@ -17,7 +17,7 @@ log = logging.getLogger(__name__) # These objects are declared as namedtuples because tuples are hashable, # something that we make use of when diffing site roles against guild roles. _Role = namedtuple('Role', ('id', 'name', 'colour', 'permissions', 'position')) -_User = namedtuple('User', ('id', 'name', 'discriminator', 'avatar_hash', 'roles', 'in_guild')) +_User = namedtuple('User', ('id', 'name', 'discriminator', 'roles', 'in_guild')) _Diff = namedtuple('Diff', ('created', 'updated', 'deleted')) @@ -298,7 +298,6 @@ class UserSyncer(Syncer): id=member.id, name=member.name, discriminator=int(member.discriminator), - avatar_hash=member.avatar, roles=tuple(sorted(role.id for role in member.roles)), in_guild=True ) -- cgit v1.2.3 From 8e0cdb258ea6e0f25977d18336a2e07b20b5d1ee Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 09:42:57 +0200 Subject: Fix failing tests related to avatar_hash --- tests/bot/cogs/sync/test_cog.py | 3 --- tests/bot/cogs/sync/test_users.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/tests/bot/cogs/sync/test_cog.py b/tests/bot/cogs/sync/test_cog.py index 81398c61f..14fd909c4 100644 --- a/tests/bot/cogs/sync/test_cog.py +++ b/tests/bot/cogs/sync/test_cog.py @@ -247,14 +247,12 @@ class SyncCogListenerTests(SyncCogTestCase): before_data = { "name": "old name", "discriminator": "1234", - "avatar": "old avatar", "bot": False, } subtests = ( (True, "name", "name", "new name", "new name"), (True, "discriminator", "discriminator", "8765", 8765), - (True, "avatar", "avatar_hash", "9j2e9", "9j2e9"), (False, "bot", "bot", True, True), ) @@ -295,7 +293,6 @@ class SyncCogListenerTests(SyncCogTestCase): ) data = { - "avatar_hash": member.avatar, "discriminator": int(member.discriminator), "id": member.id, "in_guild": True, diff --git a/tests/bot/cogs/sync/test_users.py b/tests/bot/cogs/sync/test_users.py index 818883012..002a947ad 100644 --- a/tests/bot/cogs/sync/test_users.py +++ b/tests/bot/cogs/sync/test_users.py @@ -10,7 +10,6 @@ def fake_user(**kwargs): kwargs.setdefault("id", 43) kwargs.setdefault("name", "bob the test man") kwargs.setdefault("discriminator", 1337) - kwargs.setdefault("avatar_hash", None) kwargs.setdefault("roles", (666,)) kwargs.setdefault("in_guild", True) @@ -32,7 +31,6 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): for member in members: member = member.copy() - member["avatar"] = member.pop("avatar_hash") del member["in_guild"] mock_member = helpers.MockMember(**member) -- cgit v1.2.3 From 35a1de37307b1745c061e490be4e96c8467de212 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 12:21:58 +0200 Subject: Clear cache in asyncSetUp instead of tests. --- tests/bot/utils/test_redis_cache.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 1b05ae350..900a6d035 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -15,6 +15,7 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): """Sets up the objects that only have to be initialized once.""" self.bot = helpers.MockBot() self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() + await self.redis.clear() def test_class_attribute_namespace(self): """Test that RedisDict creates a namespace automatically for class attributes.""" @@ -76,8 +77,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_items(self): """Test that the RedisDict can be iterated.""" - await self.redis.clear() - # Set up our test cases in the Redis cache test_cases = [ ('favorite_turtle', 'Donatello'), @@ -101,7 +100,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_length(self): """Test that we can get the correct .length from the RedisDict.""" - await self.redis.clear() await self.redis.set('one', 1) await self.redis.set('two', 2) await self.redis.set('three', 3) @@ -119,7 +117,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_clear(self): """Test that the .clear method removes the entire hash.""" - await self.redis.clear() await self.redis.set('teddy', 'with me') await self.redis.set('in my dreams', 'you have a weird hat') self.assertEqual(await self.redis.length(), 2) @@ -129,7 +126,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_pop(self): """Test that we can .pop an item from the RedisDict.""" - await self.redis.clear() await self.redis.set('john', 'was afraid') self.assertEqual(await self.redis.pop('john'), 'was afraid') @@ -138,7 +134,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_update(self): """Test that we can .update the RedisDict with multiple items.""" - await self.redis.clear() await self.redis.set("reckfried", "lona") await self.redis.set("bel air", "prince") await self.redis.update({ -- cgit v1.2.3 From b18930735e05e09ba615cb54fe1dbdfd41bb0f81 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 12:51:40 +0200 Subject: Refactor .increment and add lock test. The way we were doing the asyncio.Lock() stuff for increment was slightly problematic. @aeros has adviced us that it's better to just initialize the lock as None in __init__, and then initialize it inside the first coroutine that uses it instead. This ensures that the correct loop gets attached to the lock, so we don't end up getting errors like this one: RuntimeError: got Future attached to a different loop This happens because the lock and the actual calling coroutines aren't on the same loop. When creating a new test, test_increment_lock, we discovered that we needed a small refactor here and also in the test class to make this new test pass. So, now we're creating a DummyCog for every test method, and this will ensure the loop streams never cross. Cause we all know we must never cross the streams. --- bot/utils/redis_cache.py | 11 ++- tests/bot/utils/test_redis_cache.py | 163 ++++++++++++++++++++++-------------- 2 files changed, 109 insertions(+), 65 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 895a12da4..33e5d5852 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -81,7 +81,7 @@ class RedisCache: """Initialize the RedisCache.""" self._namespace = None self.bot = None - self._increment_lock = asyncio.Lock() + self._increment_lock = None def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" @@ -345,6 +345,15 @@ class RedisCache: """ log.trace(f"Attempting to increment/decrement the value with the key {key} by {amount}.") + # We initialize the lock here, because we need to ensure we get it + # running on the same loop as the calling coroutine. + # + # If we initialized the lock in the __init__, the loop that the coroutine this method + # would be called from might not exist yet, and so the lock would be on a different + # loop, which would raise RuntimeErrors. + if self._increment_lock is None: + self._increment_lock = asyncio.Lock() + # Since this has several API calls, we need a lock to prevent race conditions async with self._increment_lock: value = await self.get(key) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 900a6d035..efd168dac 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -1,3 +1,4 @@ +import asyncio import unittest import fakeredis.aioredis @@ -9,17 +10,30 @@ from tests import helpers class RedisCacheTests(unittest.IsolatedAsyncioTestCase): """Tests the RedisCache class from utils.redis_dict.py.""" - redis = RedisCache() - async def asyncSetUp(self): # noqa: N802 """Sets up the objects that only have to be initialized once.""" self.bot = helpers.MockBot() self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() - await self.redis.clear() + + # Okay, so this is necessary so that we can create a clean new + # class for every test method, and we want that because it will + # ensure we get a fresh loop, which is necessary for test_increment_lock + # to be able to pass. + class DummyCog: + """A dummy cog, for dummies.""" + + redis = RedisCache() + + def __init__(self, bot: helpers.MockBot): + self.bot = bot + + self.cog = DummyCog(self.bot) + + await self.cog.redis.clear() def test_class_attribute_namespace(self): """Test that RedisDict creates a namespace automatically for class attributes.""" - self.assertEqual(self.redis._namespace, "RedisCacheTests.redis") + self.assertEqual(self.cog.redis._namespace, "DummyCog.redis") async def test_class_attribute_required(self): """Test that errors are raised when not assigned as a class attribute.""" @@ -31,9 +45,13 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): def test_namespace_collision(self): """Test that we prevent colliding namespaces.""" - bad_cache = RedisCache() - bad_cache._set_namespace("RedisCacheTests.redis") - self.assertEqual(bad_cache._namespace, "RedisCacheTests.redis_") + bob_cache_1 = RedisCache() + bob_cache_1._set_namespace("BobRoss") + self.assertEqual(bob_cache_1._namespace, "BobRoss") + + bob_cache_2 = RedisCache() + bob_cache_2._set_namespace("BobRoss") + self.assertEqual(bob_cache_2._namespace, "BobRoss_") async def test_set_get_item(self): """Test that users can set and get items from the RedisDict.""" @@ -45,35 +63,35 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): # Test that we can get and set different types. for test in test_cases: - await self.redis.set(*test) - self.assertEqual(await self.redis.get(test[0]), test[1]) + await self.cog.redis.set(*test) + self.assertEqual(await self.cog.redis.get(test[0]), test[1]) # Test that .get allows a default value - self.assertEqual(await self.redis.get('favorite_nothing', "bearclaw"), "bearclaw") + self.assertEqual(await self.cog.redis.get('favorite_nothing', "bearclaw"), "bearclaw") async def test_set_item_type(self): """Test that .set rejects keys and values that are not strings, ints or floats.""" fruits = ["lemon", "melon", "apple"] with self.assertRaises(TypeError): - await self.redis.set(fruits, "nice") + await self.cog.redis.set(fruits, "nice") async def test_delete_item(self): """Test that .delete allows us to delete stuff from the RedisCache.""" # Add an item and verify that it gets added - await self.redis.set("internet", "firetruck") - self.assertEqual(await self.redis.get("internet"), "firetruck") + await self.cog.redis.set("internet", "firetruck") + self.assertEqual(await self.cog.redis.get("internet"), "firetruck") # Delete that item and verify that it gets deleted - await self.redis.delete("internet") - self.assertIs(await self.redis.get("internet"), None) + await self.cog.redis.delete("internet") + self.assertIs(await self.cog.redis.get("internet"), None) async def test_contains(self): """Test that we can check membership with .contains.""" - await self.redis.set('favorite_country', "Burkina Faso") + await self.cog.redis.set('favorite_country', "Burkina Faso") - self.assertIs(await self.redis.contains('favorite_country'), True) - self.assertIs(await self.redis.contains('favorite_dentist'), False) + self.assertIs(await self.cog.redis.contains('favorite_country'), True) + self.assertIs(await self.cog.redis.contains('favorite_dentist'), False) async def test_items(self): """Test that the RedisDict can be iterated.""" @@ -84,10 +102,10 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): ('third_favorite_turtle', 'Raphael'), ] for key, value in test_cases: - await self.redis.set(key, value) + await self.cog.redis.set(key, value) # Consume the AsyncIterator into a regular list, easier to compare that way. - redis_items = [item for item in await self.redis.items()] + redis_items = [item for item in await self.cog.redis.items()] # These sequences are probably in the same order now, but probably # isn't good enough for tests. Let's not rely on .hgetall always @@ -100,43 +118,43 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_length(self): """Test that we can get the correct .length from the RedisDict.""" - await self.redis.set('one', 1) - await self.redis.set('two', 2) - await self.redis.set('three', 3) - self.assertEqual(await self.redis.length(), 3) + await self.cog.redis.set('one', 1) + await self.cog.redis.set('two', 2) + await self.cog.redis.set('three', 3) + self.assertEqual(await self.cog.redis.length(), 3) - await self.redis.set('four', 4) - self.assertEqual(await self.redis.length(), 4) + await self.cog.redis.set('four', 4) + self.assertEqual(await self.cog.redis.length(), 4) async def test_to_dict(self): """Test that the .to_dict method returns a workable dictionary copy.""" - copy = await self.redis.to_dict() - local_copy = {key: value for key, value in await self.redis.items()} + copy = await self.cog.redis.to_dict() + local_copy = {key: value for key, value in await self.cog.redis.items()} self.assertIs(type(copy), dict) self.assertDictEqual(copy, local_copy) async def test_clear(self): """Test that the .clear method removes the entire hash.""" - await self.redis.set('teddy', 'with me') - await self.redis.set('in my dreams', 'you have a weird hat') - self.assertEqual(await self.redis.length(), 2) + await self.cog.redis.set('teddy', 'with me') + await self.cog.redis.set('in my dreams', 'you have a weird hat') + self.assertEqual(await self.cog.redis.length(), 2) - await self.redis.clear() - self.assertEqual(await self.redis.length(), 0) + await self.cog.redis.clear() + self.assertEqual(await self.cog.redis.length(), 0) async def test_pop(self): """Test that we can .pop an item from the RedisDict.""" - await self.redis.set('john', 'was afraid') + await self.cog.redis.set('john', 'was afraid') - self.assertEqual(await self.redis.pop('john'), 'was afraid') - self.assertEqual(await self.redis.pop('pete', 'breakneck'), 'breakneck') - self.assertEqual(await self.redis.length(), 0) + self.assertEqual(await self.cog.redis.pop('john'), 'was afraid') + self.assertEqual(await self.cog.redis.pop('pete', 'breakneck'), 'breakneck') + self.assertEqual(await self.cog.redis.length(), 0) async def test_update(self): """Test that we can .update the RedisDict with multiple items.""" - await self.redis.set("reckfried", "lona") - await self.redis.set("bel air", "prince") - await self.redis.update({ + await self.cog.redis.set("reckfried", "lona") + await self.cog.redis.set("bel air", "prince") + await self.cog.redis.update({ "reckfried": "jona", "mega": "hungry, though", }) @@ -146,7 +164,7 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): "bel air": "prince", "mega": "hungry, though", } - self.assertDictEqual(await self.redis.to_dict(), result) + self.assertDictEqual(await self.cog.redis.to_dict(), result) def test_typestring_conversion(self): """Test the typestring-related helper functions.""" @@ -158,58 +176,75 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): # Test conversion to typestring for _input, expected in conversion_tests: - self.assertEqual(self.redis._to_typestring(_input), expected) + self.assertEqual(self.cog.redis._to_typestring(_input), expected) # Test conversion from typestrings for _input, expected in conversion_tests: - self.assertEqual(self.redis._from_typestring(expected), _input) + self.assertEqual(self.cog.redis._from_typestring(expected), _input) # Test that exceptions are raised on invalid input with self.assertRaises(TypeError): - self.redis._to_typestring(["internet"]) - self.redis._from_typestring("o|firedog") + self.cog.redis._to_typestring(["internet"]) + self.cog.redis._from_typestring("o|firedog") async def test_increment_decrement(self): """Test .increment and .decrement methods.""" - await self.redis.set("entropic", 5) - await self.redis.set("disentropic", 12.5) + await self.cog.redis.set("entropic", 5) + await self.cog.redis.set("disentropic", 12.5) # Test default increment - await self.redis.increment("entropic") - self.assertEqual(await self.redis.get("entropic"), 6) + await self.cog.redis.increment("entropic") + self.assertEqual(await self.cog.redis.get("entropic"), 6) # Test default decrement - await self.redis.decrement("entropic") - self.assertEqual(await self.redis.get("entropic"), 5) + await self.cog.redis.decrement("entropic") + self.assertEqual(await self.cog.redis.get("entropic"), 5) # Test float increment with float - await self.redis.increment("disentropic", 2.0) - self.assertEqual(await self.redis.get("disentropic"), 14.5) + await self.cog.redis.increment("disentropic", 2.0) + self.assertEqual(await self.cog.redis.get("disentropic"), 14.5) # Test float increment with int - await self.redis.increment("disentropic", 2) - self.assertEqual(await self.redis.get("disentropic"), 16.5) + await self.cog.redis.increment("disentropic", 2) + self.assertEqual(await self.cog.redis.get("disentropic"), 16.5) # Test negative increments, because why not. - await self.redis.increment("entropic", -5) - self.assertEqual(await self.redis.get("entropic"), 0) + await self.cog.redis.increment("entropic", -5) + self.assertEqual(await self.cog.redis.get("entropic"), 0) # Negative decrements? Sure. - await self.redis.decrement("entropic", -5) - self.assertEqual(await self.redis.get("entropic"), 5) + await self.cog.redis.decrement("entropic", -5) + self.assertEqual(await self.cog.redis.get("entropic"), 5) # What about if we use a negative float to decrement an int? # This should convert the type into a float. - await self.redis.decrement("entropic", -2.5) - self.assertEqual(await self.redis.get("entropic"), 7.5) + await self.cog.redis.decrement("entropic", -2.5) + self.assertEqual(await self.cog.redis.get("entropic"), 7.5) # Let's test that they raise the right errors with self.assertRaises(KeyError): - await self.redis.increment("doesn't_exist!") + await self.cog.redis.increment("doesn't_exist!") - await self.redis.set("stringthing", "stringthing") + await self.cog.redis.set("stringthing", "stringthing") with self.assertRaises(TypeError): - await self.redis.increment("stringthing") + await self.cog.redis.increment("stringthing") + + async def test_increment_lock(self): + """Test that we can't produce a race condition in .increment.""" + await self.cog.redis.set("test_key", 0) + tasks = [] + + # Increment this a lot in different tasks + for _ in range(100): + task = asyncio.create_task( + self.cog.redis.increment("test_key", 1) + ) + tasks.append(task) + await asyncio.gather(*tasks) + + # Confirm that the value has been incremented the exact right number of times. + value = await self.cog.redis.get("test_key") + self.assertEqual(value, 100) async def test_exceptions_raised(self): """Testing that the various RuntimeErrors are reachable.""" -- cgit v1.2.3 From db0a384e91a463ff9668ab4f9ea5268aa332ab2d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 13:27:34 +0200 Subject: Remove the now deprecated in_channel_check. This check was no longer being used anywhere, having been replaced by in_whitelist_check. --- bot/utils/checks.py | 8 -------- tests/bot/utils/test_checks.py | 8 -------- 2 files changed, 16 deletions(-) diff --git a/bot/utils/checks.py b/bot/utils/checks.py index d5ebe4ec9..f0ef36302 100644 --- a/bot/utils/checks.py +++ b/bot/utils/checks.py @@ -120,14 +120,6 @@ def without_role_check(ctx: Context, *role_ids: int) -> bool: return check -def in_channel_check(ctx: Context, *channel_ids: int) -> bool: - """Checks if the command was executed inside the list of specified channels.""" - check = ctx.channel.id in channel_ids - log.trace(f"{ctx.author} tried to call the '{ctx.command.name}' command. " - f"The result of the in_channel check was {check}.") - return check - - def cooldown_with_role_bypass(rate: int, per: float, type: BucketType = BucketType.default, *, bypass_roles: Iterable[int]) -> Callable: """ diff --git a/tests/bot/utils/test_checks.py b/tests/bot/utils/test_checks.py index 9610771e5..d572b6299 100644 --- a/tests/bot/utils/test_checks.py +++ b/tests/bot/utils/test_checks.py @@ -41,11 +41,3 @@ class ChecksTests(unittest.TestCase): role_id = 42 self.ctx.author.roles.append(MockRole(id=role_id)) self.assertTrue(checks.without_role_check(self.ctx, role_id + 10)) - - def test_in_channel_check_for_correct_channel(self): - self.ctx.channel.id = 42 - self.assertTrue(checks.in_channel_check(self.ctx, *[42])) - - def test_in_channel_check_for_incorrect_channel(self): - self.ctx.channel.id = 42 + 10 - self.assertFalse(checks.in_channel_check(self.ctx, *[42])) -- cgit v1.2.3 From 876fae1856f1ad876d74036899739115fd8b86c3 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 13:39:32 +0200 Subject: Add some tests for `in_whitelist_check`. --- tests/bot/utils/test_checks.py | 48 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/bot/utils/test_checks.py b/tests/bot/utils/test_checks.py index d572b6299..de72e5748 100644 --- a/tests/bot/utils/test_checks.py +++ b/tests/bot/utils/test_checks.py @@ -1,6 +1,8 @@ import unittest +from unittest.mock import MagicMock from bot.utils import checks +from bot.utils.checks import InWhitelistCheckFailure from tests.helpers import MockContext, MockRole @@ -41,3 +43,49 @@ class ChecksTests(unittest.TestCase): role_id = 42 self.ctx.author.roles.append(MockRole(id=role_id)) self.assertTrue(checks.without_role_check(self.ctx, role_id + 10)) + + def test_in_whitelist_check_correct_channel(self): + """`in_whitelist_check` returns `True` if `Context.channel.id` is in the channel list.""" + channel_id = 3 + self.ctx.channel.id = channel_id + self.assertTrue(checks.in_whitelist_check(self.ctx, [channel_id])) + + def test_in_whitelist_check_incorrect_channel(self): + """`in_whitelist_check` raises InWhitelistCheckFailure if there's no channel match.""" + self.ctx.channel.id = 3 + with self.assertRaises(InWhitelistCheckFailure): + checks.in_whitelist_check(self.ctx, [4]) + + def test_in_whitelist_check_correct_category(self): + """`in_whitelist_check` returns `True` if `Context.channel.category_id` is in the category list.""" + category_id = 3 + self.ctx.channel.category_id = category_id + self.assertTrue(checks.in_whitelist_check(self.ctx, categories=[category_id])) + + def test_in_whitelist_check_incorrect_category(self): + """`in_whitelist_check` raises InWhitelistCheckFailure if there's no category match.""" + self.ctx.channel.category_id = 3 + with self.assertRaises(InWhitelistCheckFailure): + checks.in_whitelist_check(self.ctx, categories=[4]) + + def test_in_whitelist_check_correct_role(self): + """`in_whitelist_check` returns `True` if any of the `Context.author.roles` are in the roles list.""" + self.ctx.author.roles = (MagicMock(id=1), MagicMock(id=2)) + self.assertTrue(checks.in_whitelist_check(self.ctx, roles=[2, 6])) + + def test_in_whitelist_check_incorrect_role(self): + """`in_whitelist_check` raises InWhitelistCheckFailure if there's no role match.""" + self.ctx.author.roles = (MagicMock(id=1), MagicMock(id=2)) + with self.assertRaises(InWhitelistCheckFailure): + checks.in_whitelist_check(self.ctx, roles=[4]) + + def test_in_whitelist_check_fail_silently(self): + """`in_whitelist_check` test no exception raised if `fail_silently` is `True`""" + self.assertFalse(checks.in_whitelist_check(self.ctx, roles=[2, 6], fail_silently=True)) + + def test_in_whitelist_check_complex(self): + """`in_whitelist_check` test with multiple parameters""" + self.ctx.author.roles = (MagicMock(id=1), MagicMock(id=2)) + self.ctx.channel.category_id = 3 + self.ctx.channel.id = 5 + self.assertTrue(checks.in_whitelist_check(self.ctx, channels=[1], categories=[8], roles=[2])) -- cgit v1.2.3 From 4db313e9a7899666f1597094b0d88447c7b64311 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 20:15:19 +0200 Subject: Floats are no longer permitted as RedisCache keys. Also added a test for this. This is the DRYest approach I could find. It's a little ugly, but I think it's probably good enough. --- bot/utils/redis_cache.py | 116 ++++++++++++++++++++++++------------ tests/bot/utils/test_redis_cache.py | 13 ++-- 2 files changed, 86 insertions(+), 43 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 33e5d5852..afd37f8f8 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -2,26 +2,42 @@ from __future__ import annotations import asyncio import logging -from typing import Any, Dict, ItemsView, Optional, Union +import typing +from typing import Any, Dict, ItemsView, Optional, Tuple, Union from bot.bot import Bot log = logging.getLogger(__name__) -RedisType = Union[str, int, float] -TYPESTRING_PREFIXES = ( +# Type aliases +RedisKeyType = Union[str, int] +RedisValueType = Union[str, int, float] + +# Prefix tuples +PrefixTuple = Tuple[Tuple[str, Any]] +TYPESTRING_VALUE_PREFIXES = ( ("f|", float), ("i|", int), ("s|", str), ) +TYPESTRING_KEY_PREFIXES = ( + ("i|", int), + ("s|", str), +) # Makes a nice list like "float, int, and str" -NICE_TYPE_LIST = ", ".join(str(_type.__name__) for _, _type in TYPESTRING_PREFIXES) -NICE_TYPE_LIST = ", and ".join(NICE_TYPE_LIST.rsplit(", ", 1)) +NICE_VALUE_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisValueType)) +NICE_VALUE_TYPE_LIST = ", and ".join(NICE_VALUE_TYPE_LIST.rsplit(", ", 1)) + +NICE_KEY_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisKeyType)) +NICE_KEY_TYPE_LIST = ", and ".join(NICE_KEY_TYPE_LIST.rsplit(", ", 1)) # Makes a list like "'f|', 'i|', and 's|'" -NICE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_PREFIXES]) -NICE_PREFIX_LIST = ", and ".join(NICE_PREFIX_LIST.rsplit(", ", 1)) +NICE_VALUE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_VALUE_PREFIXES]) +NICE_VALUE_PREFIX_LIST = ", and ".join(NICE_VALUE_PREFIX_LIST.rsplit(", ", 1)) + +NICE_KEY_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_KEY_PREFIXES]) +NICE_KEY_PREFIX_LIST = ", and ".join(NICE_KEY_PREFIX_LIST.rsplit(", ", 1)) class RedisCache: @@ -99,33 +115,57 @@ class RedisCache: self._namespace = namespace @staticmethod - def _to_typestring(value: RedisType) -> str: + def _to_typestring( + key_or_value: Union[RedisKeyType, RedisValueType], + prefixes: PrefixTuple, + nice_type_list: str + ) -> str: """Turn a valid Redis type into a typestring.""" - for prefix, _type in TYPESTRING_PREFIXES: - if isinstance(value, _type): - return f"{prefix}{value}" - raise TypeError(f"RedisCache._from_typestring only supports the types {NICE_TYPE_LIST}.") + for prefix, _type in prefixes: + if isinstance(key_or_value, _type): + return f"{prefix}{key_or_value}" + raise TypeError(f"RedisCache._from_typestring only supports the types {nice_type_list}.") @staticmethod - def _from_typestring(value: Union[bytes, str]) -> RedisType: - """Turn a typestring into a valid Redis type.""" + def _from_typestring( + key_or_value: Union[bytes, str], + prefixes: PrefixTuple, + nice_prefix_list: str, + ) -> Union[RedisKeyType, RedisValueType]: + """Deserialize a typestring into a valid Redis type.""" # Stuff that comes out of Redis will be bytestrings, so let's decode those. - if isinstance(value, bytes): - value = value.decode('utf-8') + if isinstance(key_or_value, bytes): + key_or_value = key_or_value.decode('utf-8') # Now we convert our unicode string back into the type it originally was. - for prefix, _type in TYPESTRING_PREFIXES: - if value.startswith(prefix): - return _type(value[len(prefix):]) - raise TypeError(f"RedisCache._to_typestring only supports the prefixes {NICE_PREFIX_LIST}.") + for prefix, _type in prefixes: + if key_or_value.startswith(prefix): + return _type(key_or_value[len(prefix):]) + raise TypeError(f"RedisCache._to_typestring only supports the prefixes {nice_prefix_list}.") + + def _key_to_typestring(self, key: RedisKeyType) -> str: + """Serialize a RedisKeyType object into a typestring.""" + return self._to_typestring(key, TYPESTRING_KEY_PREFIXES, NICE_KEY_TYPE_LIST) + + def _value_to_typestring(self, value: RedisValueType) -> str: + """Serialize a RedisValueType object into a typestring.""" + return self._to_typestring(value, TYPESTRING_VALUE_PREFIXES, NICE_VALUE_TYPE_LIST) + + def _key_from_typestring(self, key: Union[bytes, str]) -> RedisKeyType: + """Deserialize a RedisKeyType object from a typestring.""" + return self._from_typestring(key, TYPESTRING_KEY_PREFIXES, NICE_KEY_PREFIX_LIST) + + def _value_from_typestring(self, value: Union[bytes, str]) -> RedisValueType: + """Deserialize a RedisValueType object from a typestring.""" + return self._from_typestring(value, TYPESTRING_VALUE_PREFIXES, NICE_VALUE_PREFIX_LIST) def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" - return {self._from_typestring(key): self._from_typestring(value) for key, value in dictionary.items()} + return {self._key_from_typestring(key): self._value_from_typestring(value) for key, value in dictionary.items()} def _dict_to_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into typestrings.""" - return {self._to_typestring(key): self._to_typestring(value) for key, value in dictionary.items()} + return {self._key_to_typestring(key): self._value_to_typestring(value) for key, value in dictionary.items()} async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" @@ -209,21 +249,21 @@ class RedisCache: """Return a beautiful representation of this object instance.""" return f"RedisCache(namespace={self._namespace!r})" - async def set(self, key: RedisType, value: RedisType) -> None: + async def set(self, key: RedisKeyType, value: RedisValueType) -> None: """Store an item in the Redis cache.""" await self._validate_cache() # Convert to a typestring and then set it - key = self._to_typestring(key) - value = self._to_typestring(value) + key = self._key_to_typestring(key) + value = self._value_to_typestring(value) log.trace(f"Setting {key} to {value}.") await self._redis.hset(self._namespace, key, value) - async def get(self, key: RedisType, default: Optional[RedisType] = None) -> Optional[RedisType]: + async def get(self, key: RedisKeyType, default: Optional[RedisValueType] = None) -> Optional[RedisValueType]: """Get an item from the Redis cache.""" await self._validate_cache() - key = self._to_typestring(key) + key = self._key_to_typestring(key) log.trace(f"Attempting to retrieve {key}.") value = await self._redis.hget(self._namespace, key) @@ -232,11 +272,11 @@ class RedisCache: log.trace(f"Value not found, returning default value {default}") return default else: - value = self._from_typestring(value) + value = self._value_from_typestring(value) log.trace(f"Value found, returning value {value}") return value - async def delete(self, key: RedisType) -> None: + async def delete(self, key: RedisKeyType) -> None: """ Delete an item from the Redis cache. @@ -245,19 +285,19 @@ class RedisCache: See https://redis.io/commands/hdel for more info on how this works. """ await self._validate_cache() - key = self._to_typestring(key) + key = self._key_to_typestring(key) log.trace(f"Attempting to delete {key}.") return await self._redis.hdel(self._namespace, key) - async def contains(self, key: RedisType) -> bool: + async def contains(self, key: RedisKeyType) -> bool: """ Check if a key exists in the Redis cache. Return True if the key exists, otherwise False. """ await self._validate_cache() - key = self._to_typestring(key) + key = self._key_to_typestring(key) exists = await self._redis.hexists(self._namespace, key) log.trace(f"Testing if {key} exists in the RedisCache - Result is {exists}") @@ -304,7 +344,7 @@ class RedisCache: log.trace("Clearing the cache of all key/value pairs.") await self._redis.delete(self._namespace) - async def pop(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: + async def pop(self, key: RedisKeyType, default: Optional[RedisValueType] = None) -> RedisValueType: """Get the item, remove it from the cache, and provide a default if not found.""" log.trace(f"Attempting to pop {key}.") value = await self.get(key, default) @@ -317,7 +357,7 @@ class RedisCache: return value - async def update(self, items: Dict[RedisType, RedisType]) -> None: + async def update(self, items: Dict[RedisKeyType, RedisValueType]) -> None: """ Update the Redis cache with multiple values. @@ -326,14 +366,14 @@ class RedisCache: do not exist in the RedisCache, they are created. If they do exist, the values are updated with the new ones from `items`. - Please note that both the keys and the values in the `items` dictionary - must consist of valid RedisTypes - ints, floats, or strings. + Please note that keys and the values in the `items` dictionary + must consist of valid RedisKeyTypes and RedisValueTypes. """ await self._validate_cache() log.trace(f"Updating the cache with the following items:\n{items}") await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) - async def increment(self, key: RedisType, amount: Optional[int, float] = 1) -> None: + async def increment(self, key: RedisKeyType, amount: Optional[int, float] = 1) -> None: """ Increment the value by `amount`. @@ -373,7 +413,7 @@ class RedisCache: log.error(error_message) raise TypeError(error_message) - async def decrement(self, key: RedisType, amount: Optional[int, float] = 1) -> None: + async def decrement(self, key: RedisKeyType, amount: Optional[int, float] = 1) -> None: """ Decrement the value by `amount`. diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index efd168dac..4f95dff03 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -70,12 +70,15 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(await self.cog.redis.get('favorite_nothing', "bearclaw"), "bearclaw") async def test_set_item_type(self): - """Test that .set rejects keys and values that are not strings, ints or floats.""" + """Test that .set rejects keys and values that are not permitted.""" fruits = ["lemon", "melon", "apple"] with self.assertRaises(TypeError): await self.cog.redis.set(fruits, "nice") + with self.assertRaises(TypeError): + await self.cog.redis.set(4.23, "nice") + async def test_delete_item(self): """Test that .delete allows us to delete stuff from the RedisCache.""" # Add an item and verify that it gets added @@ -176,16 +179,16 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): # Test conversion to typestring for _input, expected in conversion_tests: - self.assertEqual(self.cog.redis._to_typestring(_input), expected) + self.assertEqual(self.cog.redis._value_to_typestring(_input), expected) # Test conversion from typestrings for _input, expected in conversion_tests: - self.assertEqual(self.cog.redis._from_typestring(expected), _input) + self.assertEqual(self.cog.redis._value_from_typestring(expected), _input) # Test that exceptions are raised on invalid input with self.assertRaises(TypeError): - self.cog.redis._to_typestring(["internet"]) - self.cog.redis._from_typestring("o|firedog") + self.cog.redis._value_to_typestring(["internet"]) + self.cog.redis._value_from_typestring("o|firedog") async def test_increment_decrement(self): """Test .increment and .decrement methods.""" -- cgit v1.2.3 From b6093bf7df00be1ed04a51119a65dbdd74ae0e58 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 21:01:24 +0200 Subject: Refactor typestring converters to partialmethods. We're using functools.partialmethod to make the code a little cleaner and more readable here. Read more about them here: https://docs.python.org/3/library/functools.html#functools.partial https://docs.python.org/3/library/functools.html#functools.partialmethod --- bot/utils/redis_cache.py | 54 +++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index afd37f8f8..dd24b83e8 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import logging import typing +from functools import partialmethod from typing import Any, Dict, ItemsView, Optional, Tuple, Union from bot.bot import Bot @@ -15,29 +16,29 @@ RedisValueType = Union[str, int, float] # Prefix tuples PrefixTuple = Tuple[Tuple[str, Any]] -TYPESTRING_VALUE_PREFIXES = ( +VALUE_PREFIXES = ( ("f|", float), ("i|", int), ("s|", str), ) -TYPESTRING_KEY_PREFIXES = ( +KEY_PREFIXES = ( ("i|", int), ("s|", str), ) # Makes a nice list like "float, int, and str" -NICE_VALUE_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisValueType)) -NICE_VALUE_TYPE_LIST = ", and ".join(NICE_VALUE_TYPE_LIST.rsplit(", ", 1)) +VALUE_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisValueType)) +VALUE_TYPE_LIST = ", and ".join(VALUE_TYPE_LIST.rsplit(", ", 1)) -NICE_KEY_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisKeyType)) -NICE_KEY_TYPE_LIST = ", and ".join(NICE_KEY_TYPE_LIST.rsplit(", ", 1)) +KEY_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisKeyType)) +KEY_TYPE_LIST = ", and ".join(KEY_TYPE_LIST.rsplit(", ", 1)) # Makes a list like "'f|', 'i|', and 's|'" -NICE_VALUE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_VALUE_PREFIXES]) -NICE_VALUE_PREFIX_LIST = ", and ".join(NICE_VALUE_PREFIX_LIST.rsplit(", ", 1)) +VALUE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in VALUE_PREFIXES]) +VALUE_PREFIX_LIST = ", and ".join(VALUE_PREFIX_LIST.rsplit(", ", 1)) -NICE_KEY_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_KEY_PREFIXES]) -NICE_KEY_PREFIX_LIST = ", and ".join(NICE_KEY_PREFIX_LIST.rsplit(", ", 1)) +KEY_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in KEY_PREFIXES]) +KEY_PREFIX_LIST = ", and ".join(KEY_PREFIX_LIST.rsplit(", ", 1)) class RedisCache: @@ -118,19 +119,19 @@ class RedisCache: def _to_typestring( key_or_value: Union[RedisKeyType, RedisValueType], prefixes: PrefixTuple, - nice_type_list: str + types_string: str ) -> str: """Turn a valid Redis type into a typestring.""" for prefix, _type in prefixes: if isinstance(key_or_value, _type): return f"{prefix}{key_or_value}" - raise TypeError(f"RedisCache._from_typestring only supports the types {nice_type_list}.") + raise TypeError(f"RedisCache._from_typestring only supports the types {types_string}.") @staticmethod def _from_typestring( key_or_value: Union[bytes, str], prefixes: PrefixTuple, - nice_prefix_list: str, + prefixes_string: str, ) -> Union[RedisKeyType, RedisValueType]: """Deserialize a typestring into a valid Redis type.""" # Stuff that comes out of Redis will be bytestrings, so let's decode those. @@ -141,23 +142,16 @@ class RedisCache: for prefix, _type in prefixes: if key_or_value.startswith(prefix): return _type(key_or_value[len(prefix):]) - raise TypeError(f"RedisCache._to_typestring only supports the prefixes {nice_prefix_list}.") - - def _key_to_typestring(self, key: RedisKeyType) -> str: - """Serialize a RedisKeyType object into a typestring.""" - return self._to_typestring(key, TYPESTRING_KEY_PREFIXES, NICE_KEY_TYPE_LIST) - - def _value_to_typestring(self, value: RedisValueType) -> str: - """Serialize a RedisValueType object into a typestring.""" - return self._to_typestring(value, TYPESTRING_VALUE_PREFIXES, NICE_VALUE_TYPE_LIST) - - def _key_from_typestring(self, key: Union[bytes, str]) -> RedisKeyType: - """Deserialize a RedisKeyType object from a typestring.""" - return self._from_typestring(key, TYPESTRING_KEY_PREFIXES, NICE_KEY_PREFIX_LIST) - - def _value_from_typestring(self, value: Union[bytes, str]) -> RedisValueType: - """Deserialize a RedisValueType object from a typestring.""" - return self._from_typestring(value, TYPESTRING_VALUE_PREFIXES, NICE_VALUE_PREFIX_LIST) + raise TypeError(f"RedisCache._to_typestring only supports the prefixes {prefixes_string}.") + + # Add some nice partials to call our generic typestring converters. + # These are basically functions that will fill in some of the parameters for you, so that + # any call to _key_to_typestring will be like calling _to_typestring with those two parameters + # pre-filled. + _key_to_typestring = partialmethod(_to_typestring, prefixes=KEY_PREFIXES, types_string=KEY_TYPE_LIST) + _value_to_typestring = partialmethod(_to_typestring, prefixes=VALUE_PREFIXES, types_string=VALUE_TYPE_LIST) + _key_from_typestring = partialmethod(_from_typestring, prefixes=KEY_PREFIXES, prefixes_string=KEY_PREFIX_LIST) + _value_from_typestring = partialmethod(_from_typestring, prefixes=VALUE_PREFIXES, prefixes_string=VALUE_PREFIX_LIST) def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" -- cgit v1.2.3 From 63b81b04da3cbc4d1824e65c977ec61532dbe605 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 21:04:08 +0200 Subject: Fix ATROCIOUS comment. I should be shot. --- bot/utils/redis_cache.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index dd24b83e8..a71ad2191 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -145,9 +145,11 @@ class RedisCache: raise TypeError(f"RedisCache._to_typestring only supports the prefixes {prefixes_string}.") # Add some nice partials to call our generic typestring converters. - # These are basically functions that will fill in some of the parameters for you, so that - # any call to _key_to_typestring will be like calling _to_typestring with those two parameters - # pre-filled. + # These are basically methods that will fill in some of the parameters for you, so that + # any call to _key_to_typestring will be like calling _to_typestring with the two parameters + # at `prefixes` and `types_string` pre-filled. + # + # See https://docs.python.org/3/library/functools.html#functools.partialmethod _key_to_typestring = partialmethod(_to_typestring, prefixes=KEY_PREFIXES, types_string=KEY_TYPE_LIST) _value_to_typestring = partialmethod(_to_typestring, prefixes=VALUE_PREFIXES, types_string=VALUE_TYPE_LIST) _key_from_typestring = partialmethod(_from_typestring, prefixes=KEY_PREFIXES, prefixes_string=KEY_PREFIX_LIST) -- cgit v1.2.3 From bdb7bbc5e98bd840785def6ac08c9f5a313847cb Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 28 May 2020 00:43:58 +0200 Subject: Reduce complexity on some of the typestring stuff. - Refactor error messages in _to_typestring and _from_typestring to just print the prefix tuples instead of that custom error string. - Create a RedisKeyOrValue type to simplify some annotations. - Simplify partialmethod calls. - Make the signatures for _to_typestring and _from_typestring one-liners - Fix a typo in the errors. --- bot/utils/redis_cache.py | 40 +++++++++------------------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index a71ad2191..0b682d378 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -2,7 +2,6 @@ from __future__ import annotations import asyncio import logging -import typing from functools import partialmethod from typing import Any, Dict, ItemsView, Optional, Tuple, Union @@ -13,6 +12,7 @@ log = logging.getLogger(__name__) # Type aliases RedisKeyType = Union[str, int] RedisValueType = Union[str, int, float] +RedisKeyOrValue = Union[RedisKeyType, RedisValueType] # Prefix tuples PrefixTuple = Tuple[Tuple[str, Any]] @@ -26,20 +26,6 @@ KEY_PREFIXES = ( ("s|", str), ) -# Makes a nice list like "float, int, and str" -VALUE_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisValueType)) -VALUE_TYPE_LIST = ", and ".join(VALUE_TYPE_LIST.rsplit(", ", 1)) - -KEY_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisKeyType)) -KEY_TYPE_LIST = ", and ".join(KEY_TYPE_LIST.rsplit(", ", 1)) - -# Makes a list like "'f|', 'i|', and 's|'" -VALUE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in VALUE_PREFIXES]) -VALUE_PREFIX_LIST = ", and ".join(VALUE_PREFIX_LIST.rsplit(", ", 1)) - -KEY_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in KEY_PREFIXES]) -KEY_PREFIX_LIST = ", and ".join(KEY_PREFIX_LIST.rsplit(", ", 1)) - class RedisCache: """ @@ -116,23 +102,15 @@ class RedisCache: self._namespace = namespace @staticmethod - def _to_typestring( - key_or_value: Union[RedisKeyType, RedisValueType], - prefixes: PrefixTuple, - types_string: str - ) -> str: + def _to_typestring(key_or_value: RedisKeyOrValue, prefixes: PrefixTuple) -> str: """Turn a valid Redis type into a typestring.""" for prefix, _type in prefixes: if isinstance(key_or_value, _type): return f"{prefix}{key_or_value}" - raise TypeError(f"RedisCache._from_typestring only supports the types {types_string}.") + raise TypeError(f"RedisCache._to_typestring only supports the following: {prefixes}.") @staticmethod - def _from_typestring( - key_or_value: Union[bytes, str], - prefixes: PrefixTuple, - prefixes_string: str, - ) -> Union[RedisKeyType, RedisValueType]: + def _from_typestring(key_or_value: Union[bytes, str], prefixes: PrefixTuple) -> RedisKeyOrValue: """Deserialize a typestring into a valid Redis type.""" # Stuff that comes out of Redis will be bytestrings, so let's decode those. if isinstance(key_or_value, bytes): @@ -142,7 +120,7 @@ class RedisCache: for prefix, _type in prefixes: if key_or_value.startswith(prefix): return _type(key_or_value[len(prefix):]) - raise TypeError(f"RedisCache._to_typestring only supports the prefixes {prefixes_string}.") + raise TypeError(f"RedisCache._from_typestring only supports the following: {prefixes}.") # Add some nice partials to call our generic typestring converters. # These are basically methods that will fill in some of the parameters for you, so that @@ -150,10 +128,10 @@ class RedisCache: # at `prefixes` and `types_string` pre-filled. # # See https://docs.python.org/3/library/functools.html#functools.partialmethod - _key_to_typestring = partialmethod(_to_typestring, prefixes=KEY_PREFIXES, types_string=KEY_TYPE_LIST) - _value_to_typestring = partialmethod(_to_typestring, prefixes=VALUE_PREFIXES, types_string=VALUE_TYPE_LIST) - _key_from_typestring = partialmethod(_from_typestring, prefixes=KEY_PREFIXES, prefixes_string=KEY_PREFIX_LIST) - _value_from_typestring = partialmethod(_from_typestring, prefixes=VALUE_PREFIXES, prefixes_string=VALUE_PREFIX_LIST) + _key_to_typestring = partialmethod(_to_typestring, prefixes=KEY_PREFIXES) + _value_to_typestring = partialmethod(_to_typestring, prefixes=VALUE_PREFIXES) + _key_from_typestring = partialmethod(_from_typestring, prefixes=KEY_PREFIXES) + _value_from_typestring = partialmethod(_from_typestring, prefixes=VALUE_PREFIXES) def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" -- cgit v1.2.3 From 11542fbcc7c32fb9a18577c45ae3c331eaa12db8 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 28 May 2020 00:56:10 +0200 Subject: Make prefix consts private and more precise. --- bot/utils/redis_cache.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 0b682d378..979ea5d47 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -15,13 +15,13 @@ RedisValueType = Union[str, int, float] RedisKeyOrValue = Union[RedisKeyType, RedisValueType] # Prefix tuples -PrefixTuple = Tuple[Tuple[str, Any]] -VALUE_PREFIXES = ( +_PrefixTuple = Tuple[Tuple[str, Any], ...] +_VALUE_PREFIXES = ( ("f|", float), ("i|", int), ("s|", str), ) -KEY_PREFIXES = ( +_KEY_PREFIXES = ( ("i|", int), ("s|", str), ) @@ -102,7 +102,7 @@ class RedisCache: self._namespace = namespace @staticmethod - def _to_typestring(key_or_value: RedisKeyOrValue, prefixes: PrefixTuple) -> str: + def _to_typestring(key_or_value: RedisKeyOrValue, prefixes: _PrefixTuple) -> str: """Turn a valid Redis type into a typestring.""" for prefix, _type in prefixes: if isinstance(key_or_value, _type): @@ -110,7 +110,7 @@ class RedisCache: raise TypeError(f"RedisCache._to_typestring only supports the following: {prefixes}.") @staticmethod - def _from_typestring(key_or_value: Union[bytes, str], prefixes: PrefixTuple) -> RedisKeyOrValue: + def _from_typestring(key_or_value: Union[bytes, str], prefixes: _PrefixTuple) -> RedisKeyOrValue: """Deserialize a typestring into a valid Redis type.""" # Stuff that comes out of Redis will be bytestrings, so let's decode those. if isinstance(key_or_value, bytes): @@ -128,10 +128,10 @@ class RedisCache: # at `prefixes` and `types_string` pre-filled. # # See https://docs.python.org/3/library/functools.html#functools.partialmethod - _key_to_typestring = partialmethod(_to_typestring, prefixes=KEY_PREFIXES) - _value_to_typestring = partialmethod(_to_typestring, prefixes=VALUE_PREFIXES) - _key_from_typestring = partialmethod(_from_typestring, prefixes=KEY_PREFIXES) - _value_from_typestring = partialmethod(_from_typestring, prefixes=VALUE_PREFIXES) + _key_to_typestring = partialmethod(_to_typestring, prefixes=_KEY_PREFIXES) + _value_to_typestring = partialmethod(_to_typestring, prefixes=_VALUE_PREFIXES) + _key_from_typestring = partialmethod(_from_typestring, prefixes=_KEY_PREFIXES) + _value_from_typestring = partialmethod(_from_typestring, prefixes=_VALUE_PREFIXES) def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" -- cgit v1.2.3 From f66a63501fe1ef8fb5390dfbe42ae9f95ea2bc28 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 28 May 2020 01:29:34 +0200 Subject: Add custom exceptions for each error state. The bot can get into trouble in three distinct ways: - It has no Bot instance - It has no namespace - It has no parent instance. These happen only if you're using it wrong. To make the test more precise, and to add a little bit more readability (RuntimeError could be anything!), we'll introduce some custom exceptions for these three states. This addresses a review comment by @aeros. --- bot/utils/redis_cache.py | 22 +++++++++++++++++----- tests/bot/utils/test_redis_cache.py | 7 ++++--- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 979ea5d47..6b3c68979 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -27,6 +27,18 @@ _KEY_PREFIXES = ( ) +class NoBotInstanceError(RuntimeError): + """Raised when RedisCache is created without an available bot instance on the owner class.""" + + +class NoNamespaceError(RuntimeError): + """Raised when RedisCache has no namespace, for example if it is not assigned to a class attribute.""" + + +class NoParentInstanceError(RuntimeError): + """Raised when the parent instance is available, for example if called by accessing the parent class directly.""" + + class RedisCache: """ A simplified interface for a Redis connection. @@ -149,7 +161,7 @@ class RedisCache: "This object must be initialized as a class attribute." ) log.error(error_message) - raise RuntimeError(error_message) + raise NoNamespaceError(error_message) if self.bot is None: error_message = ( @@ -159,7 +171,7 @@ class RedisCache: "the RedisCache inside a class that has a Bot instance attribute." ) log.error(error_message) - raise RuntimeError(error_message) + raise NoBotInstanceError(error_message) await self.bot.redis_ready.wait() @@ -194,7 +206,7 @@ class RedisCache: if self._namespace is None: error_message = "RedisCache must be a class attribute." log.error(error_message) - raise RuntimeError(error_message) + raise NoNamespaceError(error_message) if instance is None: error_message = ( @@ -202,7 +214,7 @@ class RedisCache: "before accessing it using the cog's class object." ) log.error(error_message) - raise RuntimeError(error_message) + raise NoParentInstanceError(error_message) for attribute in vars(instance).values(): if isinstance(attribute, Bot): @@ -217,7 +229,7 @@ class RedisCache: "the RedisCache inside a class that has a Bot instance attribute." ) log.error(error_message) - raise RuntimeError(error_message) + raise NoBotInstanceError(error_message) def __repr__(self) -> str: """Return a beautiful representation of this object instance.""" diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 4f95dff03..8c1a40640 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -4,6 +4,7 @@ import unittest import fakeredis.aioredis from bot.utils import RedisCache +from bot.utils.redis_cache import NoBotInstanceError, NoNamespaceError, NoParentInstanceError from tests import helpers @@ -260,13 +261,13 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): cog = MyCog() # Raises "No Bot instance" - with self.assertRaises(RuntimeError): + with self.assertRaises(NoBotInstanceError): await cog.cache.get("john") # Raises "RedisCache has no namespace" - with self.assertRaises(RuntimeError): + with self.assertRaises(NoNamespaceError): await cog.other_cache.get("was") # Raises "You must access the RedisCache instance through the cog instance" - with self.assertRaises(RuntimeError): + with self.assertRaises(NoParentInstanceError): await MyCog.cache.get("afraid") -- cgit v1.2.3 From 96db6087254c957fcb8fb45aad7ffcddb46ee839 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Wed, 27 May 2020 17:08:18 -0700 Subject: Switch findall to finditer in assertions `find_token_in_message` now uses the latter so the tests should adjust accordingly. --- tests/bot/cogs/test_token_remover.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 971bc93fc..4fff3ab33 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -94,18 +94,18 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): return_value = TokenRemover.find_token_in_message(self.msg) self.assertIsNone(return_value) - token_re.findall.assert_not_called() + token_re.finditer.assert_not_called() @autospec(TokenRemover, "is_maybe_token") @autospec("bot.cogs.token_remover", "TOKEN_RE") def test_find_token_no_matches_returns_none(self, token_re, is_maybe_token): """None should be returned if the regex matches no tokens in a message.""" - token_re.findall.return_value = () + token_re.finditer.return_value = () return_value = TokenRemover.find_token_in_message(self.msg) self.assertIsNone(return_value) - token_re.findall.assert_called_once_with(self.msg.content) + token_re.finditer.assert_called_once_with(self.msg.content) is_maybe_token.assert_not_called() @autospec(TokenRemover, "is_maybe_token") @@ -123,7 +123,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): return_value = TokenRemover.find_token_in_message(self.msg) self.assertEqual(return_value, matches[true_index]) - token_re.findall.assert_called_once_with(self.msg.content) + token_re.finditer.assert_called_once_with(self.msg.content) # assert_has_calls isn't used cause it'd allow for extra calls before or after. # The function should short-circuit, so nothing past true_index should have been used. -- cgit v1.2.3 From f937032466a4124bacf217d1bfd0af097fc3395d Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Wed, 27 May 2020 19:31:55 -0700 Subject: Adjust token remover tests to use the Token NamedTuple --- tests/bot/cogs/test_token_remover.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 4fff3ab33..65bc1ee58 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -7,7 +7,7 @@ from discord import Colour from bot import constants from bot.cogs import token_remover from bot.cogs.moderation import ModLog -from bot.cogs.token_remover import TokenRemover +from bot.cogs.token_remover import Token, TokenRemover from tests.helpers import MockBot, MockMessage, autospec @@ -224,17 +224,19 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): @autospec("bot.cogs.token_remover", "LOG_MESSAGE") def test_format_log_message(self, log_message): """Should correctly format the log message with info from the message and token.""" + token = Token("NDY3MjIzMjMwNjUwNzc3NjQx", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") log_message.format.return_value = "Howdy" - return_value = TokenRemover.format_log_message(self.msg, "MTIz.DN9R_A.xyz") + + return_value = TokenRemover.format_log_message(self.msg, token) self.assertEqual(return_value, log_message.format.return_value) log_message.format.assert_called_once_with( author=self.msg.author, author_id=self.msg.author.id, channel=self.msg.channel.mention, - user_id="MTIz", - timestamp="DN9R_A", - hmac="xxx", + user_id=token.user_id, + timestamp=token.timestamp, + hmac="x" * len(token.hmac), ) @mock.patch.object(TokenRemover, "mod_log", new_callable=mock.PropertyMock) @@ -244,7 +246,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): """Should delete the message and send a mod log.""" cog = TokenRemover(self.bot) mod_log = mock.create_autospec(ModLog, spec_set=True, instance=True) - token = "MTIz.DN9R_A.xyz" + token = mock.create_autospec(Token, spec_set=True, instance=True) log_msg = "testing123" mod_log_property.return_value = mod_log -- cgit v1.2.3 From 12b8f5002807144451a313180c639bf6b4925f2e Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Wed, 27 May 2020 20:00:33 -0700 Subject: Add more thorough and realistic inputs for token ID and timestamp tests The tests for valid inputs and invalid inputs were split to make them more readable. --- tests/bot/cogs/test_token_remover.py | 70 ++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 65bc1ee58..ffe76865a 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -24,31 +24,65 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): self.msg.author.__str__ = MagicMock(return_value=self.msg.author.name) self.msg.author.avatar_url_as.return_value = "picture-lemon.png" - def test_is_valid_user_id(self): - """Should correctly discern valid user IDs and ignore non-numeric and non-ASCII IDs.""" - subtests = ( - ("MTIz", True), # base64(123) - ("YWJj", False), # base64(abc) - ("λδµ", False), + def test_is_valid_user_id_valid(self): + """Should consider user IDs valid if they decode entirely to ASCII digits.""" + ids = ( + "NDcyMjY1OTQzMDYyNDEzMzMy", + "NDc1MDczNjI5Mzk5NTQ3OTA0", + "NDY3MjIzMjMwNjUwNzc3NjQx", ) - for user_id, is_valid in subtests: - with self.subTest(user_id=user_id, is_valid=is_valid): + for user_id in ids: + with self.subTest(user_id=user_id): result = TokenRemover.is_valid_user_id(user_id) - self.assertIs(result, is_valid) + self.assertTrue(result) + + def test_is_valid_user_id_invalid(self): + """Should consider non-digit and non-ASCII IDs invalid.""" + ids = ( + ("SGVsbG8gd29ybGQ", "non-digit ASCII"), + ("0J_RgNC40LLQtdGCINC80LjRgA", "cyrillic text"), + ("4pO14p6L4p6C4pG34p264pGl8J-EiOKSj-KCieKBsA", "Unicode digits"), + ("4oaA4oaB4oWh4oWi4Lyz4Lyq4Lyr4LG9", "Unicode numerals"), + ("8J2fjvCdn5nwnZ-k8J2fr_Cdn7rgravvvJngr6c", "Unicode decimals"), + ("{hello}[world]&(bye!)", "ASCII invalid Base64"), + ("Þíß-ï§-ňøẗ-våłìÐ", "Unicode invalid Base64"), + ) - def test_is_valid_timestamp(self): - """Should correctly discern valid timestamps.""" - subtests = ( - ("DN9r_A", True), - ("MTIz", False), # base64(123) - ("λδµ", False), + for user_id, msg in ids: + with self.subTest(msg=msg): + result = TokenRemover.is_valid_user_id(user_id) + self.assertFalse(result) + + def test_is_valid_timestamp_valid(self): + """Should consider timestamps valid if they're greater than the Discord epoch.""" + timestamps = ( + "XsyRkw", + "Xrim9Q", + "XsyR-w", + "XsySD_", + "Dn9r_A", + ) + + for timestamp in timestamps: + with self.subTest(timestamp=timestamp): + result = TokenRemover.is_valid_timestamp(timestamp) + self.assertTrue(result) + + def test_is_valid_timestamp_invalid(self): + """Should consider timestamps invalid if they're before Discord epoch or can't be parsed.""" + timestamps = ( + ("B4Yffw", "DISCORD_EPOCH - TOKEN_EPOCH - 1"), + ("ew", "123"), + ("AoIKgA", "42076800"), + ("{hello}[world]&(bye!)", "ASCII invalid Base64"), + ("Þíß-ï§-ňøẗ-våłìÐ", "Unicode invalid Base64"), ) - for timestamp, is_valid in subtests: - with self.subTest(timestamp=timestamp, is_valid=is_valid): + for timestamp, msg in timestamps: + with self.subTest(msg=msg): result = TokenRemover.is_valid_timestamp(timestamp) - self.assertIs(result, is_valid) + self.assertFalse(result) def test_mod_log_property(self): """The `mod_log` property should ask the bot to return the `ModLog` cog.""" -- cgit v1.2.3 From b7c30d41e263605a680cfe0f623b8e7ed5936b7d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 28 May 2020 10:35:17 +0200 Subject: Prevent a state where a coro could wait forever. This addresses a review comment by @aeros. --- bot/bot.py | 5 ++++- bot/utils/redis_cache.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bot/bot.py b/bot/bot.py index f1365d532..ba09ce207 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -32,6 +32,7 @@ class Bot(commands.Bot): self.http_session: Optional[aiohttp.ClientSession] = None self.redis_session: Optional[aioredis.Redis] = None self.redis_ready = asyncio.Event() + self.redis_closed = False self.api_client = api.APIClient(loop=self.loop) self._connector = None @@ -106,8 +107,9 @@ class Bot(commands.Bot): self.stats._transport.close() if self.redis_session: - self.redis_ready.clear() + self.redis_closed = True self.redis_session.close() + self.redis_ready.clear() await self.redis_session.wait_closed() async def login(self, *args, **kwargs) -> None: @@ -135,6 +137,7 @@ class Bot(commands.Bot): # Create the redis session self.loop.create_task(self._create_redis_session()) + self.redis_closed = False # Use AF_INET as its socket family to prevent HTTPS related problems both locally # and in production. diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 6b3c68979..de80cee84 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -173,7 +173,8 @@ class RedisCache: log.error(error_message) raise NoBotInstanceError(error_message) - await self.bot.redis_ready.wait() + if not self.bot.redis_closed: + await self.bot.redis_ready.wait() def __set_name__(self, owner: Any, attribute_name: str) -> None: """ -- cgit v1.2.3 From cc45960406f64a791a15cf9de76614103fda384b Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 28 May 2020 12:45:26 +0200 Subject: Move the `self.redis_closed` into session create. --- bot/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/bot.py b/bot/bot.py index ba09ce207..313652d11 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -70,6 +70,7 @@ class Bot(commands.Bot): password=constants.Redis.password, ) + self.redis_closed = False self.redis_ready.set() def add_cog(self, cog: commands.Cog) -> None: @@ -137,7 +138,6 @@ class Bot(commands.Bot): # Create the redis session self.loop.create_task(self._create_redis_session()) - self.redis_closed = False # Use AF_INET as its socket family to prevent HTTPS related problems both locally # and in production. -- cgit v1.2.3 From 67472080fef5c38b21d74daa2178c3f35081b58f Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 28 May 2020 19:52:41 -0700 Subject: Remove is_maybe_token tests The function was removed due to redundancy. Therefore, its tests are obsolete. --- tests/bot/cogs/test_token_remover.py | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index ffe76865a..5dd12636c 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -213,39 +213,6 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): results = [match[0] for match in results] self.assertCountEqual((token_1, token_2), results) - @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") - def test_is_maybe_token_missing_part_returns_false(self, valid_user, valid_time): - """False should be returned for tokens which do not have all 3 parts.""" - return_value = TokenRemover.is_maybe_token("x.y") - - self.assertFalse(return_value) - valid_user.assert_not_called() - valid_time.assert_not_called() - - @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") - def test_is_maybe_token(self, valid_user, valid_time): - """Should return True if the user ID and timestamp are valid or return False otherwise.""" - subtests = ( - (False, True, False), - (True, False, False), - (True, True, True), - ) - - for user_return, time_return, expected in subtests: - valid_user.reset_mock() - valid_time.reset_mock() - - with self.subTest(user_return=user_return, time_return=time_return, expected=expected): - valid_user.return_value = user_return - valid_time.return_value = time_return - - actual = TokenRemover.is_maybe_token("x.y.z") - self.assertIs(actual, expected) - - valid_user.assert_called_once_with("x") - if user_return: - valid_time.assert_called_once_with("y") - async def test_delete_message(self): """The message should be deleted, and a message should be sent to the same channel.""" await TokenRemover.delete_message(self.msg) -- cgit v1.2.3 From 84cd8235863acc80b7f140309424c33180cc34ea Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 28 May 2020 20:32:48 -0700 Subject: Adjust find_token_in_message tests for the recent cog changes It now supports the changes that switched to finditer, added match groups, and added the Token NamedTuple. It also accounts for the is_maybe_token function being removed. For the sake of simplicity, call assertions on is_valid_user_id and is_valid_timestamp were not made. --- tests/bot/cogs/test_token_remover.py | 39 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 5dd12636c..8238e235a 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -1,4 +1,5 @@ import unittest +from re import Match from unittest import mock from unittest.mock import MagicMock @@ -130,9 +131,8 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): self.assertIsNone(return_value) token_re.finditer.assert_not_called() - @autospec(TokenRemover, "is_maybe_token") @autospec("bot.cogs.token_remover", "TOKEN_RE") - def test_find_token_no_matches_returns_none(self, token_re, is_maybe_token): + def test_find_token_no_matches(self, token_re): """None should be returned if the regex matches no tokens in a message.""" token_re.finditer.return_value = () @@ -140,30 +140,31 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): self.assertIsNone(return_value) token_re.finditer.assert_called_once_with(self.msg.content) - is_maybe_token.assert_not_called() - @autospec(TokenRemover, "is_maybe_token") + @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") + @autospec("bot.cogs.token_remover", "Token") @autospec("bot.cogs.token_remover", "TOKEN_RE") - def test_find_token_returns_found_token(self, token_re, is_maybe_token): - """The found token should be returned.""" - true_index = 1 - matches = ("foo", "bar", "baz") - side_effects = [False] * len(matches) - side_effects[true_index] = True - - token_re.findall.return_value = matches - is_maybe_token.side_effect = side_effects + def test_find_token_valid_match(self, token_re, token_cls, is_valid_id, is_valid_timestamp): + """The first match with a valid user ID and timestamp should be returned as a `Token`.""" + matches = [ + mock.create_autospec(Match, spec_set=True, instance=True), + mock.create_autospec(Match, spec_set=True, instance=True), + ] + tokens = [ + mock.create_autospec(Token, spec_set=True, instance=True), + mock.create_autospec(Token, spec_set=True, instance=True), + ] + + token_re.finditer.return_value = matches + token_cls.side_effect = tokens + is_valid_id.side_effect = (False, True) # The 1st match will be invalid, 2nd one valid. + is_valid_timestamp.return_value = True return_value = TokenRemover.find_token_in_message(self.msg) - self.assertEqual(return_value, matches[true_index]) + self.assertEqual(tokens[1], return_value) token_re.finditer.assert_called_once_with(self.msg.content) - # assert_has_calls isn't used cause it'd allow for extra calls before or after. - # The function should short-circuit, so nothing past true_index should have been used. - calls = [mock.call(match) for match in matches[:true_index + 1]] - self.assertEqual(is_maybe_token.mock_calls, calls) - def test_regex_invalid_tokens(self): """Messages without anything looking like a token are not matched.""" tokens = ( -- cgit v1.2.3 From 5930a044b8347019d474a809fc86f89263574ad0 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 28 May 2020 20:33:34 -0700 Subject: Test find_token_in_message returns None for invalid matches This covers the case when a token is matched, but its user ID and timestamp turn out to be invalid. --- tests/bot/cogs/test_token_remover.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 8238e235a..9b4b04ecd 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -165,6 +165,21 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(tokens[1], return_value) token_re.finditer.assert_called_once_with(self.msg.content) + @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") + @autospec("bot.cogs.token_remover", "Token") + @autospec("bot.cogs.token_remover", "TOKEN_RE") + def test_find_token_invalid_matches(self, token_re, token_cls, is_valid_id, is_valid_timestamp): + """None should be returned if no matches have valid user IDs or timestamps.""" + token_re.finditer.return_value = [mock.create_autospec(Match, spec_set=True, instance=True)] + token_cls.return_value = mock.create_autospec(Token, spec_set=True, instance=True) + is_valid_id.return_value = False + is_valid_timestamp.return_value = False + + return_value = TokenRemover.find_token_in_message(self.msg) + + self.assertIsNone(return_value) + token_re.finditer.assert_called_once_with(self.msg.content) + def test_regex_invalid_tokens(self): """Messages without anything looking like a token are not matched.""" tokens = ( -- cgit v1.2.3 From 2d36b0a89410c229eb8c7629c49d46ffb7f1523d Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 29 May 2020 09:18:26 +0300 Subject: Filtering: Implement bad words detection in nicknames --- bot/cogs/filtering.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 1d9fddb12..d54beeabf 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -1,8 +1,10 @@ import logging import re +from datetime import datetime, timedelta from typing import Optional, Union import discord.errors +from dateutil import parser from dateutil.relativedelta import relativedelta from discord import Colour, Member, Message, TextChannel from discord.ext.commands import Cog @@ -14,6 +16,7 @@ from bot.constants import ( Channels, Colours, Filter, Icons, URLs ) +from bot.utils.redis_cache import RedisCache log = logging.getLogger(__name__) @@ -52,6 +55,9 @@ def expand_spoilers(text: str) -> str: class Filtering(Cog): """Filtering out invites, blacklisting domains, and warning us of certain regular expressions.""" + # Redis cache for last bad words in nickname alert sent per user. + name_alerts = RedisCache() + def __init__(self, bot: Bot): self.bot = bot @@ -126,6 +132,41 @@ class Filtering(Cog): delta = relativedelta(after.edited_at, before.edited_at).microseconds await self._filter_message(after, delta) + @Cog.listener('on_message') + async def bad_words_in_name(self, msg: Message) -> None: + """Check bad words from user display name. When there is more than 3 days after last alert, send new alert.""" + if await self.name_alerts.contains(msg.author.id): + last_alert = parser.isoparse(await self.name_alerts.get(msg.author.id)) + + # When there is less than 3 days after last alert, return + if datetime.now() - timedelta(days=3) < last_alert: + return + + # Check does nickname have match in filters. + matches = [] + for pattern in WATCHLIST_PATTERNS: + match = pattern.search(msg.author.display_name) + if match: + matches.append(match) + + # When there is any match, then send alert to mods. + if matches: + log_string = ( + f"**User:** {msg.author.mention} (`{msg.author.id}`)\n" + f"**Display Name:** {msg.author.display_name}\n" + f"**Bad Matches:** {', '.join(match.group() for match in matches)}" + ) + await self.mod_log.send_log_message( + icon_url=Icons.token_removed, + colour=Colours.soft_red, + title="Username filtering alert", + text=log_string, + channel_id=Channels.mod_alerts + ) + + # Update time when alert sent + await self.name_alerts.set(msg.author.id, datetime.now().isoformat()) + async def _filter_message(self, msg: Message, delta: Optional[int] = None) -> None: """Filter the input message to see if it violates any of our rules, and then respond accordingly.""" # Should we filter this message? -- cgit v1.2.3 From d12e84fe6834a0bc574e365a3283bc358c2ae4d9 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Fri, 29 May 2020 17:57:09 +0200 Subject: Ignore response when posting python news Sometimes a mailing list user doesn't press respond correctly to the email, and so a response is sent as a separate thread. To keep only new threads in the channel, we need to ignore those. --- bot/cogs/python_news.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/cogs/python_news.py b/bot/cogs/python_news.py index d28af4a0b..d15d0371e 100644 --- a/bot/cogs/python_news.py +++ b/bot/cogs/python_news.py @@ -153,6 +153,7 @@ class PythonNews(Cog): if ( thread_information["thread_id"] in existing_news["data"][maillist] + or 'Re: ' in thread_information["subject"] or new_date.date() < date.today() ): continue -- cgit v1.2.3 From 9ee955454141d093f1cd71fc84a5340f803fa142 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 29 May 2020 19:21:39 +0300 Subject: Filtering: Refactor bad names checking - Make `bad_words_in_name` and attach it to current `on_message`. - Implement `asyncio.Lock` to avoid race conditions. - Made that this first check is there matches and when there is, check for alert. --- bot/cogs/filtering.py | 67 +++++++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index d54beeabf..17113d551 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -1,3 +1,4 @@ +import asyncio import logging import re from datetime import datetime, timedelta @@ -60,6 +61,7 @@ class Filtering(Cog): def __init__(self, bot: Bot): self.bot = bot + self.name_lock: Optional[asyncio.Lock] = None staff_mistake_str = "If you believe this was a mistake, please let staff know!" self.filters = { @@ -118,6 +120,7 @@ class Filtering(Cog): async def on_message(self, msg: Message) -> None: """Invoke message filter for new messages.""" await self._filter_message(msg) + await self.bad_words_in_name(msg) @Cog.listener() async def on_message_edit(self, before: Message, after: Message) -> None: @@ -132,40 +135,42 @@ class Filtering(Cog): delta = relativedelta(after.edited_at, before.edited_at).microseconds await self._filter_message(after, delta) - @Cog.listener('on_message') async def bad_words_in_name(self, msg: Message) -> None: """Check bad words from user display name. When there is more than 3 days after last alert, send new alert.""" - if await self.name_alerts.contains(msg.author.id): - last_alert = parser.isoparse(await self.name_alerts.get(msg.author.id)) - - # When there is less than 3 days after last alert, return - if datetime.now() - timedelta(days=3) < last_alert: - return - - # Check does nickname have match in filters. - matches = [] - for pattern in WATCHLIST_PATTERNS: - match = pattern.search(msg.author.display_name) - if match: - matches.append(match) - - # When there is any match, then send alert to mods. - if matches: - log_string = ( - f"**User:** {msg.author.mention} (`{msg.author.id}`)\n" - f"**Display Name:** {msg.author.display_name}\n" - f"**Bad Matches:** {', '.join(match.group() for match in matches)}" - ) - await self.mod_log.send_log_message( - icon_url=Icons.token_removed, - colour=Colours.soft_red, - title="Username filtering alert", - text=log_string, - channel_id=Channels.mod_alerts - ) + if not self.name_lock: + self.name_lock = asyncio.Lock() + + # Use lock to avoid race conditions + async with self.name_lock: + # Check does nickname have match in filters. + matches = [] + for pattern in WATCHLIST_PATTERNS: + match = pattern.search(msg.author.display_name) + if match: + matches.append(match) + + if matches: + last_alert = await self.name_alerts.get(msg.author.id) + if last_alert: + last_alert = parser.isoparse(last_alert) + if datetime.now() - timedelta(days=3) < last_alert: + return + + log_string = ( + f"**User:** {msg.author.mention} (`{msg.author.id}`)\n" + f"**Display Name:** {msg.author.display_name}\n" + f"**Bad Matches:** {', '.join(match.group() for match in matches)}" + ) + await self.mod_log.send_log_message( + icon_url=Icons.token_removed, + colour=Colours.soft_red, + title="Username filtering alert", + text=log_string, + channel_id=Channels.mod_alerts + ) - # Update time when alert sent - await self.name_alerts.set(msg.author.id, datetime.now().isoformat()) + # Update time when alert sent + await self.name_alerts.set(msg.author.id, datetime.now().isoformat()) async def _filter_message(self, msg: Message, delta: Optional[int] = None) -> None: """Filter the input message to see if it violates any of our rules, and then respond accordingly.""" -- cgit v1.2.3 From 63f5028641e9b78c61d3bcfe3bbaa6f80c8a288a Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff Date: Fri, 29 May 2020 19:56:40 +0200 Subject: Fix `check_for_answer` breaking on missing cache The `check_for_answer` method of the HelpChannels cog relies on the channel->claimant cache being available. However, as this cache is (currently) lost during bot restarts, this method may fail with a KeyError exception. I've used `dict.get` with an `if not claimant: return` to circumvent this issue. --- bot/cogs/help_channels.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index d2a55fba6..2221132d4 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -660,10 +660,13 @@ class HelpChannels(Scheduler, commands.Cog): # Check if there is an entry in unanswered (does not persist across restarts) if channel.id in self.unanswered: - claimant_id = self.help_channel_claimants[channel].id + claimant = self.help_channel_claimants.get(channel) + if not claimant: + # The mapping for this channel was lost, we can't do anything. + return # Check the message did not come from the claimant - if claimant_id != message.author.id: + if claimant.id != message.author.id: # Mark the channel as answered self.unanswered[channel.id] = False -- cgit v1.2.3 From 28f20b969556e0ce1363ac44a5b9ff2bff2a6575 Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff Date: Fri, 29 May 2020 19:49:00 +0200 Subject: Reduce the number of help channel name changes Discord has introduced a new, strict rate limit for individual channel edits that reduces the number of allow channel name/channel topic changes to 2 per 10 minutes per channel. Unfortunately, our help channel system frequently goes over that rate limit as it edits the name and topic of a channel on all three "move" actions we have: to available, to occupied, and to dormant. In addition, our "unanswered" feature adds another channel name change on top of the move-related edits. That's why I've removed the topic/emoji changing features from the help channel system. This means we now have a generic topic that fits all three categories and no status emojis in the channel names. --- bot/cogs/help_channels.py | 32 +++----------------------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 2221132d4..70cef339a 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -24,18 +24,8 @@ ASKING_GUIDE_URL = "https://pythondiscord.com/pages/asking-good-questions/" MAX_CHANNELS_PER_CATEGORY = 50 EXCLUDED_CHANNELS = (constants.Channels.how_to_get_help,) -AVAILABLE_TOPIC = """ -This channel is available. Feel free to ask a question in order to claim this channel! -""" - -IN_USE_TOPIC = """ -This channel is currently in use. If you'd like to discuss a different problem, please claim a new \ -channel from the Help: Available category. -""" - -DORMANT_TOPIC = """ -This channel is temporarily archived. If you'd like to ask a question, please use one of the \ -channels in the Help: Available category. +HELP_CHANNEL_TOPIC = """ +This is a Python help channel. You can claim your own help channel in the Python Help: Available category. """ AVAILABLE_MSG = f""" @@ -64,11 +54,6 @@ question to maximize your chance of getting a good answer. If you're not sure ho through our guide for [asking a good question]({ASKING_GUIDE_URL}). """ -AVAILABLE_EMOJI = "✅" -IN_USE_ANSWERED_EMOJI = "⌛" -IN_USE_UNANSWERED_EMOJI = "⏳" -NAME_SEPARATOR = "｜" - CoroutineFunc = t.Callable[..., t.Coroutine] @@ -196,7 +181,7 @@ class HelpChannels(Scheduler, commands.Cog): return None log.debug(f"Creating a new dormant channel named {name}.") - return await self.dormant_category.create_text_channel(name) + return await self.dormant_category.create_text_channel(name, topic=HELP_CHANNEL_TOPIC) def create_name_queue(self) -> deque: """Return a queue of element names to use for creating new channels.""" @@ -542,8 +527,6 @@ class HelpChannels(Scheduler, commands.Cog): await self.move_to_bottom_position( channel=channel, category_id=constants.Categories.help_available, - name=f"{AVAILABLE_EMOJI}{NAME_SEPARATOR}{self.get_clean_channel_name(channel)}", - topic=AVAILABLE_TOPIC, ) self.report_stats() @@ -559,8 +542,6 @@ class HelpChannels(Scheduler, commands.Cog): await self.move_to_bottom_position( channel=channel, category_id=constants.Categories.help_dormant, - name=self.get_clean_channel_name(channel), - topic=DORMANT_TOPIC, ) self.bot.stats.incr(f"help.dormant_calls.{caller}") @@ -593,8 +574,6 @@ class HelpChannels(Scheduler, commands.Cog): await self.move_to_bottom_position( channel=channel, category_id=constants.Categories.help_in_use, - name=f"{IN_USE_UNANSWERED_EMOJI}{NAME_SEPARATOR}{self.get_clean_channel_name(channel)}", - topic=IN_USE_TOPIC, ) timeout = constants.HelpChannels.idle_minutes * 60 @@ -670,11 +649,6 @@ class HelpChannels(Scheduler, commands.Cog): # Mark the channel as answered self.unanswered[channel.id] = False - # Change the emoji in the channel name to signify activity - log.trace(f"#{channel} ({channel.id}) has been answered; changing its emoji") - name = self.get_clean_channel_name(channel) - await channel.edit(name=f"{IN_USE_ANSWERED_EMOJI}{NAME_SEPARATOR}{name}") - @commands.Cog.listener() async def on_message(self, message: discord.Message) -> None: """Move an available channel to the In Use category and replace it with a dormant one.""" -- cgit v1.2.3 From aa46d01fa6c6fd14a9613412783ce377fe7e967d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Fri, 29 May 2020 23:52:56 +0200 Subject: Clean up channel counts and add staff channels. Cleaning up a particularly dirty line by turning it into like 10 lines, and also adding the number of channels that are hidden to the `@everyone` role - which we're classifying as "Staff channels". --- bot/cogs/information.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index f0eb3a1ea..8309cff4b 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -104,8 +104,27 @@ class Information(Cog): member_count = ctx.guild.member_count # How many of each type of channel? - channels = Counter(c.type for c in ctx.guild.channels) - channel_counts = "".join(sorted(f"{str(ch).title()} channels: {channels[ch]}\n" for ch in channels)).strip() + channel_counter = Counter(c.type for c in ctx.guild.channels) + channel_type_list = [] + for channel in channel_counter: + channel_type = str(channel).title() + channel_type_list.append(f"{channel_type} channels: {channel_counter[channel]}") + + channel_type_list = sorted(channel_type_list) + channel_counts = "\n".join(channel_type_list).strip() + + # How many channels are for staff only? + everyone_role = ctx.guild.roles[0] + hidden_channels = 0 + + for channel in ctx.guild.channels: + overwrites = channel.overwrites_for(everyone_role) + if overwrites.is_empty(): + continue + + for perm, value in overwrites: + if perm == 'read_messages' and value is False: + hidden_channels += 1 # How many of each user status? statuses = Counter(member.status for member in ctx.guild.members) @@ -126,6 +145,7 @@ class Information(Cog): Members: {member_count:,} Roles: {roles} $channel_counts + Staff channels: {hidden_channels} **Members** {constants.Emojis.status_online} {statuses[Status.online]:,} -- cgit v1.2.3 From 171c1e2713355570baf50c687e7466daea834b89 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 00:02:47 +0200 Subject: Adding staff member count to !server. --- bot/cogs/information.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index 8309cff4b..d830806c1 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -130,6 +130,9 @@ class Information(Cog): statuses = Counter(member.status for member in ctx.guild.members) embed = Embed(colour=Colour.blurple()) + # How many staff members? + staff_members = len(ctx.guild.get_role(constants.Roles.helpers).members) + # Because channel_counts lacks leading whitespace, it breaks the dedent if it's inserted directly by the # f-string. While this is correctly formated by Discord, it makes unit testing difficult. To keep the formatting # without joining a tuple of strings we can use a Template string to insert the already-formatted channel_counts @@ -141,13 +144,16 @@ class Information(Cog): Voice region: {region} Features: {features} - **Counts** - Members: {member_count:,} - Roles: {roles} + **Channel counts** $channel_counts Staff channels: {hidden_channels} - **Members** + **Member counts** + Members: {member_count:,} + Staff members: {staff_members} + Roles: {roles} + + **Member statuses** {constants.Emojis.status_online} {statuses[Status.online]:,} {constants.Emojis.status_idle} {statuses[Status.idle]:,} {constants.Emojis.status_dnd} {statuses[Status.dnd]:,} -- cgit v1.2.3 From b258f77d1c9a1b62fef26e7fecdb89e57719dfac Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 00:12:44 +0200 Subject: Removing the periodic ping from verification. It's no longer needed, and causes problems with anti-raid and anti-spam. --- bot/cogs/verification.py | 44 +------------------------------------------- 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 99be3cdaa..0a087cee9 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -1,9 +1,7 @@ import logging from contextlib import suppress -from datetime import datetime from discord import Colour, Forbidden, Message, NotFound, Object -from discord.ext import tasks from discord.ext.commands import Cog, Context, command from bot import constants @@ -34,14 +32,6 @@ If you'd like to unsubscribe from the announcement notifications, simply send `! <#{constants.Channels.bot_commands}>. """ -if constants.DEBUG_MODE: - PERIODIC_PING = "Periodic checkpoint message successfully sent." -else: - PERIODIC_PING = ( - f"@everyone To verify that you have read our rules, please type `{constants.Bot.prefix}accept`." - " If you encounter any problems during the verification process, " - f"send a direct message to a staff member." - ) BOT_MESSAGE_DELETE_DELAY = 10 @@ -50,7 +40,6 @@ class Verification(Cog): def __init__(self, bot: Bot): self.bot = bot - self.periodic_ping.start() @property def mod_log(self) -> ModLog: @@ -65,10 +54,7 @@ class Verification(Cog): if message.author.bot: # They're a bot, delete their message after the delay. - # But not the periodic ping; we like that one. - if message.content != PERIODIC_PING: - await message.delete(delay=BOT_MESSAGE_DELETE_DELAY) - return + await message.delete(delay=BOT_MESSAGE_DELETE_DELAY) # if a user mentions a role or guild member # alert the mods in mod-alerts channel @@ -198,34 +184,6 @@ class Verification(Cog): else: return True - @tasks.loop(hours=12) - async def periodic_ping(self) -> None: - """Every week, mention @everyone to remind them to verify.""" - messages = self.bot.get_channel(constants.Channels.verification).history(limit=10) - need_to_post = True # True if a new message needs to be sent. - - async for message in messages: - if message.author == self.bot.user and message.content == PERIODIC_PING: - delta = datetime.utcnow() - message.created_at # Time since last message. - if delta.days >= 7: # Message is older than a week. - await message.delete() - else: - need_to_post = False - - break - - if need_to_post: - await self.bot.get_channel(constants.Channels.verification).send(PERIODIC_PING) - - @periodic_ping.before_loop - async def before_ping(self) -> None: - """Only start the loop when the bot is ready.""" - await self.bot.wait_until_guild_available() - - def cog_unload(self) -> None: - """Cancel the periodic ping task when the cog is unloaded.""" - self.periodic_ping.cancel() - def setup(bot: Bot) -> None: """Load the Verification cog.""" -- cgit v1.2.3 From f7fe7df271b0e0930fa8b997c087bb3be141d016 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 11 Apr 2020 07:43:56 +0200 Subject: Tags: explicitly use UTF-8 to read files Not all operating systems use UTF-8 as the default encoding. For systems that don't, reading tag files with Unicode would cause an unhandled exception. (cherry picked from commit adc75ff9bbcf8b905bd78c78f253522ae5e42fc3) --- bot/cogs/tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index bc7f53f68..6f03a3475 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -44,7 +44,7 @@ class Tags(Cog): tag = { "title": tag_title, "embed": { - "description": file.read_text(), + "description": file.read_text(encoding="utf8"), }, "restricted_to": "developers", } -- cgit v1.2.3 From 89752c5ff15bc55bb1986d131f562bedfdf9e63a Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 01:35:19 +0200 Subject: More precise staff-channel check. We now check: - Does the @everyone role have explicit read deny permissions? - Do staff roles have explicit read allow permissions? If the answer to both of these are yes, it's a staff channel. By 'staff roles', I mean Helpers, Moderators or Admins. --- bot/cogs/information.py | 66 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index d830806c1..715623620 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -1,11 +1,13 @@ import colorsys +import functools import logging import pprint import textwrap from collections import Counter, defaultdict from string import Template -from typing import Any, Mapping, Optional, Union +from typing import Any, List, Mapping, Optional, Union +import more_itertools from discord import Colour, Embed, Member, Message, Role, Status, utils from discord.ext.commands import BucketType, Cog, Context, Paginator, command, group from discord.utils import escape_markdown @@ -26,6 +28,34 @@ class Information(Cog): def __init__(self, bot: Bot): self.bot = bot + @staticmethod + def _get_channels_with_role_permission(ctx: Context, role: Role, perm: str, value: Optional[bool]) -> List[int]: + """Get a list of channel IDs where a role has a specific permission set to a specific value.""" + channel_ids = [] + + for channel in ctx.guild.channels: + overwrites = channel.overwrites_for(role) + if overwrites.is_empty(): + continue + + for _perm, _value in overwrites: + if _perm == perm and _value is value: + channel_ids.append(channel.id) + + return channel_ids + + _get_channels_where_role_can_read = functools.partialmethod( + _get_channels_with_role_permission, + perm='read_messages', + value=True + ) + + _get_channels_where_role_cannot_read = functools.partialmethod( + _get_channels_with_role_permission, + perm='read_messages', + value=False + ) + @with_role(*constants.MODERATION_ROLES) @command(name="roles") async def roles_info(self, ctx: Context) -> None: @@ -114,17 +144,27 @@ class Information(Cog): channel_counts = "\n".join(channel_type_list).strip() # How many channels are for staff only? - everyone_role = ctx.guild.roles[0] - hidden_channels = 0 - - for channel in ctx.guild.channels: - overwrites = channel.overwrites_for(everyone_role) - if overwrites.is_empty(): - continue - - for perm, value in overwrites: - if perm == 'read_messages' and value is False: - hidden_channels += 1 + # We need to know two things about a channel: + # - Does the @everyone role have explicit read deny permissions? + # - Do staff roles have explicit read allow permissions? + # + # If the answer to both of these questions is yes, it's a staff channel. + helpers = ctx.guild.get_role(constants.Roles.helpers) + moderators = ctx.guild.get_role(constants.Roles.moderators) + admins = ctx.guild.get_role(constants.Roles.admins) + everyone = ctx.guild.roles[0] + + # Let's build some lists of channels. + everyone_denied = self._get_channels_where_role_cannot_read(ctx, everyone) + staff_allowed = more_itertools.flatten([ + self._get_channels_where_role_can_read(ctx, admins), # Admins has explicit read message allow + self._get_channels_where_role_can_read(ctx, moderators), # Moderators has explicit read message allow + self._get_channels_where_role_can_read(ctx, helpers), # Helpers has explicit read message allow + ]) + + # Now we need to check which channels are both denied for everyone and permitted for staff + staff_channels = [cid for cid in staff_allowed if cid in everyone_denied] + staff_channel_count = len(staff_channels) # How many of each user status? statuses = Counter(member.status for member in ctx.guild.members) @@ -146,7 +186,7 @@ class Information(Cog): **Channel counts** $channel_counts - Staff channels: {hidden_channels} + Staff channels: {staff_channel_count} **Member counts** Members: {member_count:,} -- cgit v1.2.3 From f59e63454ffa582765847e8a26d9d97dcd9ff7b2 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 01:42:02 +0200 Subject: Fix busted test_information test. I wish this test didn't exist. --- tests/bot/cogs/test_information.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/bot/cogs/test_information.py b/tests/bot/cogs/test_information.py index aca6b594f..79c0e0ad3 100644 --- a/tests/bot/cogs/test_information.py +++ b/tests/bot/cogs/test_information.py @@ -148,14 +148,18 @@ class InformationCogTests(unittest.TestCase): Voice region: {self.ctx.guild.region} Features: {', '.join(self.ctx.guild.features)} - **Counts** - Members: {self.ctx.guild.member_count:,} - Roles: {len(self.ctx.guild.roles)} + **Channel counts** Category channels: 1 Text channels: 1 Voice channels: 1 + Staff channels: 0 + + **Member counts** + Members: {self.ctx.guild.member_count:,} + Staff members: 0 + Roles: {len(self.ctx.guild.roles)} - **Members** + **Member statuses** {constants.Emojis.status_online} 2 {constants.Emojis.status_idle} 1 {constants.Emojis.status_dnd} 4 -- cgit v1.2.3 From 96b026198a4ca2074f4fd7ea68e8a09acd5b38e4 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:34:39 +0300 Subject: Simplify infraction reason truncation tests --- tests/bot/cogs/moderation/test_infractions.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 5548d9f68..ad3c95958 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -27,15 +27,14 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): self.cog.apply_infraction = AsyncMock() self.bot.get_cog.return_value = AsyncMock() self.cog.mod_log.ignore = Mock() + self.ctx.guild.ban = Mock() await self.cog.apply_ban(self.ctx, self.target, "foo bar" * 3000) - ban = self.cog.apply_infraction.call_args[0][3] - self.assertEqual( - ban.cr_frame.f_locals["kwargs"]["reason"], - textwrap.shorten("foo bar" * 3000, 512, placeholder="...") + self.ctx.guild.ban.assert_called_once_with( + self.target, + reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="..."), + delete_message_days=0 ) - # Await ban to avoid not awaited coroutine warning - await ban @patch("bot.cogs.moderation.utils.post_infraction") async def test_apply_kick_reason_truncation(self, post_infraction_mock): @@ -44,12 +43,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): self.cog.apply_infraction = AsyncMock() self.cog.mod_log.ignore = Mock() + self.target.kick = Mock() await self.cog.apply_kick(self.ctx, self.target, "foo bar" * 3000) - kick = self.cog.apply_infraction.call_args[0][3] - self.assertEqual( - kick.cr_frame.f_locals["kwargs"]["reason"], - textwrap.shorten("foo bar" * 3000, 512, placeholder="...") - ) - # Await kick to avoid not awaited coroutine warning - await kick + self.target.kick.assert_called_once_with(reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="...")) -- cgit v1.2.3 From 7f827abfa1922a4ec81d2f49fa2811471588269d Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:44:01 +0300 Subject: Scheduler: Move inline f-string if-else statement to normal if statement --- bot/cogs/moderation/scheduler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index b65048f4c..80a58484c 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -174,12 +174,15 @@ class InfractionScheduler(Scheduler): dm_result = f"{constants.Emojis.failmail} " log.trace(f"Deleted infraction {infraction['id']} from database because applying infraction failed.") await self.bot.api_client.delete(f"bot/infractions/{infraction['id']}") + infr_message = "" + else: + infr_message = f"**{infr_type}** to {user.mention}{expiry_msg}{end_msg}" # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} confirmation message.") await ctx.send( f"{dm_result}{confirm_msg} " - f"{f'**{infr_type}** to {user.mention}{expiry_msg}{end_msg}' if not failed else ''}." + f"{infr_message}." ) # Send a log message to the mod log. -- cgit v1.2.3 From 136ef112999b9387f04c3e0b800a3008ac07934f Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:44:36 +0300 Subject: Scheduler: Remove invalid comment --- bot/cogs/moderation/scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 80a58484c..7e8455740 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -84,7 +84,6 @@ class InfractionScheduler(Scheduler): """Apply an infraction to the user, log the infraction, and optionally notify the user.""" infr_type = infraction["type"] icon = utils.INFRACTION_ICONS[infr_type][0] - # Truncate reason when it's too long to avoid raising error on sending ModLog entry reason = infraction["reason"] expiry = time.format_infraction_with_duration(infraction["expires_at"]) id_ = infraction['id'] -- cgit v1.2.3 From f42ddf64abfd487fd69eec275d1011112eb76166 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:53:38 +0300 Subject: Scheduler: Add try-except to infraction deletion --- bot/cogs/moderation/scheduler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 7e8455740..dcc0001f8 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -172,7 +172,12 @@ class InfractionScheduler(Scheduler): dm_log_text = "\nDM: **Canceled**" dm_result = f"{constants.Emojis.failmail} " log.trace(f"Deleted infraction {infraction['id']} from database because applying infraction failed.") - await self.bot.api_client.delete(f"bot/infractions/{infraction['id']}") + try: + await self.bot.api_client.delete(f"bot/infractions/{id_}") + except ResponseCodeError as e: + confirm_msg += f" and failed to delete" + log_title += " and failed to delete" + log.error(f"Deletion of {infr_type} infraction #{id_} failed with error code {e.status}.") infr_message = "" else: infr_message = f"**{infr_type}** to {user.mention}{expiry_msg}{end_msg}" -- cgit v1.2.3 From e71beb79f6f78b348ff17974844806c981da3c2d Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:56:37 +0300 Subject: Scheduler: Remove unnecessary `f` before string --- bot/cogs/moderation/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index dcc0001f8..0d4f0ffba 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -175,7 +175,7 @@ class InfractionScheduler(Scheduler): try: await self.bot.api_client.delete(f"bot/infractions/{id_}") except ResponseCodeError as e: - confirm_msg += f" and failed to delete" + confirm_msg += " and failed to delete" log_title += " and failed to delete" log.error(f"Deletion of {infr_type} infraction #{id_} failed with error code {e.status}.") infr_message = "" -- cgit v1.2.3 From 854b27593e13f1e35810c37f4be91e5c8c4516b2 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:58:14 +0300 Subject: Scheduler: Fix spaces for modlog text Co-authored-by: Mark --- bot/cogs/moderation/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 0d4f0ffba..8b28afa69 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -198,7 +198,7 @@ class InfractionScheduler(Scheduler): thumbnail=user.avatar_url_as(static_format="png"), text=textwrap.dedent(f""" Member: {user.mention} (`{user.id}`) - Actor: {ctx.message.author}{dm_log_text} {expiry_log_text} + Actor: {ctx.message.author}{dm_log_text}{expiry_log_text} Reason: {reason} """), content=log_content, -- cgit v1.2.3 From 323317496310ef474a39d468e273703106e44768 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 10:07:21 +0300 Subject: Infr. Tests: Add `apply_infraction` awaiting assertion with args --- tests/bot/cogs/moderation/test_infractions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index ad3c95958..da4e92ccc 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -35,6 +35,9 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="..."), delete_message_days=0 ) + self.cog.apply_infraction.assert_awaited_once_with( + self.ctx, {"foo": "bar"}, self.target, self.ctx.guild.ban.return_value + ) @patch("bot.cogs.moderation.utils.post_infraction") async def test_apply_kick_reason_truncation(self, post_infraction_mock): @@ -47,3 +50,6 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): await self.cog.apply_kick(self.ctx, self.target, "foo bar" * 3000) self.target.kick.assert_called_once_with(reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="...")) + self.cog.apply_infraction.assert_awaited_once_with( + self.ctx, {"foo": "bar"}, self.target, self.target.kick.return_value + ) -- cgit v1.2.3 From e236113612c560326176da91f5a743c514ac988b Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 10:10:34 +0300 Subject: Scheduler: Remove line splitting from `ctx.send` after 7f827ab --- bot/cogs/moderation/scheduler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 8b28afa69..3679561a3 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -184,10 +184,7 @@ class InfractionScheduler(Scheduler): # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} confirmation message.") - await ctx.send( - f"{dm_result}{confirm_msg} " - f"{infr_message}." - ) + await ctx.send(f"{dm_result}{confirm_msg} {infr_message}.") # Send a log message to the mod log. log.trace(f"Sending apply mod log for infraction #{id_}.") -- cgit v1.2.3 From f1f2c488dc29d731b3343c949fe49cc3eaced842 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 10:17:05 +0300 Subject: Scheduler: Move space from f-string of `ctx.send` to `infr_message` --- bot/cogs/moderation/scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 3679561a3..1c7786df4 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -180,11 +180,11 @@ class InfractionScheduler(Scheduler): log.error(f"Deletion of {infr_type} infraction #{id_} failed with error code {e.status}.") infr_message = "" else: - infr_message = f"**{infr_type}** to {user.mention}{expiry_msg}{end_msg}" + infr_message = f" **{infr_type}** to {user.mention}{expiry_msg}{end_msg}" # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} confirmation message.") - await ctx.send(f"{dm_result}{confirm_msg} {infr_message}.") + await ctx.send(f"{dm_result}{confirm_msg}{infr_message}.") # Send a log message to the mod log. log.trace(f"Sending apply mod log for infraction #{id_}.") -- cgit v1.2.3 From fa3cd5fef8e2f80a85ebc460cffb4e59c8e6387a Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 10:37:41 +0200 Subject: Prevent duplicates, and break into function. - We're using a set comprehension and flipping the order for counting the number of channels that are both staff allow and @everyone deny. - We're breaking the staff channel count stuff into a separate helper function so it doesn't crowd the server_info() scope. These fixes are both to address the code review from @MarkKoz, thanks Mark. --- bot/cogs/information.py | 59 +++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index 715623620..9ebb89300 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -30,7 +30,7 @@ class Information(Cog): @staticmethod def _get_channels_with_role_permission(ctx: Context, role: Role, perm: str, value: Optional[bool]) -> List[int]: - """Get a list of channel IDs where a role has a specific permission set to a specific value.""" + """Get a list of channel IDs where one of the specified roles can read.""" channel_ids = [] for channel in ctx.guild.channels: @@ -56,6 +56,33 @@ class Information(Cog): value=False ) + def _get_staff_channel_count(self, ctx: Context) -> int: + """ + Get number of channels that are staff-only. + + We need to know two things about a channel: + - Does the @everyone role have explicit read deny permissions? + - Do staff roles have explicit read allow permissions? + + If the answer to both of these questions is yes, it's a staff channel. + """ + helpers = ctx.guild.get_role(constants.Roles.helpers) + moderators = ctx.guild.get_role(constants.Roles.moderators) + admins = ctx.guild.get_role(constants.Roles.admins) + everyone = ctx.guild.default_role + + # Let's build some lists of channels. + everyone_denied = self._get_channels_where_role_cannot_read(ctx, everyone) + staff_allowed = more_itertools.flatten([ + self._get_channels_where_role_can_read(ctx, admins), # Admins has explicit read message allow + self._get_channels_where_role_can_read(ctx, moderators), # Moderators has explicit read message allow + self._get_channels_where_role_can_read(ctx, helpers), # Helpers has explicit read message allow + ]) + + # Now we need to check which channels are both denied for @everyone and permitted for staff + staff_channels = set(cid for cid in everyone_denied if cid in staff_allowed) + return len(staff_channels) + @with_role(*constants.MODERATION_ROLES) @command(name="roles") async def roles_info(self, ctx: Context) -> None: @@ -143,35 +170,13 @@ class Information(Cog): channel_type_list = sorted(channel_type_list) channel_counts = "\n".join(channel_type_list).strip() - # How many channels are for staff only? - # We need to know two things about a channel: - # - Does the @everyone role have explicit read deny permissions? - # - Do staff roles have explicit read allow permissions? - # - # If the answer to both of these questions is yes, it's a staff channel. - helpers = ctx.guild.get_role(constants.Roles.helpers) - moderators = ctx.guild.get_role(constants.Roles.moderators) - admins = ctx.guild.get_role(constants.Roles.admins) - everyone = ctx.guild.roles[0] - - # Let's build some lists of channels. - everyone_denied = self._get_channels_where_role_cannot_read(ctx, everyone) - staff_allowed = more_itertools.flatten([ - self._get_channels_where_role_can_read(ctx, admins), # Admins has explicit read message allow - self._get_channels_where_role_can_read(ctx, moderators), # Moderators has explicit read message allow - self._get_channels_where_role_can_read(ctx, helpers), # Helpers has explicit read message allow - ]) - - # Now we need to check which channels are both denied for everyone and permitted for staff - staff_channels = [cid for cid in staff_allowed if cid in everyone_denied] - staff_channel_count = len(staff_channels) - # How many of each user status? statuses = Counter(member.status for member in ctx.guild.members) embed = Embed(colour=Colour.blurple()) - # How many staff members? - staff_members = len(ctx.guild.get_role(constants.Roles.helpers).members) + # How many staff members and staff channels do we have? + staff_member_count = len(ctx.guild.get_role(constants.Roles.helpers).members) + staff_channel_count = self._get_staff_channel_count() # Because channel_counts lacks leading whitespace, it breaks the dedent if it's inserted directly by the # f-string. While this is correctly formated by Discord, it makes unit testing difficult. To keep the formatting @@ -190,7 +195,7 @@ class Information(Cog): **Member counts** Members: {member_count:,} - Staff members: {staff_members} + Staff members: {staff_member_count} Roles: {roles} **Member statuses** -- cgit v1.2.3 From 8562ed2feb6ef465b8b502c725566de4f0a06cbb Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 10:49:38 +0200 Subject: Don't membership check in an itertools.chain. We're using the set comprehension to prevent duplicates anyway, so flipping these back makes more sense. Also added a missing ctx and tested ok. --- bot/cogs/information.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index 9ebb89300..d3a2768d4 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -80,7 +80,7 @@ class Information(Cog): ]) # Now we need to check which channels are both denied for @everyone and permitted for staff - staff_channels = set(cid for cid in everyone_denied if cid in staff_allowed) + staff_channels = set(cid for cid in staff_allowed if cid in everyone_denied) return len(staff_channels) @with_role(*constants.MODERATION_ROLES) @@ -176,7 +176,7 @@ class Information(Cog): # How many staff members and staff channels do we have? staff_member_count = len(ctx.guild.get_role(constants.Roles.helpers).members) - staff_channel_count = self._get_staff_channel_count() + staff_channel_count = self._get_staff_channel_count(ctx) # Because channel_counts lacks leading whitespace, it breaks the dedent if it's inserted directly by the # f-string. While this is correctly formated by Discord, it makes unit testing difficult. To keep the formatting -- cgit v1.2.3 From 0e12ff189a416127421a878c2421d7f5a369d26e Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 16:32:07 +0300 Subject: Filtering: Create lock in `__init__` Move lock creation from `bad_words_in_name` to `__init__` --- bot/cogs/filtering.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 17113d551..c57ab0688 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -61,7 +61,7 @@ class Filtering(Cog): def __init__(self, bot: Bot): self.bot = bot - self.name_lock: Optional[asyncio.Lock] = None + self.name_lock = asyncio.Lock() staff_mistake_str = "If you believe this was a mistake, please let staff know!" self.filters = { @@ -137,9 +137,6 @@ class Filtering(Cog): async def bad_words_in_name(self, msg: Message) -> None: """Check bad words from user display name. When there is more than 3 days after last alert, send new alert.""" - if not self.name_lock: - self.name_lock = asyncio.Lock() - # Use lock to avoid race conditions async with self.name_lock: # Check does nickname have match in filters. -- cgit v1.2.3 From 4bbb5b127315727b1534c5a0e77bfdd48173847b Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sat, 30 May 2020 21:05:13 +0200 Subject: Free tag: link #how-to-get-help This creates a clickable link in the response embed. Referencing the category is no longer necessary. --- bot/resources/tags/free.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/resources/tags/free.md b/bot/resources/tags/free.md index 582cca9da..1493076c7 100644 --- a/bot/resources/tags/free.md +++ b/bot/resources/tags/free.md @@ -1,5 +1,5 @@ **We have a new help channel system!** -We recently moved to a new help channel system. You can now use any channel in the **<#691405807388196926>** category to ask your question. +Please see <#704250143020417084> for further information. -For more information, check out [our website](https://pythondiscord.com/pages/resources/guides/help-channels/). +A more detailed guide can be found on [our website](https://pythondiscord.com/pages/resources/guides/help-channels/). -- cgit v1.2.3 From 4549fa3defb7b9aba22505b438493bf03e74378d Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 30 May 2020 12:43:11 -0700 Subject: Simplify counting of staff channels and improve efficiency Simplification comes from being able to access permissions as attributes on the overwrite object. This removes the need to iterate all permissions. Efficiency comes from checking all roles within a single iteration of all channels. This also removes the need to flatten and filter the channels afterwards, which required additional iterations. --- bot/cogs/information.py | 75 +++++++++++++++++-------------------------------- 1 file changed, 26 insertions(+), 49 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index d3a2768d4..887c7c127 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -1,14 +1,13 @@ import colorsys -import functools import logging import pprint import textwrap from collections import Counter, defaultdict from string import Template -from typing import Any, List, Mapping, Optional, Union +from typing import Any, Mapping, Optional, Union -import more_itertools -from discord import Colour, Embed, Member, Message, Role, Status, utils +from discord import ChannelType, Colour, Embed, Guild, Member, Message, Role, Status, utils +from discord.abc import GuildChannel from discord.ext.commands import BucketType, Cog, Context, Paginator, command, group from discord.utils import escape_markdown @@ -29,36 +28,14 @@ class Information(Cog): self.bot = bot @staticmethod - def _get_channels_with_role_permission(ctx: Context, role: Role, perm: str, value: Optional[bool]) -> List[int]: - """Get a list of channel IDs where one of the specified roles can read.""" - channel_ids = [] + def role_can_read(channel: GuildChannel, role: Role) -> bool: + """Return True if `role` can read messages in `channel`.""" + overwrites = channel.overwrites_for(role) + return overwrites.read_messages is True - for channel in ctx.guild.channels: - overwrites = channel.overwrites_for(role) - if overwrites.is_empty(): - continue - - for _perm, _value in overwrites: - if _perm == perm and _value is value: - channel_ids.append(channel.id) - - return channel_ids - - _get_channels_where_role_can_read = functools.partialmethod( - _get_channels_with_role_permission, - perm='read_messages', - value=True - ) - - _get_channels_where_role_cannot_read = functools.partialmethod( - _get_channels_with_role_permission, - perm='read_messages', - value=False - ) - - def _get_staff_channel_count(self, ctx: Context) -> int: + def get_staff_channel_count(self, guild: Guild) -> int: """ - Get number of channels that are staff-only. + Get the number of channels that are staff-only. We need to know two things about a channel: - Does the @everyone role have explicit read deny permissions? @@ -66,22 +43,22 @@ class Information(Cog): If the answer to both of these questions is yes, it's a staff channel. """ - helpers = ctx.guild.get_role(constants.Roles.helpers) - moderators = ctx.guild.get_role(constants.Roles.moderators) - admins = ctx.guild.get_role(constants.Roles.admins) - everyone = ctx.guild.default_role - - # Let's build some lists of channels. - everyone_denied = self._get_channels_where_role_cannot_read(ctx, everyone) - staff_allowed = more_itertools.flatten([ - self._get_channels_where_role_can_read(ctx, admins), # Admins has explicit read message allow - self._get_channels_where_role_can_read(ctx, moderators), # Moderators has explicit read message allow - self._get_channels_where_role_can_read(ctx, helpers), # Helpers has explicit read message allow - ]) - - # Now we need to check which channels are both denied for @everyone and permitted for staff - staff_channels = set(cid for cid in staff_allowed if cid in everyone_denied) - return len(staff_channels) + channel_ids = set() + for channel in guild.channels: + if channel.type is ChannelType.category: + continue + + if channel in channel_ids: + continue # Only one of the roles has to have read permissions, not all + + everyone_can_read = self.role_can_read(channel, guild.default_role) + + for role in constants.STAFF_ROLES: + role_can_read = self.role_can_read(channel, guild.get_role(role)) + if role_can_read and everyone_can_read is False: + channel_ids.add(channel.id) + + return len(channel_ids) @with_role(*constants.MODERATION_ROLES) @command(name="roles") @@ -176,7 +153,7 @@ class Information(Cog): # How many staff members and staff channels do we have? staff_member_count = len(ctx.guild.get_role(constants.Roles.helpers).members) - staff_channel_count = self._get_staff_channel_count(ctx) + staff_channel_count = self.get_staff_channel_count(ctx.guild) # Because channel_counts lacks leading whitespace, it breaks the dedent if it's inserted directly by the # f-string. While this is correctly formated by Discord, it makes unit testing difficult. To keep the formatting -- cgit v1.2.3 From 8c6219cd668c814f945418000c6df896de581dc1 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 30 May 2020 12:54:22 -0700 Subject: Move counting of channels to a separate method This de-clutters the main `server_info` function and improves its readability. --- bot/cogs/information.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index 887c7c127..7c39dce5f 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -60,6 +60,18 @@ class Information(Cog): return len(channel_ids) + @staticmethod + def get_channel_type_counts(guild: Guild) -> str: + """Return the total amounts of the various types of channels in `guild`.""" + channel_counter = Counter(c.type for c in guild.channels) + channel_type_list = [] + for channel in channel_counter: + channel_type = str(channel).title() + channel_type_list.append(f"{channel_type} channels: {channel_counter[channel]}") + + channel_type_list = sorted(channel_type_list) + return "\n".join(channel_type_list).strip() + @with_role(*constants.MODERATION_ROLES) @command(name="roles") async def roles_info(self, ctx: Context) -> None: @@ -136,16 +148,7 @@ class Information(Cog): roles = len(ctx.guild.roles) member_count = ctx.guild.member_count - - # How many of each type of channel? - channel_counter = Counter(c.type for c in ctx.guild.channels) - channel_type_list = [] - for channel in channel_counter: - channel_type = str(channel).title() - channel_type_list.append(f"{channel_type} channels: {channel_counter[channel]}") - - channel_type_list = sorted(channel_type_list) - channel_counts = "\n".join(channel_type_list).strip() + channel_counts = self.get_channel_type_counts(ctx.guild) # How many of each user status? statuses = Counter(member.status for member in ctx.guild.members) -- cgit v1.2.3 From 795dea3c8030955736984cdab372595c4799f5e9 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 22:36:34 +0200 Subject: Add multichannel !purge via commands.Greedy We can now pass in as many channel mentions as we want after any !purge command - for example `!purge all 5 #python-general #python-language` --- bot/cogs/clean.py | 70 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index b5d9132cb..91e69ee89 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -1,9 +1,10 @@ import logging import random import re -from typing import Optional +from typing import Iterable, Optional from discord import Colour, Embed, Message, TextChannel, User +from discord.ext import commands from discord.ext.commands import Cog, Context, group from bot.bot import Bot @@ -41,10 +42,11 @@ class Clean(Cog): self, amount: int, ctx: Context, + channels: Iterable[TextChannel], bots_only: bool = False, user: User = None, regex: Optional[str] = None, - channel: Optional[TextChannel] = None + ) -> None: """A helper function that does the actual message cleaning.""" def predicate_bots_only(message: Message) -> bool: @@ -110,8 +112,8 @@ class Clean(Cog): predicate = None # Delete all messages # Default to using the invoking context's channel - if not channel: - channel = ctx.channel + if not channels: + channels = [ctx.channel] # Look through the history and retrieve message data messages = [] @@ -120,23 +122,24 @@ class Clean(Cog): invocation_deleted = False # To account for the invocation message, we index `amount + 1` messages. - async for message in channel.history(limit=amount + 1): + for channel in channels: + async for message in channel.history(limit=amount + 1): - # If at any point the cancel command is invoked, we should stop. - if not self.cleaning: - return + # If at any point the cancel command is invoked, we should stop. + if not self.cleaning: + return - # Always start by deleting the invocation - if not invocation_deleted: - self.mod_log.ignore(Event.message_delete, message.id) - await message.delete() - invocation_deleted = True - continue + # Always start by deleting the invocation + if not invocation_deleted: + self.mod_log.ignore(Event.message_delete, message.id) + await message.delete() + invocation_deleted = True + continue - # If the message passes predicate, let's save it. - if predicate is None or predicate(message): - message_ids.append(message.id) - messages.append(message) + # If the message passes predicate, let's save it. + if predicate is None or predicate(message): + message_ids.append(message.id) + messages.append(message) self.cleaning = False @@ -144,10 +147,11 @@ class Clean(Cog): self.mod_log.ignore(Event.message_delete, *message_ids) # Use bulk delete to actually do the cleaning. It's far faster. - await channel.purge( - limit=amount, - check=predicate - ) + for channel in channels: + await channel.purge( + limit=amount, + check=predicate + ) # Reverse the list to restore chronological order if messages: @@ -163,8 +167,12 @@ class Clean(Cog): return # Build the embed and send it + if len(channels) > 1: + target_channels = ", ".join([f"<#{channel.id}>" for channel in channels]) + else: + target_channels = f"<#{channels[0].id}>" message = ( - f"**{len(message_ids)}** messages deleted in <#{channel.id}> by **{ctx.author.name}**\n\n" + f"**{len(message_ids)}** messages deleted in {target_channels} by **{ctx.author.name}**\n\n" f"A log of the deleted messages can be found [here]({log_url})." ) @@ -189,10 +197,10 @@ class Clean(Cog): ctx: Context, user: User, amount: Optional[int] = 10, - channel: TextChannel = None + channels: commands.Greedy[TextChannel] = None ) -> None: """Delete messages posted by the provided user, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, user=user, channel=channel) + await self._clean_messages(amount, ctx, user=user, channels=channels) @clean_group.command(name="all", aliases=["everything"]) @with_role(*MODERATION_ROLES) @@ -200,10 +208,10 @@ class Clean(Cog): self, ctx: Context, amount: Optional[int] = 10, - channel: TextChannel = None + channels: commands.Greedy[TextChannel] = None ) -> None: """Delete all messages, regardless of poster, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, channel=channel) + await self._clean_messages(amount, ctx, channels=channels) @clean_group.command(name="bots", aliases=["bot"]) @with_role(*MODERATION_ROLES) @@ -211,10 +219,10 @@ class Clean(Cog): self, ctx: Context, amount: Optional[int] = 10, - channel: TextChannel = None + channels: commands.Greedy[TextChannel] = None ) -> None: """Delete all messages posted by a bot, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, bots_only=True, channel=channel) + await self._clean_messages(amount, ctx, bots_only=True, channels=channels) @clean_group.command(name="regex", aliases=["word", "expression"]) @with_role(*MODERATION_ROLES) @@ -223,10 +231,10 @@ class Clean(Cog): ctx: Context, regex: str, amount: Optional[int] = 10, - channel: TextChannel = None + channels: commands.Greedy[TextChannel] = None ) -> None: """Delete all messages that match a certain regex, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, regex=regex, channel=channel) + await self._clean_messages(amount, ctx, regex=regex, channels=channels) @clean_group.command(name="stop", aliases=["cancel", "abort"]) @with_role(*MODERATION_ROLES) -- cgit v1.2.3 From 5926f75c8d2d4b683139606bd0a39b07d28529e1 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 22:43:22 +0200 Subject: Remove a completely unacceptable newline. --- bot/cogs/clean.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index 91e69ee89..8b0b8ed05 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -46,7 +46,6 @@ class Clean(Cog): bots_only: bool = False, user: User = None, regex: Optional[str] = None, - ) -> None: """A helper function that does the actual message cleaning.""" def predicate_bots_only(message: Message) -> bool: -- cgit v1.2.3 From 1cc1b3851871dfff5690432960ade09fe8ab5794 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 23:14:21 +0200 Subject: Oops, add the return back. We do not wanna process bot messages. --- bot/cogs/verification.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 0a087cee9..ae156cf70 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -55,6 +55,7 @@ class Verification(Cog): if message.author.bot: # They're a bot, delete their message after the delay. await message.delete(delay=BOT_MESSAGE_DELETE_DELAY) + return # if a user mentions a role or guild member # alert the mods in mod-alerts channel -- cgit v1.2.3 From 4bee4f5e4e5258606da38fedc9026467dac007ae Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sun, 31 May 2020 12:24:29 +0300 Subject: Filtering: Use POSIX instead ISO format to storage alert cooldowns --- bot/cogs/filtering.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 17113d551..d1abd1193 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -5,7 +5,6 @@ from datetime import datetime, timedelta from typing import Optional, Union import discord.errors -from dateutil import parser from dateutil.relativedelta import relativedelta from discord import Colour, Member, Message, TextChannel from discord.ext.commands import Cog @@ -152,7 +151,7 @@ class Filtering(Cog): if matches: last_alert = await self.name_alerts.get(msg.author.id) if last_alert: - last_alert = parser.isoparse(last_alert) + last_alert = datetime.fromtimestamp(last_alert) if datetime.now() - timedelta(days=3) < last_alert: return @@ -170,7 +169,7 @@ class Filtering(Cog): ) # Update time when alert sent - await self.name_alerts.set(msg.author.id, datetime.now().isoformat()) + await self.name_alerts.set(msg.author.id, datetime.now().timestamp()) async def _filter_message(self, msg: Message, delta: Optional[int] = None) -> None: """Filter the input message to see if it violates any of our rules, and then respond accordingly.""" -- cgit v1.2.3 From 33a030689d8dbe68168f8e371bbcce519f24685a Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sun, 31 May 2020 12:25:39 +0300 Subject: Filtering: Rename `bad_words_in_name` to `check_is_bad_words_in_name` --- bot/cogs/filtering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index d1abd1193..909c5b78f 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -119,7 +119,7 @@ class Filtering(Cog): async def on_message(self, msg: Message) -> None: """Invoke message filter for new messages.""" await self._filter_message(msg) - await self.bad_words_in_name(msg) + await self.check_is_bad_words_in_name(msg) @Cog.listener() async def on_message_edit(self, before: Message, after: Message) -> None: @@ -134,7 +134,7 @@ class Filtering(Cog): delta = relativedelta(after.edited_at, before.edited_at).microseconds await self._filter_message(after, delta) - async def bad_words_in_name(self, msg: Message) -> None: + async def check_is_bad_words_in_name(self, msg: Message) -> None: """Check bad words from user display name. When there is more than 3 days after last alert, send new alert.""" if not self.name_lock: self.name_lock = asyncio.Lock() -- cgit v1.2.3 From 90e5b856305c1ca37cdc8e59a256dbc24dfaf5fd Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sun, 31 May 2020 12:26:32 +0300 Subject: Filtering: Add days between alerts as constant --- bot/cogs/filtering.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 909c5b78f..dbe7c6bc7 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -43,6 +43,8 @@ TOKEN_WATCHLIST_PATTERNS = [ ] WATCHLIST_PATTERNS = WORD_WATCHLIST_PATTERNS + TOKEN_WATCHLIST_PATTERNS +DAYS_BETWEEN_ALERTS = 3 + def expand_spoilers(text: str) -> str: """Return a string containing all interpretations of a spoilered message.""" @@ -152,7 +154,7 @@ class Filtering(Cog): last_alert = await self.name_alerts.get(msg.author.id) if last_alert: last_alert = datetime.fromtimestamp(last_alert) - if datetime.now() - timedelta(days=3) < last_alert: + if datetime.now() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert: return log_string = ( -- cgit v1.2.3 From cf41dc4f1964ec24242e05649f957e629c95e112 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sun, 31 May 2020 12:28:23 +0300 Subject: Filtering: On name filtering, replace Message with Embed as argument --- bot/cogs/filtering.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index dbe7c6bc7..25f5c9497 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -121,7 +121,7 @@ class Filtering(Cog): async def on_message(self, msg: Message) -> None: """Invoke message filter for new messages.""" await self._filter_message(msg) - await self.check_is_bad_words_in_name(msg) + await self.check_is_bad_words_in_name(msg.author) @Cog.listener() async def on_message_edit(self, before: Message, after: Message) -> None: @@ -136,7 +136,7 @@ class Filtering(Cog): delta = relativedelta(after.edited_at, before.edited_at).microseconds await self._filter_message(after, delta) - async def check_is_bad_words_in_name(self, msg: Message) -> None: + async def check_is_bad_words_in_name(self, member: Member) -> None: """Check bad words from user display name. When there is more than 3 days after last alert, send new alert.""" if not self.name_lock: self.name_lock = asyncio.Lock() @@ -146,20 +146,20 @@ class Filtering(Cog): # Check does nickname have match in filters. matches = [] for pattern in WATCHLIST_PATTERNS: - match = pattern.search(msg.author.display_name) + match = pattern.search(member.display_name) if match: matches.append(match) if matches: - last_alert = await self.name_alerts.get(msg.author.id) + last_alert = await self.name_alerts.get(member.id) if last_alert: last_alert = datetime.fromtimestamp(last_alert) if datetime.now() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert: return log_string = ( - f"**User:** {msg.author.mention} (`{msg.author.id}`)\n" - f"**Display Name:** {msg.author.display_name}\n" + f"**User:** {member.mention} (`{member.id}`)\n" + f"**Display Name:** {member.display_name}\n" f"**Bad Matches:** {', '.join(match.group() for match in matches)}" ) await self.mod_log.send_log_message( @@ -171,7 +171,7 @@ class Filtering(Cog): ) # Update time when alert sent - await self.name_alerts.set(msg.author.id, datetime.now().timestamp()) + await self.name_alerts.set(member.id, datetime.now().timestamp()) async def _filter_message(self, msg: Message, delta: Optional[int] = None) -> None: """Filter the input message to see if it violates any of our rules, and then respond accordingly.""" -- cgit v1.2.3 From 87872f8c681840470c55d71e798f439282fcae42 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sun, 31 May 2020 12:36:23 +0300 Subject: Filtering: Split name filtering to smaller functions --- bot/cogs/filtering.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 25f5c9497..4f1ad0986 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -2,7 +2,7 @@ import asyncio import logging import re from datetime import datetime, timedelta -from typing import Optional, Union +from typing import List, Optional, Union import discord.errors from dateutil.relativedelta import relativedelta @@ -136,6 +136,26 @@ class Filtering(Cog): delta = relativedelta(after.edited_at, before.edited_at).microseconds await self._filter_message(after, delta) + @staticmethod + def get_name_matches(name: str) -> List[re.Match]: + """Check bad words from passed string (name). Return list of matches.""" + matches = [] + for pattern in WATCHLIST_PATTERNS: + match = pattern.search(name) + if match: + matches.append(match) + return matches + + async def check_send_alert(self, member: Member) -> bool: + """When there is less than 3 days after last alert, return `False`, otherwise `True`.""" + last_alert = await self.name_alerts.get(member.id) + if last_alert: + last_alert = datetime.fromtimestamp(last_alert) + if datetime.now() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert: + return False + + return True + async def check_is_bad_words_in_name(self, member: Member) -> None: """Check bad words from user display name. When there is more than 3 days after last alert, send new alert.""" if not self.name_lock: @@ -144,18 +164,11 @@ class Filtering(Cog): # Use lock to avoid race conditions async with self.name_lock: # Check does nickname have match in filters. - matches = [] - for pattern in WATCHLIST_PATTERNS: - match = pattern.search(member.display_name) - if match: - matches.append(match) + matches = self.get_name_matches(member.display_name) if matches: - last_alert = await self.name_alerts.get(member.id) - if last_alert: - last_alert = datetime.fromtimestamp(last_alert) - if datetime.now() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert: - return + if not self.check_send_alert(member): + return log_string = ( f"**User:** {member.mention} (`{member.id}`)\n" -- cgit v1.2.3 From 5b7df0ea02df485c507d602868bce215b4290626 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sun, 31 May 2020 12:37:30 +0300 Subject: Filtering: Fix docstring Co-authored-by: Mark --- bot/cogs/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index eb587d781..737317d46 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -157,7 +157,7 @@ class Filtering(Cog): return True async def check_is_bad_words_in_name(self, member: Member) -> None: - """Check bad words from user display name. When there is more than 3 days after last alert, send new alert.""" + """Send a mod alert every 3 days if a username still matches a watchlist pattern.""" # Use lock to avoid race conditions async with self.name_lock: # Check does nickname have match in filters. -- cgit v1.2.3 From d7123487230d70b855c84fb5d99ec45f6bee6859 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 31 May 2020 18:18:18 +0200 Subject: Better channel mentions Co-authored-by: Mark --- bot/cogs/clean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index 8b0b8ed05..571a5ced8 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -167,7 +167,7 @@ class Clean(Cog): # Build the embed and send it if len(channels) > 1: - target_channels = ", ".join([f"<#{channel.id}>" for channel in channels]) + target_channels = ", ".join(channel.mention for channel in channels) else: target_channels = f"<#{channels[0].id}>" message = ( -- cgit v1.2.3 From d637053eb19a6bf33e765b25b3dff9963d7b7735 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 31 May 2020 18:19:36 +0200 Subject: Remove unnecessary conditional. Thanks @MarkKoz! --- bot/cogs/clean.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index 571a5ced8..02216a4af 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -166,10 +166,8 @@ class Clean(Cog): return # Build the embed and send it - if len(channels) > 1: - target_channels = ", ".join(channel.mention for channel in channels) - else: - target_channels = f"<#{channels[0].id}>" + target_channels = ", ".join(channel.mention for channel in channels) + message = ( f"**{len(message_ids)}** messages deleted in {target_channels} by **{ctx.author.name}**\n\n" f"A log of the deleted messages can be found [here]({log_url})." -- cgit v1.2.3 From 0737b1a63ca359e88ef580143e8e4e6a879c482e Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 31 May 2020 18:34:36 +0200 Subject: Add a mod_log.ignore_all context manager. This new context manager makes it easier to make the mod_log ignore actions like message deletions. The only existing method is the `ignore()` method, which requires that you pass all the messages you want to ignore into it. This one just ignores everything inside its scope. This isn't the DRYest approach, but it's low-cost and improves the readability of clean.py quite a bit. Ideally we should go through and give modlog a proper cleanup, because it's kinda ugly right now. --- bot/cogs/moderation/modlog.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index 9d28030d9..b3ae8e215 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -3,6 +3,7 @@ import difflib import itertools import logging import typing as t +from contextlib import contextmanager from datetime import datetime from itertools import zip_longest @@ -40,6 +41,7 @@ class ModLog(Cog, name="ModLog"): def __init__(self, bot: Bot): self.bot = bot self._ignored = {event: [] for event in Event} + self._ignore_all = False self._cached_deletes = [] self._cached_edits = [] @@ -81,6 +83,15 @@ class ModLog(Cog, name="ModLog"): if item not in self._ignored[event]: self._ignored[event].append(item) + @contextmanager + def ignore_all(self) -> None: + """Ignore all events while inside this context scope.""" + self._ignore_all = True + try: + yield + finally: + self._ignore_all = False + async def send_log_message( self, icon_url: t.Optional[str], @@ -191,6 +202,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.guild_channel_update].remove(before.id) return + if self._ignore_all: + return + # Two channel updates are sent for a single edit: 1 for topic and 1 for category change. # TODO: remove once support is added for ignoring multiple occurrences for the same channel. help_categories = (Categories.help_available, Categories.help_dormant, Categories.help_in_use) @@ -386,6 +400,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_ban].remove(member.id) return + if self._ignore_all: + return + await self.send_log_message( Icons.user_ban, Colours.soft_red, "User banned", f"{member} (`{member.id}`)", @@ -426,6 +443,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_remove].remove(member.id) return + if self._ignore_all: + return + member_str = escape_markdown(str(member)) await self.send_log_message( Icons.sign_out, Colours.soft_red, @@ -444,6 +464,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_unban].remove(member.id) return + if self._ignore_all: + return + member_str = escape_markdown(str(member)) await self.send_log_message( Icons.user_unban, Colour.blurple(), @@ -462,6 +485,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_update].remove(before.id) return + if self._ignore_all: + return + diff = DeepDiff(before, after) changes = [] done = [] @@ -564,6 +590,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.message_delete].remove(message.id) return + if self._ignore_all: + return + if author.bot: return @@ -623,6 +652,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.message_delete].remove(event.message_id) return + if self._ignore_all: + return + channel = self.bot.get_channel(event.channel_id) if channel.category: @@ -797,6 +829,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.voice_state_update].remove(member.id) return + if self._ignore_all: + return + # Exclude all channel attributes except the name. diff = DeepDiff( before, -- cgit v1.2.3 From f344dd8a72024e05577a5aeba25e2f98501417af Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 31 May 2020 18:37:56 +0200 Subject: Fix a bug with invocation deletion. This command was written to support only a single channel, and with the move to multi-channel purges, we need to rethink the way the invocation deletion happens. We may be invoking this command from a completely different channel, so we can't necessarily look inside the channels we're targeting for the invocation. So, we're solving this by just deleting the invocation by using ctx.message. We do this before we start iterating message history, and then we only need to iterate the number of messages that was passed into the command. A much cleaner approach, which solves the bug reported and identified by @MarkKoz. --- bot/cogs/clean.py | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index 02216a4af..892c638b8 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -10,8 +10,7 @@ from discord.ext.commands import Cog, Context, group from bot.bot import Bot from bot.cogs.moderation import ModLog from bot.constants import ( - Channels, CleanMessages, Colours, Event, - Icons, MODERATION_ROLES, NEGATIVE_REPLIES + Channels, CleanMessages, Colours, Icons, MODERATION_ROLES, NEGATIVE_REPLIES ) from bot.decorators import with_role @@ -114,27 +113,23 @@ class Clean(Cog): if not channels: channels = [ctx.channel] + # Delete the invocation first + with self.mod_log.ignore_all(): + await ctx.message.delete() + # Look through the history and retrieve message data + # This is only done so we can create a log to upload. messages = [] message_ids = [] self.cleaning = True - invocation_deleted = False - # To account for the invocation message, we index `amount + 1` messages. for channel in channels: - async for message in channel.history(limit=amount + 1): + async for message in channel.history(limit=amount): # If at any point the cancel command is invoked, we should stop. if not self.cleaning: return - # Always start by deleting the invocation - if not invocation_deleted: - self.mod_log.ignore(Event.message_delete, message.id) - await message.delete() - invocation_deleted = True - continue - # If the message passes predicate, let's save it. if predicate is None or predicate(message): message_ids.append(message.id) @@ -142,15 +137,13 @@ class Clean(Cog): self.cleaning = False - # We should ignore the ID's we stored, so we don't get mod-log spam. - self.mod_log.ignore(Event.message_delete, *message_ids) - - # Use bulk delete to actually do the cleaning. It's far faster. - for channel in channels: - await channel.purge( - limit=amount, - check=predicate - ) + # Now let's delete the actual messages with purge. + with self.mod_log.ignore_all(): + for channel in channels: + await channel.purge( + limit=amount, + check=predicate + ) # Reverse the list to restore chronological order if messages: -- cgit v1.2.3 From b225791c917b582fcfbed0aee30b2cf7d3fd9ac4 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 31 May 2020 18:50:16 +0200 Subject: Fix a bad check in get_staff_channel_count. This also changes a few aesthetic problems pointed out in review by @MarkKoz and @kwzrd. --- bot/cogs/information.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index 7c39dce5f..f0bd1afdb 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -48,15 +48,13 @@ class Information(Cog): if channel.type is ChannelType.category: continue - if channel in channel_ids: - continue # Only one of the roles has to have read permissions, not all - everyone_can_read = self.role_can_read(channel, guild.default_role) for role in constants.STAFF_ROLES: role_can_read = self.role_can_read(channel, guild.get_role(role)) - if role_can_read and everyone_can_read is False: + if role_can_read and not everyone_can_read: channel_ids.add(channel.id) + break return len(channel_ids) @@ -65,12 +63,12 @@ class Information(Cog): """Return the total amounts of the various types of channels in `guild`.""" channel_counter = Counter(c.type for c in guild.channels) channel_type_list = [] - for channel in channel_counter: + for channel, count in channel_counter.items(): channel_type = str(channel).title() - channel_type_list.append(f"{channel_type} channels: {channel_counter[channel]}") + channel_type_list.append(f"{channel_type} channels: {count}") channel_type_list = sorted(channel_type_list) - return "\n".join(channel_type_list).strip() + return "\n".join(channel_type_list) @with_role(*constants.MODERATION_ROLES) @command(name="roles") -- cgit v1.2.3 From 876b4846f612fe0011cc2e0b498b4df9e54d74cb Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 31 May 2020 19:17:07 +0200 Subject: Add support for bool values in RedisCache We're gonna need this for the help channel handling, and it seems like a reasonable type to support anyway. It requires a tiny bit of special handling, but nothing outrageous. --- bot/utils/redis_cache.py | 14 ++++++++++++-- tests/bot/utils/test_redis_cache.py | 4 +++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index de80cee84..2926e7a89 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import logging +from distutils.util import strtobool from functools import partialmethod from typing import Any, Dict, ItemsView, Optional, Tuple, Union @@ -11,7 +12,7 @@ log = logging.getLogger(__name__) # Type aliases RedisKeyType = Union[str, int] -RedisValueType = Union[str, int, float] +RedisValueType = Union[str, int, float, bool] RedisKeyOrValue = Union[RedisKeyType, RedisValueType] # Prefix tuples @@ -20,6 +21,7 @@ _VALUE_PREFIXES = ( ("f|", float), ("i|", int), ("s|", str), + ("b|", bool), ) _KEY_PREFIXES = ( ("i|", int), @@ -117,7 +119,8 @@ class RedisCache: def _to_typestring(key_or_value: RedisKeyOrValue, prefixes: _PrefixTuple) -> str: """Turn a valid Redis type into a typestring.""" for prefix, _type in prefixes: - if isinstance(key_or_value, _type): + # isinstance is a bad idea here, because isintance(False, int) == True. + if type(key_or_value) is _type: return f"{prefix}{key_or_value}" raise TypeError(f"RedisCache._to_typestring only supports the following: {prefixes}.") @@ -131,6 +134,13 @@ class RedisCache: # Now we convert our unicode string back into the type it originally was. for prefix, _type in prefixes: if key_or_value.startswith(prefix): + + # For booleans, we need special handling because bool("False") is True. + if prefix == "b|": + value = key_or_value[len(prefix):] + return bool(strtobool(value)) + + # Otherwise we can just convert normally. return _type(key_or_value[len(prefix):]) raise TypeError(f"RedisCache._from_typestring only supports the following: {prefixes}.") diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 8c1a40640..62c411681 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -59,7 +59,9 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): test_cases = ( ('favorite_fruit', 'melon'), ('favorite_number', 86), - ('favorite_fraction', 86.54) + ('favorite_fraction', 86.54), + ('favorite_boolean', False), + ('other_boolean', True), ) # Test that we can get and set different types. -- cgit v1.2.3 From 72adad95e06632a61ba7773289938ca69e5874aa Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sun, 31 May 2020 21:47:03 +0300 Subject: Filtering: Small fixes - Use UTC from timestamp - Rename name bad words checking function --- bot/cogs/filtering.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 737317d46..baa2e5529 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -121,7 +121,7 @@ class Filtering(Cog): async def on_message(self, msg: Message) -> None: """Invoke message filter for new messages.""" await self._filter_message(msg) - await self.check_is_bad_words_in_name(msg.author) + await self.check_bad_words_in_name(msg.author) @Cog.listener() async def on_message_edit(self, before: Message, after: Message) -> None: @@ -150,13 +150,13 @@ class Filtering(Cog): """When there is less than 3 days after last alert, return `False`, otherwise `True`.""" last_alert = await self.name_alerts.get(member.id) if last_alert: - last_alert = datetime.fromtimestamp(last_alert) - if datetime.now() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert: + last_alert = datetime.utcfromtimestamp(last_alert) + if datetime.utcnow() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert: return False return True - async def check_is_bad_words_in_name(self, member: Member) -> None: + async def check_bad_words_in_name(self, member: Member) -> None: """Send a mod alert every 3 days if a username still matches a watchlist pattern.""" # Use lock to avoid race conditions async with self.name_lock: @@ -181,7 +181,7 @@ class Filtering(Cog): ) # Update time when alert sent - await self.name_alerts.set(member.id, datetime.now().timestamp()) + await self.name_alerts.set(member.id, datetime.utcnow().timestamp()) async def _filter_message(self, msg: Message, delta: Optional[int] = None) -> None: """Filter the input message to see if it violates any of our rules, and then respond accordingly.""" -- cgit v1.2.3 From 860f4d4306fb846bf36cbcaedf8e1ee042550f06 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 31 May 2020 12:14:04 -0700 Subject: Fix missing await in bad nickname filter --- bot/cogs/filtering.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index baa2e5529..5c3d01e3a 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -163,25 +163,24 @@ class Filtering(Cog): # Check does nickname have match in filters. matches = self.get_name_matches(member.display_name) - if matches: - if not self.check_send_alert(member): - return - - log_string = ( - f"**User:** {member.mention} (`{member.id}`)\n" - f"**Display Name:** {member.display_name}\n" - f"**Bad Matches:** {', '.join(match.group() for match in matches)}" - ) - await self.mod_log.send_log_message( - icon_url=Icons.token_removed, - colour=Colours.soft_red, - title="Username filtering alert", - text=log_string, - channel_id=Channels.mod_alerts - ) + if not matches or not await self.check_send_alert(member): + return + + log_string = ( + f"**User:** {member.mention} (`{member.id}`)\n" + f"**Display Name:** {member.display_name}\n" + f"**Bad Matches:** {', '.join(match.group() for match in matches)}" + ) + await self.mod_log.send_log_message( + icon_url=Icons.token_removed, + colour=Colours.soft_red, + title="Username filtering alert", + text=log_string, + channel_id=Channels.mod_alerts + ) - # Update time when alert sent - await self.name_alerts.set(member.id, datetime.utcnow().timestamp()) + # Update time when alert sent + await self.name_alerts.set(member.id, datetime.utcnow().timestamp()) async def _filter_message(self, msg: Message, delta: Optional[int] = None) -> None: """Filter the input message to see if it violates any of our rules, and then respond accordingly.""" -- cgit v1.2.3 From 0fb6c2dad3787054c92cc732af8b52799f20e06f Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 31 May 2020 12:24:08 -0700 Subject: Add logging for the bad nickname filter --- bot/cogs/filtering.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 5c3d01e3a..caf204561 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -152,6 +152,7 @@ class Filtering(Cog): if last_alert: last_alert = datetime.utcfromtimestamp(last_alert) if datetime.utcnow() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert: + log.trace(f"Last alert was too recent for {member}'s nickname.") return False return True @@ -166,6 +167,7 @@ class Filtering(Cog): if not matches or not await self.check_send_alert(member): return + log.info(f"Sending bad nickname alert for '{member.display_name}' ({member.id}).") log_string = ( f"**User:** {member.mention} (`{member.id}`)\n" f"**Display Name:** {member.display_name}\n" -- cgit v1.2.3 From ca1a8c55d68f15d910f21d568339e6555e4b7e54 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 31 May 2020 13:43:42 -0700 Subject: Remove redis namespace collision prevention When cogs reload, it would consider their namespace as a conflict with the original namespace. This feature will be removed as a fix until we come up with a better solution. --- bot/utils/redis_cache.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index de80cee84..354e987b9 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -100,16 +100,7 @@ class RedisCache: def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" - # We need a unique namespace, to prevent collisions. This loop - # will try appending underscores to the end of the namespace until - # it finds one that is unique. - # - # For example, if `john` and `john_` are both taken, the namespace will - # be `john__` at the end of this loop. - while namespace in self._namespaces: - namespace += "_" - - log.trace(f"RedisCache setting namespace to {self._namespace}") + log.trace(f"RedisCache setting namespace to {namespace}") self._namespaces.append(namespace) self._namespace = namespace -- cgit v1.2.3 From ebbaa6274cfc278c772593b193356aa8bf066de4 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 31 May 2020 14:17:20 -0700 Subject: Remove redis namespace collision test --- tests/bot/utils/test_redis_cache.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 8c1a40640..e5d6e4078 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -44,16 +44,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): with self.assertRaises(RuntimeError): await bad_cache.set("test", "me_up_deadman") - def test_namespace_collision(self): - """Test that we prevent colliding namespaces.""" - bob_cache_1 = RedisCache() - bob_cache_1._set_namespace("BobRoss") - self.assertEqual(bob_cache_1._namespace, "BobRoss") - - bob_cache_2 = RedisCache() - bob_cache_2._set_namespace("BobRoss") - self.assertEqual(bob_cache_2._namespace, "BobRoss_") - async def test_set_get_item(self): """Test that users can set and get items from the RedisDict.""" test_cases = ( -- cgit v1.2.3 From 3139991c3dcf4ec981a49aefa3d3cd75eed93fd8 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 31 May 2020 23:33:05 +0200 Subject: Revert "Add a mod_log.ignore_all context manager." This reverts commit 0737b1a6 This isn't gonna work, because async is a thing. --- bot/cogs/moderation/modlog.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index b3ae8e215..9d28030d9 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -3,7 +3,6 @@ import difflib import itertools import logging import typing as t -from contextlib import contextmanager from datetime import datetime from itertools import zip_longest @@ -41,7 +40,6 @@ class ModLog(Cog, name="ModLog"): def __init__(self, bot: Bot): self.bot = bot self._ignored = {event: [] for event in Event} - self._ignore_all = False self._cached_deletes = [] self._cached_edits = [] @@ -83,15 +81,6 @@ class ModLog(Cog, name="ModLog"): if item not in self._ignored[event]: self._ignored[event].append(item) - @contextmanager - def ignore_all(self) -> None: - """Ignore all events while inside this context scope.""" - self._ignore_all = True - try: - yield - finally: - self._ignore_all = False - async def send_log_message( self, icon_url: t.Optional[str], @@ -202,9 +191,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.guild_channel_update].remove(before.id) return - if self._ignore_all: - return - # Two channel updates are sent for a single edit: 1 for topic and 1 for category change. # TODO: remove once support is added for ignoring multiple occurrences for the same channel. help_categories = (Categories.help_available, Categories.help_dormant, Categories.help_in_use) @@ -400,9 +386,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_ban].remove(member.id) return - if self._ignore_all: - return - await self.send_log_message( Icons.user_ban, Colours.soft_red, "User banned", f"{member} (`{member.id}`)", @@ -443,9 +426,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_remove].remove(member.id) return - if self._ignore_all: - return - member_str = escape_markdown(str(member)) await self.send_log_message( Icons.sign_out, Colours.soft_red, @@ -464,9 +444,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_unban].remove(member.id) return - if self._ignore_all: - return - member_str = escape_markdown(str(member)) await self.send_log_message( Icons.user_unban, Colour.blurple(), @@ -485,9 +462,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_update].remove(before.id) return - if self._ignore_all: - return - diff = DeepDiff(before, after) changes = [] done = [] @@ -590,9 +564,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.message_delete].remove(message.id) return - if self._ignore_all: - return - if author.bot: return @@ -652,9 +623,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.message_delete].remove(event.message_id) return - if self._ignore_all: - return - channel = self.bot.get_channel(event.channel_id) if channel.category: @@ -829,9 +797,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.voice_state_update].remove(member.id) return - if self._ignore_all: - return - # Exclude all channel attributes except the name. diff = DeepDiff( before, -- cgit v1.2.3 From 345fda6b88fef50e9bc47298085a10d8acb4fdff Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 31 May 2020 23:36:28 +0200 Subject: Revert message ignore approach. We're removing the context manager due to async concerns, so we'll go back to the old approach again of ignoring specific messages and iterating history. --- bot/cogs/clean.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index 892c638b8..b164cf232 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -10,7 +10,7 @@ from discord.ext.commands import Cog, Context, group from bot.bot import Bot from bot.cogs.moderation import ModLog from bot.constants import ( - Channels, CleanMessages, Colours, Icons, MODERATION_ROLES, NEGATIVE_REPLIES + Channels, CleanMessages, Colours, Event, Icons, MODERATION_ROLES, NEGATIVE_REPLIES ) from bot.decorators import with_role @@ -114,11 +114,10 @@ class Clean(Cog): channels = [ctx.channel] # Delete the invocation first - with self.mod_log.ignore_all(): - await ctx.message.delete() + self.mod_log.ignore(Event.message_delete, ctx.message.id) + await ctx.message.delete() # Look through the history and retrieve message data - # This is only done so we can create a log to upload. messages = [] message_ids = [] self.cleaning = True @@ -138,12 +137,12 @@ class Clean(Cog): self.cleaning = False # Now let's delete the actual messages with purge. - with self.mod_log.ignore_all(): - for channel in channels: - await channel.purge( - limit=amount, - check=predicate - ) + self.mod_log.ignore(Event.message_delete, *message_ids) + for channel in channels: + await channel.purge( + limit=amount, + check=predicate + ) # Reverse the list to restore chronological order if messages: -- cgit v1.2.3 From 196ce8a828a0fed7450cad1ee0bba25ef608214a Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 31 May 2020 15:27:53 -0700 Subject: Use the messages returned by `purge` to upload message logs This ensures that only what was actually deleted will be uploaded. I managed to get a 400 response from our API when purging twice in quick succession. Searching the history manually for these messages is unreliable cause of some sort of race condition. --- bot/cogs/clean.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index b164cf232..368d91c85 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -117,11 +117,11 @@ class Clean(Cog): self.mod_log.ignore(Event.message_delete, ctx.message.id) await ctx.message.delete() - # Look through the history and retrieve message data messages = [] message_ids = [] self.cleaning = True + # Find the IDs of the messages to delete. IDs are needed in order to ignore mod log events. for channel in channels: async for message in channel.history(limit=amount): @@ -132,21 +132,17 @@ class Clean(Cog): # If the message passes predicate, let's save it. if predicate is None or predicate(message): message_ids.append(message.id) - messages.append(message) self.cleaning = False # Now let's delete the actual messages with purge. self.mod_log.ignore(Event.message_delete, *message_ids) for channel in channels: - await channel.purge( - limit=amount, - check=predicate - ) + messages += await channel.purge(limit=amount, check=predicate) # Reverse the list to restore chronological order if messages: - messages = list(reversed(messages)) + messages = reversed(messages) log_url = await self.mod_log.upload_log(messages, ctx.author.id) else: # Can't build an embed, nothing to clean! -- cgit v1.2.3 From eac3be892a31a508f966fd73f1802086d83ed954 Mon Sep 17 00:00:00 2001 From: ItsDrike Date: Mon, 1 Jun 2020 19:09:22 +0200 Subject: Use Scheduler instead of asyncio.sleep on silence cog `asyncio.sleep` doesn't provide us with the ability to stop that timer, while in most of the cases, this is fine, there is a possibility that channel will be unsilenced manually and silenced again, but this sleep from the first silence will cancel the second (new) silence. This will replace this `asyncio.sleep` with Scheduler which provides the ability to cancel the unsilencing task when aborted manually. That means we also have the ability to send a response if the channel is not silenced and someone tries to unsilence it. --- bot/cogs/moderation/silence.py | 66 +++++++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index 25febfa51..a2fd39906 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -1,19 +1,54 @@ import asyncio +import datetime import logging +from collections import namedtuple from contextlib import suppress from typing import Optional -from discord import TextChannel -from discord.ext import commands, tasks -from discord.ext.commands import Context - from bot.bot import Bot -from bot.constants import Channels, Emojis, Guild, MODERATION_ROLES, Roles +from bot.constants import MODERATION_ROLES, Channels, Emojis, Guild, Roles from bot.converters import HushDurationConverter +from bot.utils import time from bot.utils.checks import with_role_check +from bot.utils.scheduling import Scheduler +from discord import TextChannel +from discord.ext import commands, tasks +from discord.ext.commands import Context log = logging.getLogger(__name__) +SilencedChannel = namedtuple( + "SilencedChannel", ("id", "ctx", "silence", "stop") +) + + +class UnsilenceScheduler(Scheduler): + """Scheduler for unsilencing channels""" + + def __init__(self, bot: Bot): + super().__init__() + + self.bot = bot + + async def schedule_unsilence(self, channel: SilencedChannel) -> None: + """Schedule expiration for silenced channels""" + await self.bot.wait_until_guild_available() + log.debug("Scheduling unsilencer") + self.schedule_task(channel.id, channel) + + async def _scheduled_task(self, channel: SilencedChannel) -> None: + """ + Removes expired silenced channel from `silence.muted_channels` + and calls `silence.unsilence` to unsilence the channel + after the silence expires + """ + await time.wait_until(channel.stop) + log.info("Unsilencing channel after set delay.") + + # Because `silence.unsilence` explicitly cancels this scheduled task, it is shielded + # to avoid prematurely cancelling itself. + await asyncio.shield(channel.ctx.invoke(channel.silence.unsilence)) + class SilenceNotifier(tasks.Loop): """Loop notifier for posting notices to `alert_channel` containing added channels.""" @@ -61,6 +96,7 @@ class Silence(commands.Cog): self.muted_channels = set() self._get_instance_vars_task = self.bot.loop.create_task(self._get_instance_vars()) self._get_instance_vars_event = asyncio.Event() + self.scheduler = UnsilenceScheduler(bot) async def _get_instance_vars(self) -> None: """Get instance variables after they're available to get from the guild.""" @@ -90,9 +126,15 @@ class Silence(commands.Cog): return await ctx.send(f"{Emojis.check_mark} silenced current channel for {duration} minute(s).") - await asyncio.sleep(duration*60) - log.info("Unsilencing channel after set delay.") - await ctx.invoke(self.unsilence) + + channel = SilencedChannel( + id=ctx.channel.id, + ctx=ctx, + silence=self, + stop=datetime.datetime.now() + datetime.timedelta(minutes=duration), + ) + + await self.scheduler.schedule_unsilence(channel) @commands.command(aliases=("unhush",)) async def unsilence(self, ctx: Context) -> None: @@ -103,8 +145,11 @@ class Silence(commands.Cog): """ await self._get_instance_vars_event.wait() log.debug(f"Unsilencing channel #{ctx.channel} from {ctx.author}'s command.") - if await self._unsilence(ctx.channel): - await ctx.send(f"{Emojis.check_mark} unsilenced current channel.") + if not await self._unsilence(ctx.channel): + await ctx.send(f"{Emojis.cross_mark} current channel is not silenced.") + return + + await ctx.send(f"{Emojis.check_mark} unsilenced current channel.") async def _silence(self, channel: TextChannel, persistent: bool, duration: Optional[int]) -> bool: """ @@ -141,6 +186,7 @@ class Silence(commands.Cog): await channel.set_permissions(self._verified_role, **dict(current_overwrite, send_messages=None)) log.info(f"Unsilenced channel #{channel} ({channel.id}).") self.notifier.remove_channel(channel) + self.scheduler.cancel_task(channel.id) self.muted_channels.discard(channel) return True log.info(f"Tried to unsilence channel #{channel} ({channel.id}) but the channel was not silenced.") -- cgit v1.2.3 From 9b0df33c90326f62d699afc70c13d3d375affeab Mon Sep 17 00:00:00 2001 From: ItsDrike Date: Mon, 1 Jun 2020 23:08:32 +0200 Subject: Fix Formatting/Styling --- bot/cogs/moderation/silence.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index a2fd39906..448f17966 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -6,11 +6,12 @@ from contextlib import suppress from typing import Optional from bot.bot import Bot -from bot.constants import MODERATION_ROLES, Channels, Emojis, Guild, Roles +from bot.constants import Channels, Emojis, Guild, MODERATION_ROLES, Roles from bot.converters import HushDurationConverter from bot.utils import time from bot.utils.checks import with_role_check from bot.utils.scheduling import Scheduler + from discord import TextChannel from discord.ext import commands, tasks from discord.ext.commands import Context @@ -23,7 +24,7 @@ SilencedChannel = namedtuple( class UnsilenceScheduler(Scheduler): - """Scheduler for unsilencing channels""" + """Scheduler for unsilencing channels.""" def __init__(self, bot: Bot): super().__init__() @@ -31,17 +32,13 @@ class UnsilenceScheduler(Scheduler): self.bot = bot async def schedule_unsilence(self, channel: SilencedChannel) -> None: - """Schedule expiration for silenced channels""" + """Schedule expiration for silenced channels.""" await self.bot.wait_until_guild_available() log.debug("Scheduling unsilencer") self.schedule_task(channel.id, channel) async def _scheduled_task(self, channel: SilencedChannel) -> None: - """ - Removes expired silenced channel from `silence.muted_channels` - and calls `silence.unsilence` to unsilence the channel - after the silence expires - """ + """Calls `silence.unsilence` on expired silenced channel to unsilence it.""" await time.wait_until(channel.stop) log.info("Unsilencing channel after set delay.") -- cgit v1.2.3 From 0a7b64df552d394e9d1f38fb167ec93334b9bead Mon Sep 17 00:00:00 2001 From: ItsDrike Date: Mon, 1 Jun 2020 23:24:07 +0200 Subject: Optimize Imports --- bot/cogs/moderation/silence.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index 448f17966..5dfa9cc8a 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -5,6 +5,10 @@ from collections import namedtuple from contextlib import suppress from typing import Optional +from discord import TextChannel +from discord.ext import commands, tasks +from discord.ext.commands import Context + from bot.bot import Bot from bot.constants import Channels, Emojis, Guild, MODERATION_ROLES, Roles from bot.converters import HushDurationConverter @@ -12,10 +16,6 @@ from bot.utils import time from bot.utils.checks import with_role_check from bot.utils.scheduling import Scheduler -from discord import TextChannel -from discord.ext import commands, tasks -from discord.ext.commands import Context - log = logging.getLogger(__name__) SilencedChannel = namedtuple( -- cgit v1.2.3 From 629817eaa87d869cc7857d5bde48d53cce6bcdc0 Mon Sep 17 00:00:00 2001 From: Rasmus Moorats Date: Tue, 2 Jun 2020 17:41:13 +0300 Subject: add modmail tag --- bot/resources/tags/modmail.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 bot/resources/tags/modmail.md diff --git a/bot/resources/tags/modmail.md b/bot/resources/tags/modmail.md new file mode 100644 index 000000000..7545419ee --- /dev/null +++ b/bot/resources/tags/modmail.md @@ -0,0 +1,9 @@ +**Contacting the moderation team via ModMail** + +<@!683001325440860340> is a bot that will relay your messages to our moderation team, so that you can start a conversation with the moderation team. Your messages will be relayed to the entire moderator team, who will be able to respond to you via the bot. + +It supports attachments, codeblocks, and reactions. As communication happens over direct messages, the conversation will stay between you and the mod team. + +**To use it, simply send a direct message to the bot.** + +Should there be an urgent and immediate need for a moderator or admin to look at a channel, feel free to ping the <@&267629731250176001> or <@&267628507062992896> role instead. -- cgit v1.2.3 From 9b3ab7df5ae1ecf95705f2fab7d99fdb36eb98ea Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 2 Jun 2020 19:22:49 -0700 Subject: Token remover: remove the `delete_message` function It's redundant; there's no benefit here in abstracting two lines of code into a function. --- bot/cogs/token_remover.py | 9 ++------- tests/bot/cogs/test_token_remover.py | 19 +++++++------------ 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index 46329e207..d55e079e9 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -79,7 +79,8 @@ class TokenRemover(Cog): async def take_action(self, msg: Message, found_token: Token) -> None: """Remove the `msg` containing the `found_token` and send a mod log message.""" self.mod_log.ignore(Event.message_delete, msg.id) - await self.delete_message(msg) + await msg.delete() + await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention)) log_message = self.format_log_message(msg, found_token) log.debug(log_message) @@ -96,12 +97,6 @@ class TokenRemover(Cog): self.bot.stats.incr("tokens.removed_tokens") - @staticmethod - async def delete_message(msg: Message) -> None: - """Remove a `msg` containing a token and send an explanatory message in the same channel.""" - await msg.delete() - await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention)) - @staticmethod def format_log_message(msg: Message, token: Token) -> str: """Return the log message to send for `token` being censored in `msg`.""" diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 9b4b04ecd..a10124d2d 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -229,15 +229,6 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): results = [match[0] for match in results] self.assertCountEqual((token_1, token_2), results) - async def test_delete_message(self): - """The message should be deleted, and a message should be sent to the same channel.""" - await TokenRemover.delete_message(self.msg) - - self.msg.delete.assert_called_once_with() - self.msg.channel.send.assert_called_once_with( - token_remover.DELETION_MESSAGE_TEMPLATE.format(mention=self.msg.author.mention) - ) - @autospec("bot.cogs.token_remover", "LOG_MESSAGE") def test_format_log_message(self, log_message): """Should correctly format the log message with info from the message and token.""" @@ -258,8 +249,8 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): @mock.patch.object(TokenRemover, "mod_log", new_callable=mock.PropertyMock) @autospec("bot.cogs.token_remover", "log") - @autospec(TokenRemover, "delete_message", "format_log_message") - async def test_take_action(self, delete_message, format_log_message, logger, mod_log_property): + @autospec(TokenRemover, "format_log_message") + async def test_take_action(self, format_log_message, logger, mod_log_property): """Should delete the message and send a mod log.""" cog = TokenRemover(self.bot) mod_log = mock.create_autospec(ModLog, spec_set=True, instance=True) @@ -271,7 +262,11 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): await cog.take_action(self.msg, token) - delete_message.assert_awaited_once_with(self.msg) + self.msg.delete.assert_called_once_with() + self.msg.channel.send.assert_called_once_with( + token_remover.DELETION_MESSAGE_TEMPLATE.format(mention=self.msg.author.mention) + ) + format_log_message.assert_called_once_with(self.msg, token) logger.debug.assert_called_with(log_msg) self.bot.stats.incr.assert_called_once_with("tokens.removed_tokens") -- cgit v1.2.3 From 469692d53a4ed74500a5806273e9c778a97afae8 Mon Sep 17 00:00:00 2001 From: ItsDrike Date: Thu, 4 Jun 2020 22:50:36 +0200 Subject: Use Scheduler inside the cog - There shouldn't be another class only for Scheduler instead, we can implement it directly into Silence class --- bot/cogs/moderation/silence.py | 50 +++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index 5dfa9cc8a..398a70e51 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -19,34 +19,10 @@ from bot.utils.scheduling import Scheduler log = logging.getLogger(__name__) SilencedChannel = namedtuple( - "SilencedChannel", ("id", "ctx", "silence", "stop") + "SilencedChannel", ("id", "ctx", "stop") ) -class UnsilenceScheduler(Scheduler): - """Scheduler for unsilencing channels.""" - - def __init__(self, bot: Bot): - super().__init__() - - self.bot = bot - - async def schedule_unsilence(self, channel: SilencedChannel) -> None: - """Schedule expiration for silenced channels.""" - await self.bot.wait_until_guild_available() - log.debug("Scheduling unsilencer") - self.schedule_task(channel.id, channel) - - async def _scheduled_task(self, channel: SilencedChannel) -> None: - """Calls `silence.unsilence` on expired silenced channel to unsilence it.""" - await time.wait_until(channel.stop) - log.info("Unsilencing channel after set delay.") - - # Because `silence.unsilence` explicitly cancels this scheduled task, it is shielded - # to avoid prematurely cancelling itself. - await asyncio.shield(channel.ctx.invoke(channel.silence.unsilence)) - - class SilenceNotifier(tasks.Loop): """Loop notifier for posting notices to `alert_channel` containing added channels.""" @@ -85,7 +61,7 @@ class SilenceNotifier(tasks.Loop): await self._alert_channel.send(f"<@&{Roles.moderators}> currently silenced channels: {channels_text}") -class Silence(commands.Cog): +class Silence(Scheduler, commands.Cog): """Commands for stopping channel messages for `verified` role in a channel.""" def __init__(self, bot: Bot): @@ -93,7 +69,22 @@ class Silence(commands.Cog): self.muted_channels = set() self._get_instance_vars_task = self.bot.loop.create_task(self._get_instance_vars()) self._get_instance_vars_event = asyncio.Event() - self.scheduler = UnsilenceScheduler(bot) + super().__init__() + + async def schedule_unsilence(self, channel: SilencedChannel) -> None: + """Schedule expiration for silenced channels.""" + await self.bot.wait_until_guild_available() + log.debug("Scheduling unsilencer") + self.schedule_task(channel.id, channel) + + async def _scheduled_task(self, channel: SilencedChannel) -> None: + """Calls `self.unsilence` on expired silenced channel to unsilence it.""" + await time.wait_until(channel.stop) + log.info("Unsilencing channel after set delay.") + + # Because `self.unsilence` explicitly cancels this scheduled tas, it is shielded + # to avoid prematurely cancelling itself + await asyncio.shield(channel.ctx.invoke(self.unsilence)) async def _get_instance_vars(self) -> None: """Get instance variables after they're available to get from the guild.""" @@ -127,11 +118,10 @@ class Silence(commands.Cog): channel = SilencedChannel( id=ctx.channel.id, ctx=ctx, - silence=self, stop=datetime.datetime.now() + datetime.timedelta(minutes=duration), ) - await self.scheduler.schedule_unsilence(channel) + await self.schedule_unsilence(channel) @commands.command(aliases=("unhush",)) async def unsilence(self, ctx: Context) -> None: @@ -183,7 +173,7 @@ class Silence(commands.Cog): await channel.set_permissions(self._verified_role, **dict(current_overwrite, send_messages=None)) log.info(f"Unsilenced channel #{channel} ({channel.id}).") self.notifier.remove_channel(channel) - self.scheduler.cancel_task(channel.id) + self.cancel_task(channel.id) self.muted_channels.discard(channel) return True log.info(f"Tried to unsilence channel #{channel} ({channel.id}) but the channel was not silenced.") -- cgit v1.2.3 From 758edf044bfc24aeb8e00c8e244a770c6a247d42 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 4 Jun 2020 23:14:51 -0700 Subject: Fix AttributeError for category check Not all channels will have a category attribute. This may be fine in production, but it does cause periodic errors when testing locally. --- bot/cogs/stats.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py index 4ebb6423c..d42f55466 100644 --- a/bot/cogs/stats.py +++ b/bot/cogs/stats.py @@ -36,7 +36,8 @@ class Stats(Cog): if message.guild.id != Guild.id: return - if message.channel.category.id == Categories.modmail: + cat = getattr(message.channel, "category", None) + if cat is not None and cat.id == Categories.modmail: if message.channel.id != Channels.incidents: # Do not report modmail channels to stats, there are too many # of them for interesting statistics to be drawn out of this. -- cgit v1.2.3 From 3c305dadf7ae745fcf2ba9375d577ce750408fd3 Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff Date: Fri, 5 Jun 2020 14:55:41 +0200 Subject: Send infraction DM before applying infraction I've "reverted" the change that reversed the order of DM'ing a user about their infraction and applying the actual infraction. A recent PR reversed the order to stop us from sending DMs when applying the infraction failed. However, in order to DM a user, the bot has to share a guild with the recipient and kicking them off of our server first does not help with that. That's why I reverted the change and reverted some other minor changes made in relation to this change. Note: I did not change the code sending the DM itself; I merely moved it back to where it belongs and added a comment about the necessity of doing the DM'ing first. I couldn't cleanly revert a commit to do this, as changes were spread out over and included in multiple commits that also contained changes not related to the `DM->apply infraction` order. --- bot/cogs/moderation/scheduler.py | 41 ++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index f0a3ad1b1..b03d89537 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -106,6 +106,27 @@ class InfractionScheduler(Scheduler): log_content = None failed = False + # DM the user about the infraction if it's not a shadow/hidden infraction. + # This needs to happen before we apply the infraction, as the bot cannot + # send DMs to user that it doesn't share a guild with. If we were to + # apply kick/ban infractions first, this would mean that we'd make it + # impossible for us to deliver a DM. See python-discord/bot#982. + if not infraction["hidden"]: + dm_result = f"{constants.Emojis.failmail} " + dm_log_text = "\nDM: **Failed**" + + # Sometimes user is a discord.Object; make it a proper user. + try: + if not isinstance(user, (discord.Member, discord.User)): + user = await self.bot.fetch_user(user.id) + except discord.HTTPException as e: + log.error(f"Failed to DM {user.id}: could not fetch user (status {e.status})") + else: + # Accordingly display whether the user was successfully notified via DM. + if await utils.notify_infraction(user, infr_type, expiry, reason, icon): + dm_result = ":incoming_envelope: " + dm_log_text = "\nDM: Sent" + if infraction["actor"] == self.bot.user.id: log.trace( f"Infraction #{id_} actor is bot; including the reason in the confirmation message." @@ -150,27 +171,7 @@ class InfractionScheduler(Scheduler): log.exception(log_msg) failed = True - # DM the user about the infraction if it's not a shadow/hidden infraction. - # Don't send DM when applying failed. - if not infraction["hidden"] and not failed: - dm_result = f"{constants.Emojis.failmail} " - dm_log_text = "\nDM: **Failed**" - - # Sometimes user is a discord.Object; make it a proper user. - try: - if not isinstance(user, (discord.Member, discord.User)): - user = await self.bot.fetch_user(user.id) - except discord.HTTPException as e: - log.error(f"Failed to DM {user.id}: could not fetch user (status {e.status})") - else: - # Accordingly display whether the user was successfully notified via DM. - if await utils.notify_infraction(user, infr_type, expiry, reason, icon): - dm_result = ":incoming_envelope: " - dm_log_text = "\nDM: Sent" - if failed: - dm_log_text = "\nDM: **Canceled**" - dm_result = f"{constants.Emojis.failmail} " log.trace(f"Deleted infraction {infraction['id']} from database because applying infraction failed.") try: await self.bot.api_client.delete(f"bot/infractions/{id_}") -- cgit v1.2.3 From 5801480f0a98aabcbc13e7ee1d90d2f226d9e2a9 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 5 Jun 2020 16:16:50 +0300 Subject: Stats: Implement rules stats Increase every shown rule uses count when command called. --- bot/cogs/site.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot/cogs/site.py b/bot/cogs/site.py index 7fc2a9c34..e61cd5003 100644 --- a/bot/cogs/site.py +++ b/bot/cogs/site.py @@ -133,6 +133,9 @@ class Site(Cog): await ctx.send(f":x: Invalid rule indices: {indices}") return + for rule in rules: + self.bot.stats.incr(f"rule_uses.{rule}") + final_rules = tuple(f"**{pick}.** {full_rules[pick - 1]}" for pick in rules) await LinePaginator.paginate(final_rules, ctx, rules_embed, max_lines=3) -- cgit v1.2.3 From 699760a3d803c379dad1236f36c919eb0775e490 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 6 Jun 2020 00:49:33 +0200 Subject: Refactor help_channels.py to use RedisCache. More specifically, we're turning three dicts into RedisCaches: - help_channel_claimants - unanswered - claim_times These will still work the same way, but will now persist their contents across restarts. --- bot/cogs/help_channels.py | 60 +++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 70cef339a..8c01e5dc4 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -9,12 +9,14 @@ from contextlib import suppress from datetime import datetime from pathlib import Path +import dateutil import discord import discord.abc from discord.ext import commands from bot import constants from bot.bot import Bot +from bot.utils import RedisCache from bot.utils.checks import with_role_check from bot.utils.scheduling import Scheduler @@ -99,13 +101,24 @@ class HelpChannels(Scheduler, commands.Cog): Help channels are named after the chemical elements in `bot/resources/elements.json`. """ + # This cache tracks which channels are claimed by which members. + # RedisCache[discord.TextChannel.id, t.Union[discord.User.id, discord.Member.id]] + help_channel_claimants = RedisCache() + + # This cache maps a help channel to whether it has had any + # activity other than the original claimant. True being no other + # activity and False being other activity. + # RedisCache[discord.TextChannel.id, bool] + unanswered = RedisCache() + + # This dictionary maps a help channel to the time it was claimed + # RedisCache[discord.TextChannel.id, datetime.datetime] + claim_times = RedisCache() + def __init__(self, bot: Bot): super().__init__() self.bot = bot - self.help_channel_claimants: ( - t.Dict[discord.TextChannel, t.Union[discord.Member, discord.User]] - ) = {} # Categories self.available_category: discord.CategoryChannel = None @@ -125,16 +138,6 @@ class HelpChannels(Scheduler, commands.Cog): self.on_message_lock = asyncio.Lock() self.init_task = self.bot.loop.create_task(self.init_cog()) - # Stats - - # This dictionary maps a help channel to the time it was claimed - self.claim_times: t.Dict[int, datetime] = {} - - # This dictionary maps a help channel to whether it has had any - # activity other than the original claimant. True being no other - # activity and False being other activity. - self.unanswered: t.Dict[int, bool] = {} - def cog_unload(self) -> None: """Cancel the init task and scheduled tasks when the cog unloads.""" log.trace("Cog unload: cancelling the init_cog task") @@ -197,7 +200,7 @@ class HelpChannels(Scheduler, commands.Cog): async def dormant_check(self, ctx: commands.Context) -> bool: """Return True if the user is the help channel claimant or passes the role check.""" - if self.help_channel_claimants.get(ctx.channel) == ctx.author: + if await self.help_channel_claimants.get(ctx.channel.id) == ctx.author.id: log.trace(f"{ctx.author} is the help channel claimant, passing the check for dormant.") self.bot.stats.incr("help.dormant_invoke.claimant") return True @@ -223,7 +226,7 @@ class HelpChannels(Scheduler, commands.Cog): if ctx.channel.category == self.in_use_category: if await self.dormant_check(ctx): with suppress(KeyError): - del self.help_channel_claimants[ctx.channel] + await self.help_channel_claimants.delete(ctx.channel.id) await self.remove_cooldown_role(ctx.author) # Ignore missing task when cooldown has passed but the channel still isn't dormant. @@ -546,13 +549,14 @@ class HelpChannels(Scheduler, commands.Cog): self.bot.stats.incr(f"help.dormant_calls.{caller}") - if channel.id in self.claim_times: - claimed = self.claim_times[channel.id] + if await self.claim_times.contains(channel.id): + claimed_datestring = await self.claim_times.get(channel.id) + claimed = dateutil.parser.parse(claimed_datestring) in_use_time = datetime.now() - claimed self.bot.stats.timing("help.in_use_time", in_use_time) - if channel.id in self.unanswered: - if self.unanswered[channel.id]: + if await self.unanswered.contains(channel.id): + if await self.unanswered.get(channel.id): self.bot.stats.incr("help.sessions.unanswered") else: self.bot.stats.incr("help.sessions.answered") @@ -638,16 +642,16 @@ class HelpChannels(Scheduler, commands.Cog): log.trace(f"Checking if #{channel} ({channel.id}) has been answered.") # Check if there is an entry in unanswered (does not persist across restarts) - if channel.id in self.unanswered: - claimant = self.help_channel_claimants.get(channel) - if not claimant: - # The mapping for this channel was lost, we can't do anything. + if await self.unanswered.contains(channel.id): + claimant_id = await self.help_channel_claimants.get(channel.id) + if not claimant_id: + # The mapping for this channel doesn't exist, we can't do anything. return # Check the message did not come from the claimant - if claimant.id != message.author.id: + if claimant_id != message.author.id: # Mark the channel as answered - self.unanswered[channel.id] = False + await self.unanswered.set(channel.id, False) @commands.Cog.listener() async def on_message(self, message: discord.Message) -> None: @@ -680,12 +684,12 @@ class HelpChannels(Scheduler, commands.Cog): await self.move_to_in_use(channel) await self.revoke_send_permissions(message.author) # Add user with channel for dormant check. - self.help_channel_claimants[channel] = message.author + await self.help_channel_claimants.set(channel.id, message.author.id) self.bot.stats.incr("help.claimed") - self.claim_times[channel.id] = datetime.now() - self.unanswered[channel.id] = True + await self.claim_times.set(channel.id, str(datetime.now())) + await self.unanswered.set(channel.id, True) log.trace(f"Releasing on_message lock for {message.id}.") -- cgit v1.2.3 From 3845d01b5dcb494d37f4b10a9a7ffed5d77a96b8 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 5 Jun 2020 20:20:34 -0700 Subject: Add snekbox to the Docker compose file --- docker-compose.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 9884e35f0..cff7d33d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,14 @@ services: ports: - "127.0.0.1:6379:6379" + snekbox: + image: pythondiscord/snekbox:latest + init: true + ipc: none + ports: + - "127.0.0.1:8060:8060" + privileged: true + web: image: pythondiscord/site:latest command: ["run", "--debug"] @@ -47,6 +55,7 @@ services: depends_on: - web - redis + - snekbox environment: BOT_TOKEN: ${BOT_TOKEN} BOT_API_KEY: badbot13m0n8f570f942013fc818f234916ca531 -- cgit v1.2.3 From fc4eddc7eee3670fdbe5726b13d28ddba57a156b Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 6 Jun 2020 11:57:18 +0200 Subject: Store booleans as integers instead of strings. This means we don't need to rely on strtobool, and is a cleaner implementation overall. Thanks @MarkKoz. --- bot/utils/redis_cache.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 2926e7a89..347a0e54a 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -2,7 +2,6 @@ from __future__ import annotations import asyncio import logging -from distutils.util import strtobool from functools import partialmethod from typing import Any, Dict, ItemsView, Optional, Tuple, Union @@ -119,9 +118,15 @@ class RedisCache: def _to_typestring(key_or_value: RedisKeyOrValue, prefixes: _PrefixTuple) -> str: """Turn a valid Redis type into a typestring.""" for prefix, _type in prefixes: + # Convert bools into integers before storing them. + if type(key_or_value) is bool: + bool_int = int(key_or_value) + return f"{prefix}{bool_int}" + # isinstance is a bad idea here, because isintance(False, int) == True. if type(key_or_value) is _type: return f"{prefix}{key_or_value}" + raise TypeError(f"RedisCache._to_typestring only supports the following: {prefixes}.") @staticmethod @@ -138,7 +143,7 @@ class RedisCache: # For booleans, we need special handling because bool("False") is True. if prefix == "b|": value = key_or_value[len(prefix):] - return bool(strtobool(value)) + return bool(int(value)) # Otherwise we can just convert normally. return _type(key_or_value[len(prefix):]) -- cgit v1.2.3 From 2d7c8e8238179ece54b388deeeb0734ce330b707 Mon Sep 17 00:00:00 2001 From: ItsDrike Date: Sat, 6 Jun 2020 12:17:24 +0200 Subject: Apply suggestions from review --- bot/cogs/moderation/silence.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index 398a70e51..94c560c8e 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -1,9 +1,7 @@ import asyncio -import datetime import logging -from collections import namedtuple from contextlib import suppress -from typing import Optional +from typing import Optional, NamedTuple from discord import TextChannel from discord.ext import commands, tasks @@ -12,14 +10,13 @@ from discord.ext.commands import Context from bot.bot import Bot from bot.constants import Channels, Emojis, Guild, MODERATION_ROLES, Roles from bot.converters import HushDurationConverter -from bot.utils import time from bot.utils.checks import with_role_check from bot.utils.scheduling import Scheduler log = logging.getLogger(__name__) -SilencedChannel = namedtuple( - "SilencedChannel", ("id", "ctx", "stop") +SilencedChannel = NamedTuple( + "SilencedChannel", [("ctx", Context), ("delay", int)] ) @@ -65,11 +62,11 @@ class Silence(Scheduler, commands.Cog): """Commands for stopping channel messages for `verified` role in a channel.""" def __init__(self, bot: Bot): + super().__init__() self.bot = bot self.muted_channels = set() self._get_instance_vars_task = self.bot.loop.create_task(self._get_instance_vars()) self._get_instance_vars_event = asyncio.Event() - super().__init__() async def schedule_unsilence(self, channel: SilencedChannel) -> None: """Schedule expiration for silenced channels.""" @@ -79,10 +76,10 @@ class Silence(Scheduler, commands.Cog): async def _scheduled_task(self, channel: SilencedChannel) -> None: """Calls `self.unsilence` on expired silenced channel to unsilence it.""" - await time.wait_until(channel.stop) + await asyncio.sleep(channel.delay) log.info("Unsilencing channel after set delay.") - # Because `self.unsilence` explicitly cancels this scheduled tas, it is shielded + # Because `self.unsilence` explicitly cancels this scheduled task, it is shielded # to avoid prematurely cancelling itself await asyncio.shield(channel.ctx.invoke(self.unsilence)) @@ -116,12 +113,11 @@ class Silence(Scheduler, commands.Cog): await ctx.send(f"{Emojis.check_mark} silenced current channel for {duration} minute(s).") channel = SilencedChannel( - id=ctx.channel.id, ctx=ctx, - stop=datetime.datetime.now() + datetime.timedelta(minutes=duration), + stop=duration*60, ) - await self.schedule_unsilence(channel) + await self.schedule_task(ctx.channel.id, channel) @commands.command(aliases=("unhush",)) async def unsilence(self, ctx: Context) -> None: @@ -134,9 +130,8 @@ class Silence(Scheduler, commands.Cog): log.debug(f"Unsilencing channel #{ctx.channel} from {ctx.author}'s command.") if not await self._unsilence(ctx.channel): await ctx.send(f"{Emojis.cross_mark} current channel is not silenced.") - return - - await ctx.send(f"{Emojis.check_mark} unsilenced current channel.") + else: + await ctx.send(f"{Emojis.check_mark} unsilenced current channel.") async def _silence(self, channel: TextChannel, persistent: bool, duration: Optional[int]) -> bool: """ -- cgit v1.2.3 From 1859ae7a43bc794b735c3445b1873420da5a7001 Mon Sep 17 00:00:00 2001 From: ItsDrike Date: Sat, 6 Jun 2020 12:20:26 +0200 Subject: Fix import order --- bot/cogs/moderation/silence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index 94c560c8e..c451cea0b 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -1,7 +1,7 @@ import asyncio import logging from contextlib import suppress -from typing import Optional, NamedTuple +from typing import NamedTuple, Optional from discord import TextChannel from discord.ext import commands, tasks -- cgit v1.2.3 From 3c3c8c210507170a2502a4906265af6a2b2525ac Mon Sep 17 00:00:00 2001 From: ItsDrike Date: Sat, 6 Jun 2020 12:26:14 +0200 Subject: Remove unnecessary schedule_unsilence - As suggested, this function is not necessary - Also fixed no longer valid`stop`in SilencedChannel NamedTuple --- bot/cogs/moderation/silence.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index c451cea0b..8223df491 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -68,12 +68,6 @@ class Silence(Scheduler, commands.Cog): self._get_instance_vars_task = self.bot.loop.create_task(self._get_instance_vars()) self._get_instance_vars_event = asyncio.Event() - async def schedule_unsilence(self, channel: SilencedChannel) -> None: - """Schedule expiration for silenced channels.""" - await self.bot.wait_until_guild_available() - log.debug("Scheduling unsilencer") - self.schedule_task(channel.id, channel) - async def _scheduled_task(self, channel: SilencedChannel) -> None: """Calls `self.unsilence` on expired silenced channel to unsilence it.""" await asyncio.sleep(channel.delay) @@ -114,7 +108,7 @@ class Silence(Scheduler, commands.Cog): channel = SilencedChannel( ctx=ctx, - stop=duration*60, + delay=duration*60, ) await self.schedule_task(ctx.channel.id, channel) -- cgit v1.2.3 From 858e2e301234862e66ce03ccd71f46518dcc953f Mon Sep 17 00:00:00 2001 From: ItsDrike Date: Sat, 6 Jun 2020 12:31:00 +0200 Subject: Do not await self.schedule_task - self.schedule_task shouldn't be awaited as it isn't a coroutine --- bot/cogs/moderation/silence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index 8223df491..13f84009f 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -111,7 +111,7 @@ class Silence(Scheduler, commands.Cog): delay=duration*60, ) - await self.schedule_task(ctx.channel.id, channel) + self.schedule_task(ctx.channel.id, channel) @commands.command(aliases=("unhush",)) async def unsilence(self, ctx: Context) -> None: -- cgit v1.2.3 From 94f096fab3bde10ba0da767c568c7a8c3ff3259f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 6 Jun 2020 12:33:06 +0200 Subject: Store epoch timestamps instead of strings. We're also switching from datetime.now() to datetime.utcnow(). --- bot/cogs/help_channels.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 8c01e5dc4..dd3e3cb8b 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -9,7 +9,6 @@ from contextlib import suppress from datetime import datetime from pathlib import Path -import dateutil import discord import discord.abc from discord.ext import commands @@ -550,9 +549,9 @@ class HelpChannels(Scheduler, commands.Cog): self.bot.stats.incr(f"help.dormant_calls.{caller}") if await self.claim_times.contains(channel.id): - claimed_datestring = await self.claim_times.get(channel.id) - claimed = dateutil.parser.parse(claimed_datestring) - in_use_time = datetime.now() - claimed + claimed_timestamp = await self.claim_times.get(channel.id) + claimed = datetime.fromtimestamp(claimed_timestamp) + in_use_time = datetime.utcnow() - claimed self.bot.stats.timing("help.in_use_time", in_use_time) if await self.unanswered.contains(channel.id): @@ -688,7 +687,7 @@ class HelpChannels(Scheduler, commands.Cog): self.bot.stats.incr("help.claimed") - await self.claim_times.set(channel.id, str(datetime.now())) + await self.claim_times.set(channel.id, datetime.utcnow().timestamp()) await self.unanswered.set(channel.id, True) log.trace(f"Releasing on_message lock for {message.id}.") -- cgit v1.2.3 From 40a774e0bb6ed8947a17fe0116e2f1dc0cf89156 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 6 Jun 2020 12:37:01 +0200 Subject: Fix potential race condition. Instead of first checking if the channel.id exists and then checking what it is, we just do a single API call, to prevent cases where something fucky might happen inbetween the first and the second call. --- bot/cogs/help_channels.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index dd3e3cb8b..01c38b408 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -548,20 +548,20 @@ class HelpChannels(Scheduler, commands.Cog): self.bot.stats.incr(f"help.dormant_calls.{caller}") - if await self.claim_times.contains(channel.id): - claimed_timestamp = await self.claim_times.get(channel.id) + claimed_timestamp = await self.claim_times.get(channel.id) + if claimed_timestamp: claimed = datetime.fromtimestamp(claimed_timestamp) in_use_time = datetime.utcnow() - claimed self.bot.stats.timing("help.in_use_time", in_use_time) - if await self.unanswered.contains(channel.id): - if await self.unanswered.get(channel.id): + unanswered = await self.unanswered.get(channel.id) + if unanswered is not None: + if unanswered: self.bot.stats.incr("help.sessions.unanswered") else: self.bot.stats.incr("help.sessions.answered") log.trace(f"Position of #{channel} ({channel.id}) is actually {channel.position}.") - log.trace(f"Sending dormant message for #{channel} ({channel.id}).") embed = discord.Embed(description=DORMANT_MSG) await channel.send(embed=embed) -- cgit v1.2.3 From 93be87cea7cde7333042e2bb9529867723f567a7 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 6 Jun 2020 12:49:50 +0200 Subject: Enable the 'redis' / 'aiohttp' Sentry integrations This will provide breadcrumbs for these systems in all our Sentry events, if applicable. Closes #989. --- bot/__main__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bot/__main__.py b/bot/__main__.py index aa1d1aee8..4e0d4a111 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -3,7 +3,9 @@ import logging import discord import sentry_sdk from discord.ext.commands import when_mentioned_or +from sentry_sdk.integrations.aiohttp import AioHttpIntegration from sentry_sdk.integrations.logging import LoggingIntegration +from sentry_sdk.integrations.redis import RedisIntegration from bot import constants, patches from bot.bot import Bot @@ -15,7 +17,11 @@ sentry_logging = LoggingIntegration( sentry_sdk.init( dsn=constants.Bot.sentry_dsn, - integrations=[sentry_logging] + integrations=[ + sentry_logging, + AioHttpIntegration(), + RedisIntegration(), + ] ) bot = Bot( -- cgit v1.2.3 From 6e34de6397e6ab7d23c6e6abb74a2156375b2c2d Mon Sep 17 00:00:00 2001 From: ItsDrike Date: Sat, 6 Jun 2020 21:29:33 +0200 Subject: Move cancel_task before notifier.remove_channel - as sugested notifier.remove_channel and muted_channels.discard should be together --- bot/cogs/moderation/silence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index 13f84009f..d5b9621d2 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -161,8 +161,8 @@ class Silence(Scheduler, commands.Cog): if current_overwrite.send_messages is False: await channel.set_permissions(self._verified_role, **dict(current_overwrite, send_messages=None)) log.info(f"Unsilenced channel #{channel} ({channel.id}).") - self.notifier.remove_channel(channel) self.cancel_task(channel.id) + self.notifier.remove_channel(channel) self.muted_channels.discard(channel) return True log.info(f"Tried to unsilence channel #{channel} ({channel.id}) but the channel was not silenced.") -- cgit v1.2.3 From 27e269b874fcd038388031959186ffe682c777c0 Mon Sep 17 00:00:00 2001 From: ItsDrike Date: Sat, 6 Jun 2020 21:36:02 +0200 Subject: Change `is` to `was` for unsilenced channel message - As suggested, `was` is more fitting in the message than `is` --- bot/cogs/moderation/silence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index d5b9621d2..08f3973b0 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -123,7 +123,7 @@ class Silence(Scheduler, commands.Cog): await self._get_instance_vars_event.wait() log.debug(f"Unsilencing channel #{ctx.channel} from {ctx.author}'s command.") if not await self._unsilence(ctx.channel): - await ctx.send(f"{Emojis.cross_mark} current channel is not silenced.") + await ctx.send(f"{Emojis.cross_mark} current channel was not silenced.") else: await ctx.send(f"{Emojis.check_mark} unsilenced current channel.") -- cgit v1.2.3 From be4902cbd66c2f7223608ddbfee4aa4f0e1a011a Mon Sep 17 00:00:00 2001 From: ItsDrike Date: Sat, 6 Jun 2020 22:11:53 +0200 Subject: Test for channel not silenced message --- tests/bot/cogs/moderation/test_silence.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/bot/cogs/moderation/test_silence.py b/tests/bot/cogs/moderation/test_silence.py index 3fd149f04..ab3d0742a 100644 --- a/tests/bot/cogs/moderation/test_silence.py +++ b/tests/bot/cogs/moderation/test_silence.py @@ -127,10 +127,20 @@ class SilenceTests(unittest.IsolatedAsyncioTestCase): self.ctx.reset_mock() async def test_unsilence_sent_correct_discord_message(self): - """Proper reply after a successful unsilence.""" - with mock.patch.object(self.cog, "_unsilence", return_value=True): - await self.cog.unsilence.callback(self.cog, self.ctx) - self.ctx.send.assert_called_once_with(f"{Emojis.check_mark} unsilenced current channel.") + """Check if proper message was sent when unsilencing channel.""" + test_cases = ( + (True, f"{Emojis.check_mark} unsilenced current channel."), + (False, f"{Emojis.cross_mark} current channel was not silenced.") + ) + for _unsilence_patch_return, result_message in test_cases: + with self.subTest( + starting_silenced_state=_unsilence_patch_return, + result_message=result_message + ): + with mock.patch.object(self.cog, "_unsilence", return_value=_unsilence_patch_return): + await self.cog.unsilence.callback(self.cog, self.ctx) + self.ctx.send.assert_called_once_with(result_message) + self.ctx.reset_mock() async def test_silence_private_for_false(self): """Permissions are not set and `False` is returned in an already silenced channel.""" -- cgit v1.2.3 From 73f258488d32be6c00aaea2cfa20ff2b24d48b30 Mon Sep 17 00:00:00 2001 From: ItsDrike Date: Sat, 6 Jun 2020 23:41:59 +0200 Subject: Use class instead of NamedTuple - Using a class is more readable than using a NamedTuple --- bot/cogs/moderation/silence.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index 08f3973b0..c8ab6443b 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -15,9 +15,12 @@ from bot.utils.scheduling import Scheduler log = logging.getLogger(__name__) -SilencedChannel = NamedTuple( - "SilencedChannel", [("ctx", Context), ("delay", int)] -) + +class TaskData(NamedTuple): + """Data for a scheduled task.""" + + delay: int + ctx: Context class SilenceNotifier(tasks.Loop): @@ -68,14 +71,14 @@ class Silence(Scheduler, commands.Cog): self._get_instance_vars_task = self.bot.loop.create_task(self._get_instance_vars()) self._get_instance_vars_event = asyncio.Event() - async def _scheduled_task(self, channel: SilencedChannel) -> None: + async def _scheduled_task(self, task: TaskData) -> None: """Calls `self.unsilence` on expired silenced channel to unsilence it.""" - await asyncio.sleep(channel.delay) + await asyncio.sleep(task.delay) log.info("Unsilencing channel after set delay.") # Because `self.unsilence` explicitly cancels this scheduled task, it is shielded # to avoid prematurely cancelling itself - await asyncio.shield(channel.ctx.invoke(self.unsilence)) + await asyncio.shield(task.ctx.invoke(self.unsilence)) async def _get_instance_vars(self) -> None: """Get instance variables after they're available to get from the guild.""" @@ -106,12 +109,12 @@ class Silence(Scheduler, commands.Cog): await ctx.send(f"{Emojis.check_mark} silenced current channel for {duration} minute(s).") - channel = SilencedChannel( - ctx=ctx, + task_data = TaskData( delay=duration*60, + ctx=ctx ) - self.schedule_task(ctx.channel.id, channel) + self.schedule_task(ctx.channel.id, task_data) @commands.command(aliases=("unhush",)) async def unsilence(self, ctx: Context) -> None: -- cgit v1.2.3 From 8570bd2c9d644c82e69e4c3bbae3af24f95180e2 Mon Sep 17 00:00:00 2001 From: Daniel Nash <22755628+crazygmr101@users.noreply.github.com> Date: Sun, 7 Jun 2020 03:15:51 -0500 Subject: Create cooldown.md --- bot/resources/tags/cooldown.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 bot/resources/tags/cooldown.md diff --git a/bot/resources/tags/cooldown.md b/bot/resources/tags/cooldown.md new file mode 100644 index 000000000..a4e237872 --- /dev/null +++ b/bot/resources/tags/cooldown.md @@ -0,0 +1,22 @@ +**Cooldowns** + +Cooldowns are used in discord.py to rate-limit. + +```python +from discord.ext import commands + +class SomeCog(commands.Cog): + def __init__(self): + self._cd = commands.CooldownMapping.from_cooldown(1.0, 60.0, commands.BucketType.user) + + async def cog_check(self, ctx): + bucket = self._cd.get_bucket(ctx.message) + retry_after = bucket.update_rate_limit() + if retry_after: + # you're rate limited + # helpful message here + pass + # you're not rate limited +``` + +`from_cooldown` takes the amount of `update_rate_limit()`s needed to trigger the cooldown, the time in which the cooldown is triggered, and a [`BucketType`](discordpy.readthedocs.io/en/latest/ext/commands/api.html#discord.discord.ext.commands.BucketType). -- cgit v1.2.3 From 97710d5bb8145d10983187bebe554b845a9c0ef1 Mon Sep 17 00:00:00 2001 From: Daniel Nash <22755628+crazygmr101@users.noreply.github.com> Date: Sun, 7 Jun 2020 03:40:59 -0500 Subject: Update cooldown.md --- bot/resources/tags/cooldown.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/resources/tags/cooldown.md b/bot/resources/tags/cooldown.md index a4e237872..3d34c078b 100644 --- a/bot/resources/tags/cooldown.md +++ b/bot/resources/tags/cooldown.md @@ -1,7 +1,7 @@ **Cooldowns** Cooldowns are used in discord.py to rate-limit. - + ```python from discord.ext import commands -- cgit v1.2.3 From 15dbbcf865dd24f5f8697fd85bd60d53d9450fcf Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 7 Jun 2020 12:37:31 +0200 Subject: Remove pointless suppress. Since help_channel_claimants.delete will never raise a KeyError, it's not necessary to suppress one. --- bot/cogs/help_channels.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 01c38b408..e521e3301 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -5,7 +5,6 @@ import logging import random import typing as t from collections import deque -from contextlib import suppress from datetime import datetime from pathlib import Path @@ -224,10 +223,11 @@ class HelpChannels(Scheduler, commands.Cog): log.trace("close command invoked; checking if the channel is in-use.") if ctx.channel.category == self.in_use_category: if await self.dormant_check(ctx): - with suppress(KeyError): - await self.help_channel_claimants.delete(ctx.channel.id) + # Remove the claimant and the cooldown role + await self.help_channel_claimants.delete(ctx.channel.id) await self.remove_cooldown_role(ctx.author) + # Ignore missing task when cooldown has passed but the channel still isn't dormant. self.cancel_task(ctx.author.id, ignore_missing=True) -- cgit v1.2.3 From 290f0982be7bf0f0a709d2c65bee413b11430ba3 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Tue, 9 Jun 2020 01:29:02 +0100 Subject: Add Python Atlanta to guild whitelists --- config-default.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config-default.yml b/config-default.yml index 2c85f5ef3..3a1bdae54 100644 --- a/config-default.yml +++ b/config-default.yml @@ -297,6 +297,7 @@ filter: - 613425648685547541 # Discord Developers - 185590609631903755 # Blender Hub - 420324994703163402 # /r/FlutterDev + - 488751051629920277 # Python Atlanta domain_blacklist: - pornhub.com -- cgit v1.2.3 From 378ef81383050cf4c477afc2c23abb51b700ea68 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 8 Jun 2020 18:41:25 -0700 Subject: Help channels: fix claim timestamp being local MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The datetime module returns a local timestamp for naïve datetimes. It has to be timezone-aware to ensure it will always be in UTC. --- bot/cogs/help_channels.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index e521e3301..40e625338 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -5,7 +5,7 @@ import logging import random import typing as t from collections import deque -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path import discord @@ -110,7 +110,7 @@ class HelpChannels(Scheduler, commands.Cog): unanswered = RedisCache() # This dictionary maps a help channel to the time it was claimed - # RedisCache[discord.TextChannel.id, datetime.datetime] + # RedisCache[discord.TextChannel.id, UtcPosixTimestamp] claim_times = RedisCache() def __init__(self, bot: Bot): @@ -550,7 +550,7 @@ class HelpChannels(Scheduler, commands.Cog): claimed_timestamp = await self.claim_times.get(channel.id) if claimed_timestamp: - claimed = datetime.fromtimestamp(claimed_timestamp) + claimed = datetime.utcfromtimestamp(claimed_timestamp) in_use_time = datetime.utcnow() - claimed self.bot.stats.timing("help.in_use_time", in_use_time) @@ -687,7 +687,10 @@ class HelpChannels(Scheduler, commands.Cog): self.bot.stats.incr("help.claimed") - await self.claim_times.set(channel.id, datetime.utcnow().timestamp()) + # Must use a timezone-aware datetime to ensure a correct POSIX timestamp. + timestamp = datetime.now(timezone.utc).timestamp() + await self.claim_times.set(channel.id, timestamp) + await self.unanswered.set(channel.id, True) log.trace(f"Releasing on_message lock for {message.id}.") -- cgit v1.2.3 From 62cb3f7d9ecd861a13b594f3c63aed83dead2e0e Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 8 Jun 2020 19:01:58 -0700 Subject: Help channels: add a function to get in use time Future code will also need to get this time, so moving it out to a separate function reduces redundancy. --- bot/cogs/help_channels.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 40e625338..13dee8e80 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -5,7 +5,7 @@ import logging import random import typing as t from collections import deque -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import Path import discord @@ -286,6 +286,15 @@ class HelpChannels(Scheduler, commands.Cog): if channel.category_id == category.id and not self.is_excluded_channel(channel): yield channel + async def get_in_use_time(self, channel_id: int) -> t.Optional[timedelta]: + """Return the duration `channel_id` has been in use. Return None if it's not in use.""" + log.trace(f"Calculating in use time for channel {channel_id}.") + + claimed_timestamp = await self.claim_times.get(channel_id) + if claimed_timestamp: + claimed = datetime.utcfromtimestamp(claimed_timestamp) + return datetime.utcnow() - claimed + @staticmethod def get_names() -> t.List[str]: """ @@ -548,10 +557,8 @@ class HelpChannels(Scheduler, commands.Cog): self.bot.stats.incr(f"help.dormant_calls.{caller}") - claimed_timestamp = await self.claim_times.get(channel.id) - if claimed_timestamp: - claimed = datetime.utcfromtimestamp(claimed_timestamp) - in_use_time = datetime.utcnow() - claimed + in_use_time = await self.get_in_use_time(channel.id) + if in_use_time: self.bot.stats.timing("help.in_use_time", in_use_time) unanswered = await self.unanswered.get(channel.id) -- cgit v1.2.3 From 04b37cd054565368e5af42deec5d5ca14fc94199 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 8 Jun 2020 19:37:37 -0700 Subject: Help channels: add a function to schedule cooldown expiration Moving this code into a separate function reduces redundancy down the line. This will also get used to re-scheduled cooldowns after a restart. --- bot/cogs/help_channels.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 13dee8e80..f2785c932 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -794,11 +794,14 @@ class HelpChannels(Scheduler, commands.Cog): # Would mean the user somehow bypassed the lack of permissions (e.g. user is guild owner). self.cancel_task(member.id, ignore_missing=True) - timeout = constants.HelpChannels.claim_minutes * 60 - callback = self.remove_cooldown_role(member) + await self.schedule_cooldown_expiration(member, constants.HelpChannels.claim_minutes * 60) + + async def schedule_cooldown_expiration(self, member: discord.Member, seconds: int) -> None: + """Schedule the cooldown role for `member` to be removed after a duration of `seconds`.""" + log.trace(f"Scheduling removal of {member}'s ({member.id}) cooldown.") - log.trace(f"Scheduling {member}'s ({member.id}) send message permissions to be reinstated.") - self.schedule_task(member.id, TaskData(timeout, callback)) + callback = self.remove_cooldown_role(member) + self.schedule_task(member.id, TaskData(seconds, callback)) async def send_available_message(self, channel: discord.TextChannel) -> None: """Send the available message by editing a dormant message or sending a new message.""" -- cgit v1.2.3 From b49f3e5f4e707dece2de38882be44405563d82e4 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 8 Jun 2020 19:52:46 -0700 Subject: Help channels: use cache to remove cooldowns or re-schedule them Using the cache is more efficient since it can check only the users it expects to have a cooldown rather than searching all guild members. Furthermore, re-scheduling the cooldowns ensures members experience the full duration of the cooldown. Previously, all cooldowns were removed, regardless of whether they were expired. --- bot/cogs/help_channels.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index f2785c932..098634e96 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -397,7 +397,7 @@ class HelpChannels(Scheduler, commands.Cog): log.trace("Initialising the cog.") await self.init_categories() - await self.reset_send_permissions() + await self.check_cooldowns() self.channel_queue = self.create_channel_queue() self.name_queue = self.create_name_queue() @@ -733,15 +733,28 @@ class HelpChannels(Scheduler, commands.Cog): msg = await self.get_last_message(channel) return self.match_bot_embed(msg, AVAILABLE_MSG) - async def reset_send_permissions(self) -> None: - """Reset send permissions in the Available category for claimants.""" - log.trace("Resetting send permissions in the Available category.") + async def check_cooldowns(self) -> None: + """Remove expired cooldowns and re-schedule active ones.""" + log.trace("Checking all cooldowns to remove or re-schedule them.") guild = self.bot.get_guild(constants.Guild.id) + cooldown = constants.HelpChannels.claim_minutes * 60 - # TODO: replace with a persistent cache cause checking every member is quite slow - for member in guild.members: - if self.is_claimant(member): + for channel_id, member_id in await self.help_channel_claimants.items(): + member = guild.get_member(member_id) + if not member: + continue # Member probably left the guild. + + in_use_time = await self.get_in_use_time(channel_id) + + if not in_use_time or in_use_time.seconds > cooldown: + # Remove the role if no claim time could be retrieved or if the cooldown expired. + # Since the channel is in the claimants cache, it is definitely strange for a time + # to not exist. However, it isn't a reason to keep the user stuck with a cooldown. await self.remove_cooldown_role(member) + else: + # The member is still on a cooldown; re-schedule it for the remaining time. + remaining = cooldown - in_use_time.seconds + await self.schedule_cooldown_expiration(member, remaining) async def add_cooldown_role(self, member: discord.Member) -> None: """Add the help cooldown role to `member`.""" -- cgit v1.2.3 From be78a86abea68e05ac80c8a07085cb7f0fa6d3c1 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 8 Jun 2020 20:11:38 -0700 Subject: Help channels: revise inaccurate comment --- bot/cogs/help_channels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 098634e96..86579e940 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -647,7 +647,7 @@ class HelpChannels(Scheduler, commands.Cog): if self.is_in_category(channel, constants.Categories.help_in_use): log.trace(f"Checking if #{channel} ({channel.id}) has been answered.") - # Check if there is an entry in unanswered (does not persist across restarts) + # Check if there is an entry in unanswered if await self.unanswered.contains(channel.id): claimant_id = await self.help_channel_claimants.get(channel.id) if not claimant_id: -- cgit v1.2.3 From 06d8ab2f7203d4ee92a040444bbb1999a36accb3 Mon Sep 17 00:00:00 2001 From: Daniel Nash Date: Wed, 10 Jun 2020 15:49:14 -0500 Subject: Rename to customcooldown.md --- bot/resources/tags/cooldown.md | 22 ---------------------- bot/resources/tags/customcooldown.md | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 22 deletions(-) delete mode 100644 bot/resources/tags/cooldown.md create mode 100644 bot/resources/tags/customcooldown.md diff --git a/bot/resources/tags/cooldown.md b/bot/resources/tags/cooldown.md deleted file mode 100644 index 3d34c078b..000000000 --- a/bot/resources/tags/cooldown.md +++ /dev/null @@ -1,22 +0,0 @@ -**Cooldowns** - -Cooldowns are used in discord.py to rate-limit. - -```python -from discord.ext import commands - -class SomeCog(commands.Cog): - def __init__(self): - self._cd = commands.CooldownMapping.from_cooldown(1.0, 60.0, commands.BucketType.user) - - async def cog_check(self, ctx): - bucket = self._cd.get_bucket(ctx.message) - retry_after = bucket.update_rate_limit() - if retry_after: - # you're rate limited - # helpful message here - pass - # you're not rate limited -``` - -`from_cooldown` takes the amount of `update_rate_limit()`s needed to trigger the cooldown, the time in which the cooldown is triggered, and a [`BucketType`](discordpy.readthedocs.io/en/latest/ext/commands/api.html#discord.discord.ext.commands.BucketType). diff --git a/bot/resources/tags/customcooldown.md b/bot/resources/tags/customcooldown.md new file mode 100644 index 000000000..3d34c078b --- /dev/null +++ b/bot/resources/tags/customcooldown.md @@ -0,0 +1,22 @@ +**Cooldowns** + +Cooldowns are used in discord.py to rate-limit. + +```python +from discord.ext import commands + +class SomeCog(commands.Cog): + def __init__(self): + self._cd = commands.CooldownMapping.from_cooldown(1.0, 60.0, commands.BucketType.user) + + async def cog_check(self, ctx): + bucket = self._cd.get_bucket(ctx.message) + retry_after = bucket.update_rate_limit() + if retry_after: + # you're rate limited + # helpful message here + pass + # you're not rate limited +``` + +`from_cooldown` takes the amount of `update_rate_limit()`s needed to trigger the cooldown, the time in which the cooldown is triggered, and a [`BucketType`](discordpy.readthedocs.io/en/latest/ext/commands/api.html#discord.discord.ext.commands.BucketType). -- cgit v1.2.3 From b71ff0f2bbd7be64d1a0009b9e6530ba3c179926 Mon Sep 17 00:00:00 2001 From: Daniel Nash Date: Wed, 10 Jun 2020 15:53:33 -0500 Subject: Update example to not be in a cog --- bot/resources/tags/customcooldown.md | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/bot/resources/tags/customcooldown.md b/bot/resources/tags/customcooldown.md index 3d34c078b..35f28a1e5 100644 --- a/bot/resources/tags/customcooldown.md +++ b/bot/resources/tags/customcooldown.md @@ -1,22 +1,20 @@ **Cooldowns** -Cooldowns are used in discord.py to rate-limit. +Cooldowns can be used in discord.py to rate-limit. In this example, we're using it in an on_message. ```python from discord.ext import commands -class SomeCog(commands.Cog): - def __init__(self): - self._cd = commands.CooldownMapping.from_cooldown(1.0, 60.0, commands.BucketType.user) +_cd = commands.CooldownMapping.from_cooldown(1.0, 60.0, commands.BucketType.user) - async def cog_check(self, ctx): - bucket = self._cd.get_bucket(ctx.message) - retry_after = bucket.update_rate_limit() - if retry_after: - # you're rate limited - # helpful message here - pass - # you're not rate limited +@bot.event +async def on_message(message): + bucket = _cd.get_bucket(message) + retry_after = bucket.update_rate_limit() + if retry_after: + await message.channel.send("Slow down! You're sending messages too fast") + pass + # you're not rate limited ``` `from_cooldown` takes the amount of `update_rate_limit()`s needed to trigger the cooldown, the time in which the cooldown is triggered, and a [`BucketType`](discordpy.readthedocs.io/en/latest/ext/commands/api.html#discord.discord.ext.commands.BucketType). -- cgit v1.2.3 From b2195f1990c6720dad6819cc118920b89e24beba Mon Sep 17 00:00:00 2001 From: Daniel Nash Date: Wed, 10 Jun 2020 15:56:00 -0500 Subject: Move the not rate-limited message into else --- bot/resources/tags/customcooldown.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bot/resources/tags/customcooldown.md b/bot/resources/tags/customcooldown.md index 35f28a1e5..b44d6b12f 100644 --- a/bot/resources/tags/customcooldown.md +++ b/bot/resources/tags/customcooldown.md @@ -13,8 +13,10 @@ async def on_message(message): retry_after = bucket.update_rate_limit() if retry_after: await message.channel.send("Slow down! You're sending messages too fast") + else: + # you're not rate limited pass - # you're not rate limited + # more code here ``` `from_cooldown` takes the amount of `update_rate_limit()`s needed to trigger the cooldown, the time in which the cooldown is triggered, and a [`BucketType`](discordpy.readthedocs.io/en/latest/ext/commands/api.html#discord.discord.ext.commands.BucketType). -- cgit v1.2.3 From 76da8f09c7c8d62f8451bc561f020f489b3a0970 Mon Sep 17 00:00:00 2001 From: Daniel Nash <22755628+crazygmr101@users.noreply.github.com> Date: Wed, 10 Jun 2020 17:09:17 -0500 Subject: change _cd to message_cooldown Apply suggestions from code review Co-authored-by: Joseph Banks --- bot/resources/tags/customcooldown.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/resources/tags/customcooldown.md b/bot/resources/tags/customcooldown.md index b44d6b12f..f304de246 100644 --- a/bot/resources/tags/customcooldown.md +++ b/bot/resources/tags/customcooldown.md @@ -5,11 +5,11 @@ Cooldowns can be used in discord.py to rate-limit. In this example, we're using ```python from discord.ext import commands -_cd = commands.CooldownMapping.from_cooldown(1.0, 60.0, commands.BucketType.user) +message_cooldown = commands.CooldownMapping.from_cooldown(1.0, 60.0, commands.BucketType.user) @bot.event async def on_message(message): - bucket = _cd.get_bucket(message) + bucket = message_cooldown.get_bucket(message) retry_after = bucket.update_rate_limit() if retry_after: await message.channel.send("Slow down! You're sending messages too fast") -- cgit v1.2.3 From b1b1765596469b76d6321e81732a54e46d3b5865 Mon Sep 17 00:00:00 2001 From: Daniel Nash <22755628+crazygmr101@users.noreply.github.com> Date: Wed, 10 Jun 2020 17:35:44 -0500 Subject: Update bot/resources/tags/customcooldown.md Co-authored-by: Joseph Banks --- bot/resources/tags/customcooldown.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bot/resources/tags/customcooldown.md b/bot/resources/tags/customcooldown.md index f304de246..f7987556d 100644 --- a/bot/resources/tags/customcooldown.md +++ b/bot/resources/tags/customcooldown.md @@ -14,9 +14,7 @@ async def on_message(message): if retry_after: await message.channel.send("Slow down! You're sending messages too fast") else: - # you're not rate limited - pass - # more code here + await message.channel.send("Not ratelimited!") ``` `from_cooldown` takes the amount of `update_rate_limit()`s needed to trigger the cooldown, the time in which the cooldown is triggered, and a [`BucketType`](discordpy.readthedocs.io/en/latest/ext/commands/api.html#discord.discord.ext.commands.BucketType). -- cgit v1.2.3 From 5407e7832f668d23c2539743eceea487cf40b99c Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 11 Jun 2020 07:51:34 +0300 Subject: Filtering: Fix some comments Co-authored-by: Joseph Banks --- bot/cogs/filtering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index caf204561..45e712626 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -57,7 +57,7 @@ def expand_spoilers(text: str) -> str: class Filtering(Cog): """Filtering out invites, blacklisting domains, and warning us of certain regular expressions.""" - # Redis cache for last bad words in nickname alert sent per user. + # Redis cache mapping a user ID to the last timestamp a bad nickname alert was sent name_alerts = RedisCache() def __init__(self, bot: Bot): @@ -161,7 +161,7 @@ class Filtering(Cog): """Send a mod alert every 3 days if a username still matches a watchlist pattern.""" # Use lock to avoid race conditions async with self.name_lock: - # Check does nickname have match in filters. + # Check whether the users display name contains any words in our blacklist matches = self.get_name_matches(member.display_name) if not matches or not await self.check_send_alert(member): -- cgit v1.2.3 From 55263370183b516198d8986cc22c6bfe5d7693c9 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 11 Jun 2020 07:53:19 +0300 Subject: Filtering: Fix nickname filter alert sending spaces Co-authored-by: Joseph Banks --- bot/cogs/filtering.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 45e712626..ff915ea2c 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -168,11 +168,13 @@ class Filtering(Cog): return log.info(f"Sending bad nickname alert for '{member.display_name}' ({member.id}).") + log_string = ( f"**User:** {member.mention} (`{member.id}`)\n" f"**Display Name:** {member.display_name}\n" f"**Bad Matches:** {', '.join(match.group() for match in matches)}" ) + await self.mod_log.send_log_message( icon_url=Icons.token_removed, colour=Colours.soft_red, -- cgit v1.2.3 From c4acae166fa04b0e47a6faa5e454a1de8beba6b7 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 11 Jun 2020 08:22:01 +0300 Subject: Filtering: Use walrus for better looking of code --- bot/cogs/filtering.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index ff915ea2c..841f735e3 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -141,15 +141,13 @@ class Filtering(Cog): """Check bad words from passed string (name). Return list of matches.""" matches = [] for pattern in WATCHLIST_PATTERNS: - match = pattern.search(name) - if match: + if match := pattern.search(name): matches.append(match) return matches async def check_send_alert(self, member: Member) -> bool: """When there is less than 3 days after last alert, return `False`, otherwise `True`.""" - last_alert = await self.name_alerts.get(member.id) - if last_alert: + if last_alert := await self.name_alerts.get(member.id): last_alert = datetime.utcfromtimestamp(last_alert) if datetime.utcnow() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert: log.trace(f"Last alert was too recent for {member}'s nickname.") -- cgit v1.2.3 From f26f70c3433b5043c73986c51f6c2f18ffa60761 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 11 Jun 2020 08:32:34 +0300 Subject: Filtering: Add user avatar thumbnail to nickname alert embed --- bot/cogs/filtering.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 841f735e3..4ebc831e1 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -178,7 +178,8 @@ class Filtering(Cog): colour=Colours.soft_red, title="Username filtering alert", text=log_string, - channel_id=Channels.mod_alerts + channel_id=Channels.mod_alerts, + thumbnail=member.avatar_url ) # Update time when alert sent -- cgit v1.2.3 From efa452830e6f6db1e775371e8f7549772aa11702 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Thu, 11 Jun 2020 11:43:12 +0100 Subject: Create codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..34ba4a679 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,32 @@ +name: "Code scanning - action" + +on: + push: + pull_request: + schedule: + - cron: '0 12 * * *' + +jobs: + CodeQL-Build: + + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + fetch-depth: 2 + + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: python + + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 -- cgit v1.2.3 From b0d92ba56bdf8aad14cf09061213ee64a6f2f142 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Thu, 11 Jun 2020 11:46:36 +0100 Subject: Fix trailing whitespace in Action file --- .github/workflows/codeql-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 34ba4a679..8760b35ec 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -19,7 +19,7 @@ jobs: - run: git checkout HEAD^2 if: ${{ github.event_name == 'pull_request' }} - + - name: Initialize CodeQL uses: github/codeql-action/init@v1 with: -- cgit v1.2.3 From 16f160fda34c67c9840ed753b593d93d460a0d97 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Thu, 11 Jun 2020 12:44:17 +0100 Subject: Add cooldown channel to config-default.yml --- config-default.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config-default.yml b/config-default.yml index 3a1bdae54..3388e5f78 100644 --- a/config-default.yml +++ b/config-default.yml @@ -142,6 +142,7 @@ guild: # Python Help: Available how_to_get_help: 704250143020417084 + cooldown: 720603994149486673 # Logs attachment_log: &ATTACH_LOG 649243850006855680 -- cgit v1.2.3 From 1412d0157227526323d0ab332daa503301b6041e Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Thu, 11 Jun 2020 12:47:18 +0100 Subject: Add cooldown channel to EXCLUDED_CHANNELS tuple --- bot/cogs/help_channels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 70cef339a..6ff285c37 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -22,7 +22,7 @@ log = logging.getLogger(__name__) ASKING_GUIDE_URL = "https://pythondiscord.com/pages/asking-good-questions/" MAX_CHANNELS_PER_CATEGORY = 50 -EXCLUDED_CHANNELS = (constants.Channels.how_to_get_help,) +EXCLUDED_CHANNELS = (constants.Channels.how_to_get_help, constants.Channels.cooldown) HELP_CHANNEL_TOPIC = """ This is a Python help channel. You can claim your own help channel in the Python Help: Available category. -- cgit v1.2.3 From ab63cffa31be9e3d2a225a52fc7192c651614175 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Thu, 11 Jun 2020 12:55:39 +0100 Subject: Add cooldown to Channels in constants.py --- bot/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/constants.py b/bot/constants.py index b31a9c99e..470221369 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -389,6 +389,7 @@ class Channels(metaclass=YAMLGetter): attachment_log: int big_brother_logs: int bot_commands: int + cooldown: int defcon: int dev_contrib: int dev_core: int -- cgit v1.2.3 From df2b40ef8ac8cb69a7af6602ab77025c1549dbe1 Mon Sep 17 00:00:00 2001 From: Mark Date: Thu, 11 Jun 2020 22:07:39 -0700 Subject: Replace mention of Flask with Django The site's description still stated that it was built with Flask, which is no longer accurate due to the move to Django. --- bot/cogs/site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/site.py b/bot/cogs/site.py index e61cd5003..ac29daa1d 100644 --- a/bot/cogs/site.py +++ b/bot/cogs/site.py @@ -33,7 +33,7 @@ class Site(Cog): embed.colour = Colour.blurple() embed.description = ( f"[Our official website]({url}) is an open-source community project " - "created with Python and Flask. It contains information about the server " + "created with Python and Django. It contains information about the server " "itself, lets you sign up for upcoming events, has its own wiki, contains " "a list of valuable learning resources, and much more." ) -- cgit v1.2.3 From ccd3a17d6a8723cabb21a4b2c7e0d8a835dc9e99 Mon Sep 17 00:00:00 2001 From: Daniel Nash <22755628+crazygmr101@users.noreply.github.com> Date: Fri, 12 Jun 2020 08:42:30 -0500 Subject: Make title more specific Co-authored-by: Mark --- bot/resources/tags/customcooldown.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/resources/tags/customcooldown.md b/bot/resources/tags/customcooldown.md index f7987556d..4060a9827 100644 --- a/bot/resources/tags/customcooldown.md +++ b/bot/resources/tags/customcooldown.md @@ -1,4 +1,4 @@ -**Cooldowns** +**Cooldowns in discord.py** Cooldowns can be used in discord.py to rate-limit. In this example, we're using it in an on_message. -- cgit v1.2.3 From 844726f7bedc8c1d772c5f866a29a52cda6f3a9f Mon Sep 17 00:00:00 2001 From: Daniel Nash <22755628+crazygmr101@users.noreply.github.com> Date: Fri, 12 Jun 2020 08:48:16 -0500 Subject: Update customcooldown.md --- bot/resources/tags/customcooldown.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/resources/tags/customcooldown.md b/bot/resources/tags/customcooldown.md index 4060a9827..3f6db0ec6 100644 --- a/bot/resources/tags/customcooldown.md +++ b/bot/resources/tags/customcooldown.md @@ -10,6 +10,8 @@ message_cooldown = commands.CooldownMapping.from_cooldown(1.0, 60.0, commands.Bu @bot.event async def on_message(message): bucket = message_cooldown.get_bucket(message) + # update_rate_limit returns a time you need to wait before + # trying again retry_after = bucket.update_rate_limit() if retry_after: await message.channel.send("Slow down! You're sending messages too fast") -- cgit v1.2.3 From 08db3df18711ff57a56fef99d0ad725669448f3b Mon Sep 17 00:00:00 2001 From: Daniel Nash <22755628+crazygmr101@users.noreply.github.com> Date: Fri, 12 Jun 2020 08:50:30 -0500 Subject: Add scheme to URL --- bot/resources/tags/customcooldown.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/resources/tags/customcooldown.md b/bot/resources/tags/customcooldown.md index 3f6db0ec6..e877e4dae 100644 --- a/bot/resources/tags/customcooldown.md +++ b/bot/resources/tags/customcooldown.md @@ -19,4 +19,4 @@ async def on_message(message): await message.channel.send("Not ratelimited!") ``` -`from_cooldown` takes the amount of `update_rate_limit()`s needed to trigger the cooldown, the time in which the cooldown is triggered, and a [`BucketType`](discordpy.readthedocs.io/en/latest/ext/commands/api.html#discord.discord.ext.commands.BucketType). +`from_cooldown` takes the amount of `update_rate_limit()`s needed to trigger the cooldown, the time in which the cooldown is triggered, and a [`BucketType`](https://discordpy.readthedocs.io/en/latest/ext/commands/api.html#discord.discord.ext.commands.BucketType). -- cgit v1.2.3 From 46e770ba772e3c7048903efff41a5b969717e0d4 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 12 Jun 2020 15:32:25 -0700 Subject: Escape markdown in charinfo embed The embed displays the original character. If it's a markdown char, it would interfere with the embed's actual markdown. The backtick was especially troublesome. Fixes #996 --- bot/cogs/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/utils.py b/bot/cogs/utils.py index 73b4a1c0a..697bf60ce 100644 --- a/bot/cogs/utils.py +++ b/bot/cogs/utils.py @@ -6,7 +6,7 @@ from email.parser import HeaderParser from io import StringIO from typing import Tuple, Union -from discord import Colour, Embed +from discord import Colour, Embed, utils from discord.ext.commands import BadArgument, Cog, Context, command from bot.bot import Bot @@ -145,7 +145,7 @@ class Utils(Cog): u_code = f"\\U{digit:>08}" url = f"https://www.compart.com/en/unicode/U+{digit:>04}" name = f"[{unicodedata.name(char, '')}]({url})" - info = f"`{u_code.ljust(10)}`: {name} - {char}" + info = f"`{u_code.ljust(10)}`: {name} - {utils.escape_markdown(char)}" return info, u_code charlist, rawlist = zip(*(get_info(c) for c in characters)) -- cgit v1.2.3 From 1d0cbbeb46a811b5a049d712aae1a90a1f3a7359 Mon Sep 17 00:00:00 2001 From: Dennis Pham Date: Mon, 15 Jun 2020 00:36:53 -0400 Subject: Add the C# guild to the whitelist --- config-default.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config-default.yml b/config-default.yml index 3388e5f78..aff5fb2e1 100644 --- a/config-default.yml +++ b/config-default.yml @@ -299,6 +299,7 @@ filter: - 185590609631903755 # Blender Hub - 420324994703163402 # /r/FlutterDev - 488751051629920277 # Python Atlanta + - 143867839282020352 # C# domain_blacklist: - pornhub.com -- cgit v1.2.3 From 09f53ca77ae79ceccad91da5e0d44d7013757f0e Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 14 Jun 2020 22:36:28 -0700 Subject: Check infraction reason isn't None before shortening it --- bot/cogs/moderation/infractions.py | 10 +++++++--- bot/cogs/moderation/scheduler.py | 3 +-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 5bfaad796..f685f6991 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -226,7 +226,10 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - action = user.kick(reason=textwrap.shorten(reason, width=512, placeholder="...")) + if reason: + reason = textwrap.shorten(reason, width=512, placeholder="...") + + action = user.kick(reason=reason) await self.apply_infraction(ctx, infraction, user, action) @respect_role_hierarchy() @@ -259,9 +262,10 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - truncated_reason = textwrap.shorten(reason, width=512, placeholder="...") + if reason: + reason = textwrap.shorten(reason, width=512, placeholder="...") - action = ctx.guild.ban(user, reason=truncated_reason, delete_message_days=0) + action = ctx.guild.ban(user, reason=reason, delete_message_days=0) await self.apply_infraction(ctx, infraction, user, action) if infraction.get('expires_at') is not None: diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index b03d89537..beb201b8c 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -127,11 +127,10 @@ class InfractionScheduler(Scheduler): dm_result = ":incoming_envelope: " dm_log_text = "\nDM: Sent" - if infraction["actor"] == self.bot.user.id: + if reason and infraction["actor"] == self.bot.user.id: log.trace( f"Infraction #{id_} actor is bot; including the reason in the confirmation message." ) - end_msg = f" (reason: {textwrap.shorten(reason, width=1500, placeholder='...')})" elif ctx.channel.id not in STAFF_CHANNELS: log.trace( -- cgit v1.2.3 From 08c96f9eb07a2a86e68fb0e0837b9d07c40dab5e Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 14 Jun 2020 22:41:09 -0700 Subject: Fix check for bot actor in infractions The reason None check should be nested to avoid affecting the else/elif statements that follow. --- bot/cogs/moderation/scheduler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index beb201b8c..d75a72ddb 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -127,17 +127,17 @@ class InfractionScheduler(Scheduler): dm_result = ":incoming_envelope: " dm_log_text = "\nDM: Sent" - if reason and infraction["actor"] == self.bot.user.id: + end_msg = "" + if infraction["actor"] == self.bot.user.id: log.trace( f"Infraction #{id_} actor is bot; including the reason in the confirmation message." ) - end_msg = f" (reason: {textwrap.shorten(reason, width=1500, placeholder='...')})" + if reason: + end_msg = f" (reason: {textwrap.shorten(reason, width=1500, placeholder='...')})" elif ctx.channel.id not in STAFF_CHANNELS: log.trace( f"Infraction #{id_} context is not in a staff channel; omitting infraction count." ) - - end_msg = "" else: log.trace(f"Fetching total infraction count for {user}.") -- cgit v1.2.3 From efd27cf29f726627b9ba630e257a2e3e89d3a286 Mon Sep 17 00:00:00 2001 From: Daniel Nash <22755628+crazygmr101@users.noreply.github.com> Date: Mon, 15 Jun 2020 13:18:36 -0400 Subject: Update bot/resources/tags/customcooldown.md Co-authored-by: Mark --- bot/resources/tags/customcooldown.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bot/resources/tags/customcooldown.md b/bot/resources/tags/customcooldown.md index e877e4dae..ac7e70aee 100644 --- a/bot/resources/tags/customcooldown.md +++ b/bot/resources/tags/customcooldown.md @@ -10,11 +10,9 @@ message_cooldown = commands.CooldownMapping.from_cooldown(1.0, 60.0, commands.Bu @bot.event async def on_message(message): bucket = message_cooldown.get_bucket(message) - # update_rate_limit returns a time you need to wait before - # trying again retry_after = bucket.update_rate_limit() if retry_after: - await message.channel.send("Slow down! You're sending messages too fast") + await message.channel.send(f"Slow down! Try again in {retry_after} seconds.") else: await message.channel.send("Not ratelimited!") ``` -- cgit v1.2.3 From c7373fa1143a2d2f2d784a59d40bcb40ee765bfb Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 15 Jun 2020 10:26:23 -0700 Subject: Token remover: ignore DMs It's a private channel so there's no risk of a token "leaking". Furthermore, messages cannot be deleted in DMs. --- bot/cogs/token_remover.py | 3 +++ tests/bot/cogs/test_token_remover.py | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index d55e079e9..493479df9 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -63,6 +63,9 @@ class TokenRemover(Cog): See: https://discordapp.com/developers/docs/reference#snowflakes """ + if not msg.guild: + return # Ignore DMs; can't delete messages in there anyway. + found_token = self.find_token_in_message(msg) if found_token: await self.take_action(msg, found_token) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index a10124d2d..22c31d7b1 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -121,6 +121,16 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): find_token_in_message.assert_called_once_with(self.msg) take_action.assert_not_awaited() + @autospec(TokenRemover, "find_token_in_message") + async def test_on_message_ignores_dms(self, find_token_in_message): + """Shouldn't parse a message if it is a DM.""" + cog = TokenRemover(self.bot) + self.msg.guild = None + + await cog.on_message(self.msg) + + find_token_in_message.assert_not_called() + @autospec("bot.cogs.token_remover", "TOKEN_RE") def test_find_token_ignores_bot_messages(self, token_re): """The token finder should ignore messages authored by bots.""" -- cgit v1.2.3 From 2fa7429327e787a65803c16609da21463723bfeb Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 15 Jun 2020 10:38:46 -0700 Subject: Token remover: move bot check to on_message It just makes more sense to me to filter out messages at an earlier stage. --- bot/cogs/token_remover.py | 8 +++----- tests/bot/cogs/test_token_remover.py | 23 +++++++---------------- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index 493479df9..1f7517501 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -63,8 +63,9 @@ class TokenRemover(Cog): See: https://discordapp.com/developers/docs/reference#snowflakes """ - if not msg.guild: - return # Ignore DMs; can't delete messages in there anyway. + # Ignore DMs; can't delete messages in there anyway. + if not msg.guild or msg.author.bot: + return found_token = self.find_token_in_message(msg) if found_token: @@ -115,9 +116,6 @@ class TokenRemover(Cog): @classmethod def find_token_in_message(cls, msg: Message) -> t.Optional[Token]: """Return a seemingly valid token found in `msg` or `None` if no token is found.""" - if msg.author.bot: - return - # Use finditer rather than search to guard against method calls prematurely returning the # token check (e.g. `message.channel.send` also matches our token pattern) for match in TOKEN_RE.finditer(msg.content): diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 22c31d7b1..98ea9f823 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -122,24 +122,15 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): take_action.assert_not_awaited() @autospec(TokenRemover, "find_token_in_message") - async def test_on_message_ignores_dms(self, find_token_in_message): - """Shouldn't parse a message if it is a DM.""" + async def test_on_message_ignores_dms_bots(self, find_token_in_message): + """Shouldn't parse a message if it is a DM or authored by a bot.""" cog = TokenRemover(self.bot) - self.msg.guild = None + dm_msg = MockMessage(guild=None) + bot_msg = MockMessage(author=MagicMock(bot=True)) - await cog.on_message(self.msg) - - find_token_in_message.assert_not_called() - - @autospec("bot.cogs.token_remover", "TOKEN_RE") - def test_find_token_ignores_bot_messages(self, token_re): - """The token finder should ignore messages authored by bots.""" - self.msg.author.bot = True - - return_value = TokenRemover.find_token_in_message(self.msg) - - self.assertIsNone(return_value) - token_re.finditer.assert_not_called() + for msg in (dm_msg, bot_msg): + await cog.on_message(msg) + find_token_in_message.assert_not_called() @autospec("bot.cogs.token_remover", "TOKEN_RE") def test_find_token_no_matches(self, token_re): -- cgit v1.2.3 From 3aecf14419c87e533d47fe082abeb54ca9edb73c Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 15 Jun 2020 10:49:18 -0700 Subject: Token remover: exit early if message already deleted --- bot/cogs/token_remover.py | 10 ++++++++-- tests/bot/cogs/test_token_remover.py | 15 ++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index 1f7517501..ef979f222 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -4,7 +4,7 @@ import logging import re import typing as t -from discord import Colour, Message +from discord import Colour, Message, NotFound from discord.ext.commands import Cog from bot import utils @@ -83,7 +83,13 @@ class TokenRemover(Cog): async def take_action(self, msg: Message, found_token: Token) -> None: """Remove the `msg` containing the `found_token` and send a mod log message.""" self.mod_log.ignore(Event.message_delete, msg.id) - await msg.delete() + + try: + await msg.delete() + except NotFound: + log.debug(f"Failed to remove token in message {msg.id}: message already deleted.") + return + await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention)) log_message = self.format_log_message(msg, found_token) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 98ea9f823..3349caa73 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -3,7 +3,7 @@ from re import Match from unittest import mock from unittest.mock import MagicMock -from discord import Colour +from discord import Colour, NotFound from bot import constants from bot.cogs import token_remover @@ -282,6 +282,19 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): channel_id=constants.Channels.mod_alerts ) + @mock.patch.object(TokenRemover, "mod_log", new_callable=mock.PropertyMock) + async def test_take_action_delete_failure(self, mod_log_property): + """Shouldn't send any messages if the token message can't be deleted.""" + cog = TokenRemover(self.bot) + mod_log_property.return_value = mock.create_autospec(ModLog, spec_set=True, instance=True) + self.msg.delete.side_effect = NotFound(MagicMock(), MagicMock()) + + token = mock.create_autospec(Token, spec_set=True, instance=True) + await cog.take_action(self.msg, token) + + self.msg.delete.assert_called_once_with() + self.msg.channel.send.assert_not_awaited() + class TokenRemoverExtensionTests(unittest.TestCase): """Tests for the token_remover extension.""" -- cgit v1.2.3 From 0ad19a48680fe6bc729d0e893d32a517a21df7dc Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 15 Jun 2020 10:51:47 -0700 Subject: Webhook remover: ignore DMs and bot messages Can't remove messages in DMs, so don't bother trying. --- bot/cogs/webhook_remover.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bot/cogs/webhook_remover.py b/bot/cogs/webhook_remover.py index 1b5c3f821..74a353e98 100644 --- a/bot/cogs/webhook_remover.py +++ b/bot/cogs/webhook_remover.py @@ -59,6 +59,10 @@ class WebhookRemover(Cog): @Cog.listener() async def on_message(self, msg: Message) -> None: """Check if a Discord webhook URL is in `message`.""" + # Ignore DMs; can't delete messages in there anyway. + if not msg.guild or msg.author.bot: + return + matches = WEBHOOK_URL_RE.search(msg.content) if matches: await self.delete_and_respond(msg, matches[1] + "xxx") -- cgit v1.2.3 From 94a4f8e52f52e98ea50fb0233fedcbbe9ebe6266 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 15 Jun 2020 10:52:01 -0700 Subject: Webhook remover: exit early if message already deleted --- bot/cogs/webhook_remover.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bot/cogs/webhook_remover.py b/bot/cogs/webhook_remover.py index 74a353e98..543869215 100644 --- a/bot/cogs/webhook_remover.py +++ b/bot/cogs/webhook_remover.py @@ -1,7 +1,7 @@ import logging import re -from discord import Colour, Message +from discord import Colour, Message, NotFound from discord.ext.commands import Cog from bot.bot import Bot @@ -35,7 +35,13 @@ class WebhookRemover(Cog): """Delete `msg` and send a warning that it contained the Discord webhook `redacted_url`.""" # Don't log this, due internal delete, not by user. Will make different entry. self.mod_log.ignore(Event.message_delete, msg.id) - await msg.delete() + + try: + await msg.delete() + except NotFound: + log.debug(f"Failed to remove webhook in message {msg.id}: message already deleted.") + return + await msg.channel.send(ALERT_MESSAGE_TEMPLATE.format(user=msg.author.mention)) message = ( -- cgit v1.2.3 From ae44563fe132436d98f50e074e5eb4421eda5538 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 15 Jun 2020 17:41:34 -0700 Subject: Log exception info for failed attachment uploads --- bot/utils/messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/messages.py b/bot/utils/messages.py index de8e186f3..23519a514 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -97,7 +97,7 @@ async def send_attachments( if link_large and e.status == 413: large.append(attachment) else: - log.warning(f"{failure_msg} with status {e.status}.") + log.warning(f"{failure_msg} with status {e.status}.", exc_info=e) if link_large and large: desc = "\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large) -- cgit v1.2.3 From bcf6993de7de726683e6ca9b0f102b6ad1a732fa Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 15 Jun 2020 18:10:52 -0700 Subject: Fix 400 when "clyde" is in webhook username Discord just disallows this name. --- bot/cogs/duck_pond.py | 4 ++-- bot/cogs/python_news.py | 3 ++- bot/cogs/reddit.py | 8 +++++--- bot/cogs/watchchannels/watchchannel.py | 1 + bot/utils/messages.py | 16 ++++++++++++++-- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/bot/cogs/duck_pond.py b/bot/cogs/duck_pond.py index 37d1786a2..5b6a7fd62 100644 --- a/bot/cogs/duck_pond.py +++ b/bot/cogs/duck_pond.py @@ -7,7 +7,7 @@ from discord.ext.commands import Cog from bot import constants from bot.bot import Bot -from bot.utils.messages import send_attachments +from bot.utils.messages import send_attachments, sub_clyde log = logging.getLogger(__name__) @@ -58,7 +58,7 @@ class DuckPond(Cog): try: await self.webhook.send( content=content, - username=username, + username=sub_clyde(username), avatar_url=avatar_url, embed=embed ) diff --git a/bot/cogs/python_news.py b/bot/cogs/python_news.py index d15d0371e..adefd5c7c 100644 --- a/bot/cogs/python_news.py +++ b/bot/cogs/python_news.py @@ -10,6 +10,7 @@ from discord.ext.tasks import loop from bot import constants from bot.bot import Bot +from bot.utils.messages import sub_clyde PEPS_RSS_URL = "https://www.python.org/dev/peps/peps.rss/" @@ -208,7 +209,7 @@ class PythonNews(Cog): return await self.webhook.send( embed=embed, - username=webhook_profile_name, + username=sub_clyde(webhook_profile_name), avatar_url=AVATAR_URL, wait=True ) diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index 3b77538a0..d853ab2ea 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -16,6 +16,7 @@ from bot.constants import Channels, ERROR_REPLIES, Emojis, Reddit as RedditConfi from bot.converters import Subreddit from bot.decorators import with_role from bot.pagination import LinePaginator +from bot.utils.messages import sub_clyde log = logging.getLogger(__name__) @@ -218,7 +219,8 @@ class Reddit(Cog): for subreddit in RedditConfig.subreddits: top_posts = await self.get_top_posts(subreddit=subreddit, time="day") - message = await self.webhook.send(username=f"{subreddit} Top Daily Posts", embed=top_posts, wait=True) + username = sub_clyde(f"{subreddit} Top Daily Posts") + message = await self.webhook.send(username=username, embed=top_posts, wait=True) if message.channel.is_news(): await message.publish() @@ -228,8 +230,8 @@ class Reddit(Cog): for subreddit in RedditConfig.subreddits: # Send and pin the new weekly posts. top_posts = await self.get_top_posts(subreddit=subreddit, time="week") - - message = await self.webhook.send(wait=True, username=f"{subreddit} Top Weekly Posts", embed=top_posts) + username = sub_clyde(f"{subreddit} Top Weekly Posts") + message = await self.webhook.send(wait=True, username=username, embed=top_posts) if subreddit.lower() == "r/python": if not self.channel: diff --git a/bot/cogs/watchchannels/watchchannel.py b/bot/cogs/watchchannels/watchchannel.py index 436778c46..7c58a0fb5 100644 --- a/bot/cogs/watchchannels/watchchannel.py +++ b/bot/cogs/watchchannels/watchchannel.py @@ -204,6 +204,7 @@ class WatchChannel(metaclass=CogABCMeta): embed: Optional[Embed] = None, ) -> None: """Sends a message to the webhook with the specified kwargs.""" + username = messages.sub_clyde(username) try: await self.webhook.send(content=content, username=username, avatar_url=avatar_url, embed=embed) except discord.HTTPException as exc: diff --git a/bot/utils/messages.py b/bot/utils/messages.py index 23519a514..6ad9351cc 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -1,6 +1,7 @@ import asyncio import contextlib import logging +import re from io import BytesIO from typing import List, Optional, Sequence, Union @@ -86,7 +87,7 @@ async def send_attachments( else: await destination.send( file=attachment_file, - username=message.author.display_name, + username=sub_clyde(message.author.display_name), avatar_url=message.author.avatar_url ) elif link_large: @@ -109,8 +110,19 @@ async def send_attachments( else: await destination.send( embed=embed, - username=message.author.display_name, + username=sub_clyde(message.author.display_name), avatar_url=message.author.avatar_url ) return urls + + +def sub_clyde(username: Optional[str]) -> Optional[str]: + """ + Replace "e" in any "clyde" in `username` with a similar Unicode char and return the new string. + + Discord disallows "clyde" anywhere in the username for webhooks. It will return a 400. + Return None only if `username` is None. + """ + if username: + return re.sub(r"(clyd)e", r"\1𝖾", username, flags=re.I) -- cgit v1.2.3 From 2020df342d3aa43acdf7ead026d593f779264002 Mon Sep 17 00:00:00 2001 From: kosayoda Date: Tue, 16 Jun 2020 20:03:15 +0800 Subject: Refactor nested if-statement --- bot/cogs/help_channels.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 86579e940..4c464a7d2 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -562,11 +562,10 @@ class HelpChannels(Scheduler, commands.Cog): self.bot.stats.timing("help.in_use_time", in_use_time) unanswered = await self.unanswered.get(channel.id) - if unanswered is not None: - if unanswered: - self.bot.stats.incr("help.sessions.unanswered") - else: - self.bot.stats.incr("help.sessions.answered") + if unanswered: + self.bot.stats.incr("help.sessions.unanswered") + elif unanswered is not None: + self.bot.stats.incr("help.sessions.answered") log.trace(f"Position of #{channel} ({channel.id}) is actually {channel.position}.") log.trace(f"Sending dormant message for #{channel} ({channel.id}).") -- cgit v1.2.3 From 433f6b6843006aff57cf1e125d340e703c85669f Mon Sep 17 00:00:00 2001 From: kosayoda Date: Tue, 16 Jun 2020 20:10:23 +0800 Subject: Revise inaccurate docstring in RedisCache --- bot/utils/redis_cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 347a0e54a..f342bbb62 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -48,8 +48,8 @@ class RedisCache: behaves, and should be familiar to Python users. The biggest difference is that all the public methods in this class are coroutines, and must be awaited. - Because of limitations in Redis, this cache will only accept strings, integers and - floats both for keys and values. + Because of limitations in Redis, this cache will only accept strings and integers for keys, + and strings, integers, floats and booleans for values. Please note that this class MUST be created as a class attribute, and that that class must also contain an attribute with an instance of our Bot. See `__get__` and `__set_name__` -- cgit v1.2.3 From 2426ea2141dd75aba21562d925244c1a43af94fc Mon Sep 17 00:00:00 2001 From: kosayoda Date: Tue, 16 Jun 2020 21:55:14 +0800 Subject: Revise inaccurate typehint for Optional reason --- bot/cogs/moderation/infractions.py | 49 ++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index f685f6991..3db788eb9 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -53,7 +53,7 @@ class Infractions(InfractionScheduler, commands.Cog): # region: Permanent infractions @command() - async def warn(self, ctx: Context, user: Member, *, reason: str = None) -> None: + async def warn(self, ctx: Context, user: Member, *, reason: t.Optional[str] = None) -> None: """Warn a user for the given reason.""" infraction = await utils.post_infraction(ctx, user, "warning", reason, active=False) if infraction is None: @@ -62,12 +62,12 @@ class Infractions(InfractionScheduler, commands.Cog): await self.apply_infraction(ctx, infraction, user) @command() - async def kick(self, ctx: Context, user: Member, *, reason: str = None) -> None: + async def kick(self, ctx: Context, user: Member, *, reason: t.Optional[str] = None) -> None: """Kick a user for the given reason.""" await self.apply_kick(ctx, user, reason, active=False) @command() - async def ban(self, ctx: Context, user: FetchedMember, *, reason: str = None) -> None: + async def ban(self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None) -> None: """Permanently ban a user for the given reason and stop watching them with Big Brother.""" await self.apply_ban(ctx, user, reason) @@ -75,7 +75,9 @@ class Infractions(InfractionScheduler, commands.Cog): # region: Temporary infractions @command(aliases=["mute"]) - async def tempmute(self, ctx: Context, user: Member, duration: Expiry, *, reason: str = None) -> None: + async def tempmute( + self, ctx: Context, user: Member, duration: Expiry, *, reason: t.Optional[str] = None + ) -> None: """ Temporarily mute a user for the given reason and duration. @@ -94,7 +96,9 @@ class Infractions(InfractionScheduler, commands.Cog): await self.apply_mute(ctx, user, reason, expires_at=duration) @command() - async def tempban(self, ctx: Context, user: FetchedMember, duration: Expiry, *, reason: str = None) -> None: + async def tempban( + self, ctx: Context, user: FetchedMember, duration: Expiry, *, reason: t.Optional[str] = None + ) -> None: """ Temporarily ban a user for the given reason and duration. @@ -116,7 +120,7 @@ class Infractions(InfractionScheduler, commands.Cog): # region: Permanent shadow infractions @command(hidden=True) - async def note(self, ctx: Context, user: FetchedMember, *, reason: str = None) -> None: + async def note(self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None) -> None: """Create a private note for a user with the given reason without notifying the user.""" infraction = await utils.post_infraction(ctx, user, "note", reason, hidden=True, active=False) if infraction is None: @@ -125,12 +129,14 @@ class Infractions(InfractionScheduler, commands.Cog): await self.apply_infraction(ctx, infraction, user) @command(hidden=True, aliases=['shadowkick', 'skick']) - async def shadow_kick(self, ctx: Context, user: Member, *, reason: str = None) -> None: + async def shadow_kick(self, ctx: Context, user: Member, *, reason: t.Optional[str] = None) -> None: """Kick a user for the given reason without notifying the user.""" await self.apply_kick(ctx, user, reason, hidden=True, active=False) @command(hidden=True, aliases=['shadowban', 'sban']) - async def shadow_ban(self, ctx: Context, user: FetchedMember, *, reason: str = None) -> None: + async def shadow_ban( + self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None + ) -> None: """Permanently ban a user for the given reason without notifying the user.""" await self.apply_ban(ctx, user, reason, hidden=True) @@ -138,7 +144,14 @@ class Infractions(InfractionScheduler, commands.Cog): # region: Temporary shadow infractions @command(hidden=True, aliases=["shadowtempmute, stempmute", "shadowmute", "smute"]) - async def shadow_tempmute(self, ctx: Context, user: Member, duration: Expiry, *, reason: str = None) -> None: + async def shadow_tempmute( + self, + ctx: Context, + user: Member, + duration: Expiry, + *, + reason: t.Optional[str] = None + ) -> None: """ Temporarily mute a user for the given reason and duration without notifying the user. @@ -158,12 +171,12 @@ class Infractions(InfractionScheduler, commands.Cog): @command(hidden=True, aliases=["shadowtempban, stempban"]) async def shadow_tempban( - self, - ctx: Context, - user: FetchedMember, - duration: Expiry, - *, - reason: str = None + self, + ctx: Context, + user: FetchedMember, + duration: Expiry, + *, + reason: t.Optional[str] = None ) -> None: """ Temporarily ban a user for the given reason and duration without notifying the user. @@ -198,7 +211,7 @@ class Infractions(InfractionScheduler, commands.Cog): # endregion # region: Base apply functions - async def apply_mute(self, ctx: Context, user: Member, reason: str, **kwargs) -> None: + async def apply_mute(self, ctx: Context, user: Member, reason: t.Optional[str], **kwargs) -> None: """Apply a mute infraction with kwargs passed to `post_infraction`.""" if await utils.get_active_infraction(ctx, user, "mute"): return @@ -218,7 +231,7 @@ class Infractions(InfractionScheduler, commands.Cog): await self.apply_infraction(ctx, infraction, user, action()) @respect_role_hierarchy() - async def apply_kick(self, ctx: Context, user: Member, reason: str, **kwargs) -> None: + async def apply_kick(self, ctx: Context, user: Member, reason: t.Optional[str], **kwargs) -> None: """Apply a kick infraction with kwargs passed to `post_infraction`.""" infraction = await utils.post_infraction(ctx, user, "kick", reason, active=False, **kwargs) if infraction is None: @@ -233,7 +246,7 @@ class Infractions(InfractionScheduler, commands.Cog): await self.apply_infraction(ctx, infraction, user, action) @respect_role_hierarchy() - async def apply_ban(self, ctx: Context, user: UserSnowflake, reason: str, **kwargs) -> None: + async def apply_ban(self, ctx: Context, user: UserSnowflake, reason: t.Optional[str], **kwargs) -> None: """ Apply a ban infraction with kwargs passed to `post_infraction`. -- cgit v1.2.3 From 6fe0bc1c9ce35459b7d9b5bd2309b41dcc4c0dcc Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 16 Jun 2020 12:40:12 -0700 Subject: Add optional type annotations to reason in pardon funcs --- bot/cogs/moderation/infractions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 3db788eb9..f7747e7f8 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -298,7 +298,7 @@ class Infractions(InfractionScheduler, commands.Cog): # endregion # region: Base pardon functions - async def pardon_mute(self, user_id: int, guild: discord.Guild, reason: str) -> t.Dict[str, str]: + async def pardon_mute(self, user_id: int, guild: discord.Guild, reason: t.Optional[str]) -> t.Dict[str, str]: """Remove a user's muted role, DM them a notification, and return a log dict.""" user = guild.get_member(user_id) log_text = {} @@ -324,7 +324,7 @@ class Infractions(InfractionScheduler, commands.Cog): return log_text - async def pardon_ban(self, user_id: int, guild: discord.Guild, reason: str) -> t.Dict[str, str]: + async def pardon_ban(self, user_id: int, guild: discord.Guild, reason: t.Optional[str]) -> t.Dict[str, str]: """Remove a user's ban on the Discord guild and return a log dict.""" user = discord.Object(user_id) log_text = {} -- cgit v1.2.3 From 20a8b6fe92c398fdc246d78591600bb7bde78bca Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 16 Jun 2020 12:48:13 -0700 Subject: Format parameters with a more consistent style --- bot/cogs/moderation/infractions.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index f7747e7f8..3b28526b2 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -75,9 +75,7 @@ class Infractions(InfractionScheduler, commands.Cog): # region: Temporary infractions @command(aliases=["mute"]) - async def tempmute( - self, ctx: Context, user: Member, duration: Expiry, *, reason: t.Optional[str] = None - ) -> None: + async def tempmute(self, ctx: Context, user: Member, duration: Expiry, *, reason: t.Optional[str] = None) -> None: """ Temporarily mute a user for the given reason and duration. @@ -97,7 +95,12 @@ class Infractions(InfractionScheduler, commands.Cog): @command() async def tempban( - self, ctx: Context, user: FetchedMember, duration: Expiry, *, reason: t.Optional[str] = None + self, + ctx: Context, + user: FetchedMember, + duration: Expiry, + *, + reason: t.Optional[str] = None ) -> None: """ Temporarily ban a user for the given reason and duration. @@ -134,9 +137,7 @@ class Infractions(InfractionScheduler, commands.Cog): await self.apply_kick(ctx, user, reason, hidden=True, active=False) @command(hidden=True, aliases=['shadowban', 'sban']) - async def shadow_ban( - self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None - ) -> None: + async def shadow_ban(self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None) -> None: """Permanently ban a user for the given reason without notifying the user.""" await self.apply_ban(ctx, user, reason, hidden=True) @@ -145,12 +146,11 @@ class Infractions(InfractionScheduler, commands.Cog): @command(hidden=True, aliases=["shadowtempmute, stempmute", "shadowmute", "smute"]) async def shadow_tempmute( - self, - ctx: Context, - user: Member, - duration: Expiry, - *, - reason: t.Optional[str] = None + self, ctx: Context, + user: Member, + duration: Expiry, + *, + reason: t.Optional[str] = None ) -> None: """ Temporarily mute a user for the given reason and duration without notifying the user. @@ -171,12 +171,12 @@ class Infractions(InfractionScheduler, commands.Cog): @command(hidden=True, aliases=["shadowtempban, stempban"]) async def shadow_tempban( - self, - ctx: Context, - user: FetchedMember, - duration: Expiry, - *, - reason: t.Optional[str] = None + self, + ctx: Context, + user: FetchedMember, + duration: Expiry, + *, + reason: t.Optional[str] = None ) -> None: """ Temporarily ban a user for the given reason and duration without notifying the user. -- cgit v1.2.3 From 778635241bf6c1a97f60f48a2bc9b40791a524e9 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Wed, 17 Jun 2020 19:15:31 +0100 Subject: Add LMGTFY to domain blacklist --- config-default.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config-default.yml b/config-default.yml index aff5fb2e1..f111c64f5 100644 --- a/config-default.yml +++ b/config-default.yml @@ -331,6 +331,7 @@ filter: - ssteam.site - steamwalletgift.com - discord.gift + - lmgtfy.com word_watchlist: - goo+ks* -- cgit v1.2.3 From 47b6f65e231305c2ceb4f48a2a772a734ae190db Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Wed, 17 Jun 2020 21:20:27 +0100 Subject: Update deletion scheduler to use latest watchlist configuration --- bot/cogs/filtering.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index f7cf4c3ea..76ea68660 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -7,7 +7,7 @@ from typing import List, Mapping, Optional, Union import dateutil import discord.errors from dateutil.relativedelta import relativedelta -from discord import Colour, DMChannel, HTTPException, Member, Message, NotFound, TextChannel +from discord import Colour, HTTPException, Member, Message, NotFound, TextChannel from discord.ext.commands import Cog from discord.utils import escape_markdown @@ -56,6 +56,7 @@ def expand_spoilers(text: str) -> str: split_text[0::2] + split_text[1::2] + split_text ) + OFFENSIVE_MSG_DELETE_TIME = timedelta(days=Filter.offensive_msg_delete_days) @@ -113,6 +114,7 @@ class Filtering(Cog, Scheduler): "function": self._has_watch_regex_match, "type": "watchlist", "content_only": True, + "schedule_deletion": True }, "watch_rich_embeds": { "enabled": Filter.watch_rich_embeds, @@ -120,21 +122,7 @@ class Filtering(Cog, Scheduler): "type": "watchlist", "content_only": False, "schedule_deletion": False - }, - "watch_words": { - "enabled": Filter.watch_words, - "function": self._has_watchlist_words, - "type": "watchlist", - "content_only": True, - "schedule_deletion": True - }, - "watch_tokens": { - "enabled": Filter.watch_tokens, - "function": self._has_watchlist_tokens, - "type": "watchlist", - "content_only": True, - "schedule_deletion": True - }, + } } self.bot.loop.create_task(self.reschedule_offensive_msg_deletion()) @@ -481,7 +469,7 @@ class Filtering(Cog, Scheduler): await self.bot.wait_until_ready() response = await self.bot.api_client.get('bot/offensive-messages',) - now = datetime.datetime.utcnow() + now = datetime.utcnow() for msg in response: delete_at = dateutil.parser.isoparse(msg['delete_date']).replace(tzinfo=None) -- cgit v1.2.3 From 311326b21fe887063f0d4f757b9624f41ed28418 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Wed, 17 Jun 2020 17:04:48 -0700 Subject: Make sub_clyde case-sensitive and use Cyrillic e's The Cyrillic characters are more likely to be rendered similarly to their Latin counterparts than the math sans-serif characters. --- bot/utils/messages.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bot/utils/messages.py b/bot/utils/messages.py index 6ad9351cc..c7d756708 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -119,10 +119,14 @@ async def send_attachments( def sub_clyde(username: Optional[str]) -> Optional[str]: """ - Replace "e" in any "clyde" in `username` with a similar Unicode char and return the new string. + Replace "e"/"E" in any "clyde" in `username` with a Cyrillic "е"/"E" and return the new string. Discord disallows "clyde" anywhere in the username for webhooks. It will return a 400. Return None only if `username` is None. """ + def replace_e(match: re.Match) -> str: + char = "е" if match[2] == "e" else "Е" + return match[1] + char + if username: - return re.sub(r"(clyd)e", r"\1𝖾", username, flags=re.I) + return re.sub(r"(clyd)(e)", replace_e, username, flags=re.I) -- cgit v1.2.3 From c8cbd1e744c5c48490be321b19ef4f062443ed6d Mon Sep 17 00:00:00 2001 From: kwzrd Date: Fri, 19 Jun 2020 18:28:29 +0200 Subject: Pipenv: add script for html coverage report Similarly to the `report` script, this removes the need to invoke coverage when generating the html report. --- Pipfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Pipfile b/Pipfile index b42ca6d58..33be99587 100644 --- a/Pipfile +++ b/Pipfile @@ -50,4 +50,5 @@ precommit = "pre-commit install" build = "docker build -t pythondiscord/bot:latest -f Dockerfile ." push = "docker push pythondiscord/bot:latest" test = "coverage run -m unittest" +html = "coverage html" report = "coverage report" -- cgit v1.2.3 From 55db81a8c089c96a2e5e96110e3c80f0d36ebb58 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 19 Jun 2020 15:16:09 -0700 Subject: Preserve empty string when substituting clyde --- bot/utils/messages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/utils/messages.py b/bot/utils/messages.py index c7d756708..a40a12e98 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -130,3 +130,5 @@ def sub_clyde(username: Optional[str]) -> Optional[str]: if username: return re.sub(r"(clyd)(e)", replace_e, username, flags=re.I) + else: + return username # Empty string or None -- cgit v1.2.3 From 581573f2ece96a9ec666795431ff21068e949a63 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sat, 20 Jun 2020 01:20:35 +0200 Subject: Write unit test for `sub_clyde` --- tests/bot/utils/test_messages.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tests/bot/utils/test_messages.py diff --git a/tests/bot/utils/test_messages.py b/tests/bot/utils/test_messages.py new file mode 100644 index 000000000..9c22c9751 --- /dev/null +++ b/tests/bot/utils/test_messages.py @@ -0,0 +1,27 @@ +import unittest + +from bot.utils import messages + + +class TestMessages(unittest.TestCase): + """Tests for functions in the `bot.utils.messages` module.""" + + def test_sub_clyde(self): + """Uppercase E's and lowercase e's are substituted with their cyrillic counterparts.""" + sub_e = "\u0435" + sub_E = "\u0415" # noqa: N806: Uppercase E in variable name + + test_cases = ( + (None, None), + ("", ""), + ("clyde", f"clyd{sub_e}"), + ("CLYDE", f"CLYD{sub_E}"), + ("cLyDe", f"cLyD{sub_e}"), + ("BIGclyde", f"BIGclyd{sub_e}"), + ("small clydeus the unholy", f"small clyd{sub_e}us the unholy"), + ("BIGCLYDE, babyclyde", f"BIGCLYD{sub_E}, babyclyd{sub_e}"), + ) + + for username_in, username_out in test_cases: + with self.subTest(input=username_in, expected_output=username_out): + self.assertEqual(messages.sub_clyde(username_in), username_out) -- cgit v1.2.3