From b2367b20e9f73d269224a6dbee20e23e6b6de6b7 Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Sun, 22 Nov 2020 12:31:14 +0100 Subject: rework clean to fully use `delete_messages` instead of `purge` --- bot/exts/utils/clean.py | 86 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 26 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index bf25cb4c2..d6dd2401f 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -1,9 +1,10 @@ import logging import random import re -from typing import Iterable, Optional +import time +from typing import Dict, Iterable, List, Optional -from discord import Colour, Embed, Message, TextChannel, User +from discord import Colour, Embed, Message, NotFound, TextChannel, User from discord.ext import commands from discord.ext.commands import Cog, Context, group, has_any_role @@ -36,6 +37,14 @@ class Clean(Cog): """Get currently loaded ModLog cog instance.""" return self.bot.get_cog("ModLog") + async def _delete_messages_individually(self, messages: List[Message]) -> None: + for message in messages: + try: + await message.delete() + except NotFound: + # message doesn't exist or was already deleted + continue + async def _clean_messages( self, amount: int, @@ -107,7 +116,7 @@ class Clean(Cog): elif regex: predicate = predicate_regex # Delete messages that match regex else: - predicate = None # Delete all messages + predicate = lambda m: True # Delete all messages # noqa: E731 # Default to using the invoking context's channel if not channels: @@ -117,19 +126,28 @@ class Clean(Cog): self.mod_log.ignore(Event.message_delete, ctx.message.id) await ctx.message.delete() - messages = [] + # we need Channel to Message mapping for easier deletion via TextChannel.delete_messages + message_mappings: Dict[TextChannel, List[Message]] = {} message_ids = [] self.cleaning = True # Find the IDs of the messages to delete. IDs are needed in order to ignore mod log events. for channel in channels: + + messages = [] + async for message in channel.history(limit=amount): # If at any point the cancel command is invoked, we should stop. if not self.cleaning: return - # If we are looking for specific message. + # If the message passes predicate, let's save it. + if predicate(message): + messages.append(message) + message_ids.append(message) + + # if we are looking for specific message if until_message: # we could use ID's here however in case if the message we are looking for gets deleted, @@ -138,33 +156,49 @@ class Clean(Cog): # means we have found the message until which we were supposed to be deleting. break - # Since we will be using `delete_messages` method of a TextChannel and we need message objects to - # use it as well as to send logs we will start appending messages here instead adding them from - # purge. - messages.append(message) - - # If the message passes predicate, let's save it. - if predicate is None or predicate(message): - message_ids.append(message.id) + if len(messages) > 0: + # we don't want to create mappings of TextChannel to empty list + message_mappings[channel] = messages self.cleaning = False # Now let's delete the actual messages with purge. self.mod_log.ignore(Event.message_delete, *message_ids) - for channel in channels: - if until_message: - for i in range(0, len(messages), 100): - # while purge automatically handles the amount of messages - # delete_messages only allows for up to 100 messages at once - # thus we need to paginate the amount to always be <= 100 - await channel.delete_messages(messages[i:i + 100]) - else: - messages += await channel.purge(limit=amount, check=predicate) - # Reverse the list to restore chronological order - if messages: - messages = reversed(messages) - log_url = await self.mod_log.upload_log(messages, ctx.author.id) + # Creates ID like int object that would represent an object that is exactly 14 days old + minimum_time = int((time.time() - 14 * 24 * 60 * 60) * 1000.0 - 1420070400000) << 22 + + for channel, messages in message_mappings.items(): + + to_delete = [] + + for current_index, message in enumerate(messages): + + if message.id < minimum_time: + # further messages are too old to be deleted in bulk + await self._delete_messages_individually(messages[current_index:]) + break + + to_delete.append(message) + + if len(to_delete) == 100: + # we can only delete up to 100 messages in a bulk + await channel.delete_messages(to_delete) + to_delete.clear() + + if len(to_delete) > 0: + # deleting any leftover messages if there are any + await channel.delete_messages(to_delete) + + log_messages = [] + + for messages in message_mappings.values(): + log_messages.extend(messages) + + if log_messages: + # Reverse the list to restore chronological order + log_messages = reversed(log_messages) + log_url = await self.mod_log.upload_log(log_messages, ctx.author.id) else: # Can't build an embed, nothing to clean! embed = Embed( -- cgit v1.2.3 From 7bcc74fb600c8e64fa136d8273fea6fbb3834339 Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Sun, 22 Nov 2020 12:32:08 +0100 Subject: rename command `messages` to `until` new name should be more selfexplanatory --- bot/exts/utils/clean.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index d6dd2401f..7ee0287fd 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -277,9 +277,9 @@ class Clean(Cog): """Delete all messages that match a certain regex, stop cleaning after traversing `amount` messages.""" await self._clean_messages(amount, ctx, regex=regex, channels=channels) - @clean_group.command(name="message", aliases=["messages"]) + @clean_group.command(name="until") @has_any_role(*MODERATION_ROLES) - async def clean_message(self, ctx: Context, message: Message) -> None: + async def clean_until(self, ctx: Context, message: Message) -> None: """Delete all messages until certain message, stop cleaning after hitting the `message`.""" await self._clean_messages( CleanMessages.message_limit, -- cgit v1.2.3 From 3797474cabac3fae94a381c0e00998d563efdc5a Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Tue, 23 Feb 2021 14:50:35 +0100 Subject: Introduce cache to cleaning as well as fix cancel --- bot/exts/utils/clean.py | 125 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 89 insertions(+), 36 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index 7ee0287fd..6301ade04 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -2,7 +2,8 @@ import logging import random import re import time -from typing import Dict, Iterable, List, Optional +from collections import defaultdict +from typing import Callable, DefaultDict, Iterable, List, Optional from discord import Colour, Embed, Message, NotFound, TextChannel, User from discord.ext import commands @@ -16,6 +17,9 @@ from bot.exts.moderation.modlog import ModLog log = logging.getLogger(__name__) +# Type alias for checks +CheckHint = Callable[[Message], bool] + class Clean(Cog): """ @@ -39,18 +43,74 @@ class Clean(Cog): async def _delete_messages_individually(self, messages: List[Message]) -> None: for message in messages: + # Ensure that deletion was not canceled + if not self.cleaning: + return try: await message.delete() except NotFound: - # message doesn't exist or was already deleted + # Message doesn't exist or was already deleted continue + def _get_messages_from_cache(self, amount: int, check: CheckHint) -> List[DefaultDict, List[int]]: + """Helper function for getting messages from the cache.""" + message_mappings = defaultdict(lambda: []) + message_ids = [] + for message in self.bot.cached_messages: + if not self.cleaning: + # Cleaning was canceled + return (message_mappings, message_ids) + + if check(message): + message_mappings[message.channel].append(message) + message_ids.append(message.id) + + if len(message_ids) == amount: + # We've got the requested amount of messages + return message_mappings, message_ids + + # Amount exceeds amount of messages matching the check + return message_mappings, message_ids + + async def _get_messages_from_channels( + self, + amount: int, + channels: Iterable[TextChannel], + check: CheckHint, + until_message: Optional[Message] = None + ) -> DefaultDict: + message_mappings = defaultdict(lambda: []) + message_ids = [] + + for channel in channels: + + async for message in channel.history(amount=amount): + + if not self.cleaning: + # Cleaning was canceled + return (message_mappings, message_ids) + + if check(message): + message_mappings[message.channel].append(message) + message_ids.append(message.id) + + if until_message: + + # We could use ID's here however in case if the message we are looking for gets deleted, + # We won't have a way to figure that out thus checking for datetime should be more reliable + if message.created_at < until_message.created_at: + # Means we have found the message until which we were supposed to be deleting. + break + + return message_mappings, message_ids + async def _clean_messages( self, amount: int, ctx: Context, channels: Iterable[TextChannel], bots_only: bool = False, + use_cache: bool = False, user: User = None, regex: Optional[str] = None, until_message: Optional[Message] = None, @@ -126,41 +186,21 @@ class Clean(Cog): self.mod_log.ignore(Event.message_delete, ctx.message.id) await ctx.message.delete() - # we need Channel to Message mapping for easier deletion via TextChannel.delete_messages - message_mappings: Dict[TextChannel, List[Message]] = {} - message_ids = [] self.cleaning = True - # Find the IDs of the messages to delete. IDs are needed in order to ignore mod log events. - for channel in channels: - - messages = [] - - async for message in channel.history(limit=amount): - - # If at any point the cancel command is invoked, we should stop. - if not self.cleaning: - return - - # If the message passes predicate, let's save it. - if predicate(message): - messages.append(message) - message_ids.append(message) - - # if we are looking for specific message - if until_message: - - # we could use ID's here however in case if the message we are looking for gets deleted, - # we won't have a way to figure that out thus checking for datetime should be more reliable - if message.created_at < until_message.created_at: - # means we have found the message until which we were supposed to be deleting. - break - - if len(messages) > 0: - # we don't want to create mappings of TextChannel to empty list - message_mappings[channel] = messages + if use_cache: + message_mappings, message_ids = self._get_messages_from_cache(amount, predicate) + else: + message_mappings, message_ids = await self._get_messages_from_channels( + amount=amount, + channels=channels, + check=predicate, + until_message=until_message + ) - self.cleaning = False + if not self.cleaning: + # Means that the cleaning was canceled + return # Now let's delete the actual messages with purge. self.mod_log.ignore(Event.message_delete, *message_ids) @@ -174,9 +214,16 @@ class Clean(Cog): for current_index, message in enumerate(messages): + if not self.cleaning: + # Means that the cleaning was canceled + return + if message.id < minimum_time: # further messages are too old to be deleted in bulk await self._delete_messages_individually(messages[current_index:]) + if not self.cleaning: + # Means that deletion was canceled while deleting the individual messages + return break to_delete.append(message) @@ -241,7 +288,10 @@ class Clean(Cog): channels: commands.Greedy[TextChannel] = None ) -> None: """Delete messages posted by the provided user, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, user=user, channels=channels) + use_cache = True + if channels: + use_cache = False + await self._clean_messages(amount, ctx, user=user, channels=channels, use_cache=use_cache) @clean_group.command(name="all", aliases=["everything"]) @has_any_role(*MODERATION_ROLES) @@ -275,7 +325,10 @@ class Clean(Cog): channels: commands.Greedy[TextChannel] = None ) -> None: """Delete all messages that match a certain regex, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, regex=regex, channels=channels) + use_cache = True + if channels: + use_cache = False + await self._clean_messages(amount, ctx, regex=regex, channels=channels, use_cache=use_cache) @clean_group.command(name="until") @has_any_role(*MODERATION_ROLES) -- cgit v1.2.3 From 0a7c7283af42e2b2062a4a555781b24508c6ad38 Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Tue, 23 Feb 2021 18:48:54 +0100 Subject: Implement range clean command --- bot/exts/utils/clean.py | 63 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index 6301ade04..0788eed1d 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -52,7 +52,7 @@ class Clean(Cog): # Message doesn't exist or was already deleted continue - def _get_messages_from_cache(self, amount: int, check: CheckHint) -> List[DefaultDict, List[int]]: + def _get_messages_from_cache(self, amount: int, predicate: CheckHint) -> List[DefaultDict, List[int]]: """Helper function for getting messages from the cache.""" message_mappings = defaultdict(lambda: []) message_ids = [] @@ -61,7 +61,7 @@ class Clean(Cog): # Cleaning was canceled return (message_mappings, message_ids) - if check(message): + if predicate(message): message_mappings[message.channel].append(message) message_ids.append(message.id) @@ -76,7 +76,7 @@ class Clean(Cog): self, amount: int, channels: Iterable[TextChannel], - check: CheckHint, + predicate: CheckHint, until_message: Optional[Message] = None ) -> DefaultDict: message_mappings = defaultdict(lambda: []) @@ -90,7 +90,7 @@ class Clean(Cog): # Cleaning was canceled return (message_mappings, message_ids) - if check(message): + if predicate(message): message_mappings[message.channel].append(message) message_ids.append(message.id) @@ -114,6 +114,7 @@ class Clean(Cog): user: User = None, regex: Optional[str] = None, until_message: Optional[Message] = None, + after_message: Optional[Message] = None, ) -> None: """A helper function that does the actual message cleaning.""" def predicate_bots_only(message: Message) -> bool: @@ -148,6 +149,10 @@ class Clean(Cog): else: return bool(re.search(regex.lower(), content.lower())) + def predicate_range(message: Message) -> bool: + """Check if message is older than message provided in after_message but younger than until_message.""" + return message.created_at > after_message.created_at and message.created_at < until_message.created_at + # Is this an acceptable amount of messages to clean? if amount > CleanMessages.message_limit: embed = Embed( @@ -158,6 +163,38 @@ class Clean(Cog): await ctx.send(embed=embed) return + if after_message: + + # Ensure that until_message is specified. + if not until_message: + embed = Embed( + color=Colour(Colours.soft_red), + title=random.choice(NEGATIVE_REPLIES), + description="`until_message` must be specified if `after_message` is specified." + ) + await ctx.send(embed=embed) + return + + # Check if the messages are not in same channel + if after_message.channel != until_message.channel: + embed = Embed( + color=Colour(Colours.soft_red), + title=random.choice(NEGATIVE_REPLIES), + description="You cannot do range clean across different channel." + ) + await ctx.send(embed=embed) + return + + # Ensure that after_message is younger than until_message + if after_message.created_at >= until_message.created_at: + embed = Embed( + color=Colour(Colours.soft_red), + title=random.choice(NEGATIVE_REPLIES), + description="`after` message must be younger than `until` message" + ) + await ctx.send(embed=embed) + return + # Are we already performing a clean? if self.cleaning: embed = Embed( @@ -175,6 +212,8 @@ class Clean(Cog): predicate = predicate_specific_user # Delete messages from specific user elif regex: predicate = predicate_regex # Delete messages that match regex + elif after_message: + predicate = predicate_range # Delete messages older than specific message else: predicate = lambda m: True # Delete all messages # noqa: E731 @@ -189,12 +228,12 @@ class Clean(Cog): self.cleaning = True if use_cache: - message_mappings, message_ids = self._get_messages_from_cache(amount, predicate) + message_mappings, message_ids = self._get_messages_from_cache(amount=amount, predicate=predicate) else: message_mappings, message_ids = await self._get_messages_from_channels( amount=amount, channels=channels, - check=predicate, + predicate=predicate, until_message=until_message ) @@ -341,6 +380,18 @@ class Clean(Cog): until_message=message ) + @clean_group.command(name="from-to", aliases=["after-until", "range"]) + @has_any_role(*MODERATION_ROLES) + async def clean_from_to(self, ctx: Context, after_message: Message, until_message: Message) -> None: + """Delete all messages within range of messages.""" + await self._clean_messages( + CleanMessages.message_limit, + ctx, + channels=[until_message.channel], + until_message=until_message, + after_message=after_message, + ) + @clean_group.command(name="stop", aliases=["cancel", "abort"]) @has_any_role(*MODERATION_ROLES) async def clean_cancel(self, ctx: Context) -> None: -- cgit v1.2.3 From bb26ed30a27a1fc5951059ceed064422210df91a Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Wed, 24 Feb 2021 06:35:43 +0100 Subject: set `self.cleaning` to False once done cleaning --- bot/exts/utils/clean.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index 0788eed1d..b572f70a7 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -276,6 +276,8 @@ class Clean(Cog): # deleting any leftover messages if there are any await channel.delete_messages(to_delete) + self.cleaning = False + log_messages = [] for messages in message_mappings.values(): -- cgit v1.2.3 From 26c60c13219cdc3db80480016f61cdf90db1a187 Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Thu, 4 Mar 2021 10:26:08 +0100 Subject: Change typing, remove `range` alias --- bot/exts/utils/clean.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index f98e5c255..925d42483 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -3,7 +3,7 @@ import random import re import time from collections import defaultdict -from typing import Callable, DefaultDict, Iterable, List, Optional +from typing import Callable, DefaultDict, Iterable, List, Optional, Tuple from discord import Colour, Embed, Message, NotFound, TextChannel, User from discord.ext import commands @@ -52,7 +52,7 @@ class Clean(Cog): # Message doesn't exist or was already deleted continue - def _get_messages_from_cache(self, amount: int, predicate: CheckHint) -> List[DefaultDict, List[int]]: + def _get_messages_from_cache(self, amount: int, predicate: CheckHint) -> Tuple[DefaultDict, List[int]]: """Helper function for getting messages from the cache.""" message_mappings = defaultdict(lambda: []) message_ids = [] @@ -382,7 +382,7 @@ class Clean(Cog): until_message=message ) - @clean_group.command(name="from-to", aliases=["after-until", "range"]) + @clean_group.command(name="from-to", aliases=["after-until"]) @has_any_role(*MODERATION_ROLES) async def clean_from_to(self, ctx: Context, after_message: Message, until_message: Message) -> None: """Delete all messages within range of messages.""" -- cgit v1.2.3 From 7e74bb3608af5f1f96216db9472ccf5960c9124e Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Thu, 15 Apr 2021 18:44:49 +0200 Subject: swap predicate save order for correct deletion --- bot/exts/utils/clean.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index 925d42483..e080f7caa 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -90,10 +90,6 @@ class Clean(Cog): # Cleaning was canceled return (message_mappings, message_ids) - if predicate(message): - message_mappings[message.channel].append(message) - message_ids.append(message.id) - if until_message: # We could use ID's here however in case if the message we are looking for gets deleted, @@ -102,6 +98,10 @@ class Clean(Cog): # Means we have found the message until which we were supposed to be deleting. break + if predicate(message): + message_mappings[message.channel].append(message) + message_ids.append(message.id) + return message_mappings, message_ids async def _clean_messages( -- cgit v1.2.3 From 11be1666fce272d13e72467563fde53eae0f8419 Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Fri, 16 Apr 2021 07:09:36 +0200 Subject: replace lambda with list in defaultdict --- bot/exts/utils/clean.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index 117d63632..be92c4994 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -54,7 +54,7 @@ class Clean(Cog): def _get_messages_from_cache(self, amount: int, predicate: CheckHint) -> Tuple[DefaultDict, List[int]]: """Helper function for getting messages from the cache.""" - message_mappings = defaultdict(lambda: []) + message_mappings = defaultdict(list) message_ids = [] for message in self.bot.cached_messages: if not self.cleaning: @@ -79,7 +79,7 @@ class Clean(Cog): predicate: CheckHint, until_message: Optional[Message] = None ) -> DefaultDict: - message_mappings = defaultdict(lambda: []) + message_mappings = defaultdict(list) message_ids = [] for channel in channels: -- cgit v1.2.3 From 5bf42c7d3c4927dae2a130a95d83295db746338d Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Fri, 16 Apr 2021 07:10:48 +0200 Subject: Use correct kwarg for channel.history --- bot/exts/utils/clean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index be92c4994..d0abd6784 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -84,7 +84,7 @@ class Clean(Cog): for channel in channels: - async for message in channel.history(amount=amount): + async for message in channel.history(limit=amount): if not self.cleaning: # Cleaning was canceled -- cgit v1.2.3 From add078121aba4630fef86a44b261211de89f4f95 Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Fri, 16 Apr 2021 07:13:38 +0200 Subject: simplify use_cache var --- bot/exts/utils/clean.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index d0abd6784..e41164edc 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -333,9 +333,7 @@ class Clean(Cog): channels: commands.Greedy[TextChannel] = None ) -> None: """Delete messages posted by the provided user, stop cleaning after traversing `amount` messages.""" - use_cache = True - if channels: - use_cache = False + use_cache = not channels await self._clean_messages(amount, ctx, user=user, channels=channels, use_cache=use_cache) @clean_group.command(name="all", aliases=["everything"]) @@ -370,9 +368,7 @@ class Clean(Cog): channels: commands.Greedy[TextChannel] = None ) -> None: """Delete all messages that match a certain regex, stop cleaning after traversing `amount` messages.""" - use_cache = True - if channels: - use_cache = False + use_cache = not channels await self._clean_messages(amount, ctx, regex=regex, channels=channels, use_cache=use_cache) @clean_group.command(name="until") -- cgit v1.2.3 From 477810f86387c85ae6da0b80ffccb40dac60e26e Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Fri, 16 Apr 2021 07:16:17 +0200 Subject: Naming changes for better self documentation --- bot/exts/utils/clean.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index e41164edc..62d9f2dbe 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -18,7 +18,7 @@ from bot.exts.moderation.modlog import ModLog log = logging.getLogger(__name__) # Type alias for checks -CheckHint = Callable[[Message], bool] +Predicate = Callable[[Message], bool] class Clean(Cog): @@ -52,7 +52,7 @@ class Clean(Cog): # Message doesn't exist or was already deleted continue - def _get_messages_from_cache(self, amount: int, predicate: CheckHint) -> Tuple[DefaultDict, List[int]]: + def _get_messages_from_cache(self, amount: int, to_delete: Predicate) -> Tuple[DefaultDict, List[int]]: """Helper function for getting messages from the cache.""" message_mappings = defaultdict(list) message_ids = [] @@ -61,7 +61,7 @@ class Clean(Cog): # Cleaning was canceled return (message_mappings, message_ids) - if predicate(message): + if to_delete(message): message_mappings[message.channel].append(message) message_ids.append(message.id) @@ -76,7 +76,7 @@ class Clean(Cog): self, amount: int, channels: Iterable[TextChannel], - predicate: CheckHint, + to_delete: Predicate, until_message: Optional[Message] = None ) -> DefaultDict: message_mappings = defaultdict(list) @@ -98,7 +98,7 @@ class Clean(Cog): # Means we have found the message until which we were supposed to be deleting. break - if predicate(message): + if to_delete(message): message_mappings[message.channel].append(message) message_ids.append(message.id) -- cgit v1.2.3 From 708063c70bcea2cdeecb1eb66c703817e0195042 Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Fri, 16 Apr 2021 07:17:00 +0200 Subject: make predicate_range inclusive --- bot/exts/utils/clean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index 62d9f2dbe..af9405696 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -151,7 +151,7 @@ class Clean(Cog): def predicate_range(message: Message) -> bool: """Check if message is older than message provided in after_message but younger than until_message.""" - return message.created_at > after_message.created_at and message.created_at < until_message.created_at + return message.created_at >= after_message.created_at and message.created_at <= until_message.created_at # Is this an acceptable amount of messages to clean? if amount > CleanMessages.message_limit: -- cgit v1.2.3 From c2705492ec13984aa75f4c31532ca18b1756121d Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Fri, 16 Apr 2021 07:18:48 +0200 Subject: Better response wording, added alias --- bot/exts/utils/clean.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index af9405696..985025afe 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -180,7 +180,7 @@ class Clean(Cog): embed = Embed( color=Colour(Colours.soft_red), title=random.choice(NEGATIVE_REPLIES), - description="You cannot do range clean across different channel." + description="You cannot do range clean across several channel." ) await ctx.send(embed=embed) return @@ -382,7 +382,7 @@ class Clean(Cog): until_message=message ) - @clean_group.command(name="from-to", aliases=["after-until"]) + @clean_group.command(name="from-to", aliases=["after-until", "between"]) @has_any_role(*MODERATION_ROLES) async def clean_from_to(self, ctx: Context, after_message: Message, until_message: Message) -> None: """Delete all messages within range of messages.""" -- cgit v1.2.3 From 6c9e4f55a26f9903532a9b72f69fcab84c1a3370 Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Fri, 16 Apr 2021 11:41:33 +0200 Subject: Don't delete invocation in mod channel --- bot/exts/utils/clean.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index 985025afe..d9164738a 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -14,6 +14,7 @@ from bot.constants import ( Channels, CleanMessages, Colours, Event, Icons, MODERATION_ROLES, NEGATIVE_REPLIES ) from bot.exts.moderation.modlog import ModLog +from bot.utils.channel import is_mod_channel log = logging.getLogger(__name__) @@ -221,13 +222,15 @@ class Clean(Cog): if not channels: channels = [ctx.channel] - # Delete the invocation first - self.mod_log.ignore(Event.message_delete, ctx.message.id) - try: - await ctx.message.delete() - except errors.NotFound: - # Invocation message has already been deleted - log.info("Tried to delete invocation message, but it was already deleted.") + if not is_mod_channel(ctx.channel): + + # Delete the invocation first + self.mod_log.ignore(Event.message_delete, ctx.message.id) + try: + await ctx.message.delete() + except errors.NotFound: + # Invocation message has already been deleted + log.info("Tried to delete invocation message, but it was already deleted.") self.cleaning = True -- cgit v1.2.3 From e5e4343435c31f53e4d58cf8cbd180bfccd94023 Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Fri, 16 Apr 2021 13:03:13 +0200 Subject: document snowflake check better --- bot/exts/utils/clean.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index d9164738a..e08be79fe 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -105,6 +105,17 @@ class Clean(Cog): return message_mappings, message_ids + def is_older_than_14d(self, message: Message) -> bool: + """ + Precisely checks if message is older than 14 days, bulk deletion limit. + + Inspired by how purge works internally. + Comparison on message age could possibly be less accurate which in turn would resort in problems + with message deletion if said messages are very close to the 14d mark. + """ + two_weeks_old_snowflake = int((time.time() - 14 * 24 * 60 * 60) * 1000.0 - 1420070400000) << 22 + return message.id < two_weeks_old_snowflake + async def _clean_messages( self, amount: int, @@ -251,9 +262,6 @@ class Clean(Cog): # Now let's delete the actual messages with purge. self.mod_log.ignore(Event.message_delete, *message_ids) - # Creates ID like int object that would represent an object that is exactly 14 days old - minimum_time = int((time.time() - 14 * 24 * 60 * 60) * 1000.0 - 1420070400000) << 22 - for channel, messages in message_mappings.items(): to_delete = [] @@ -264,7 +272,7 @@ class Clean(Cog): # Means that the cleaning was canceled return - if message.id < minimum_time: + if self.is_older_than_14d(message): # further messages are too old to be deleted in bulk await self._delete_messages_individually(messages[current_index:]) if not self.cleaning: -- cgit v1.2.3 From 12f3c40954db931300ea606fd03d329f16395f19 Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Mon, 26 Jul 2021 15:27:44 +0100 Subject: Update _get_messages_from_channels return type --- bot/exts/utils/clean.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index e08be79fe..529dd9ee6 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -3,7 +3,7 @@ import random import re import time from collections import defaultdict -from typing import Callable, DefaultDict, Iterable, List, Optional, Tuple +from typing import Any, Callable, DefaultDict, Iterable, List, Optional, Tuple from discord import Colour, Embed, Message, NotFound, TextChannel, User, errors from discord.ext import commands @@ -79,7 +79,7 @@ class Clean(Cog): channels: Iterable[TextChannel], to_delete: Predicate, until_message: Optional[Message] = None - ) -> DefaultDict: + ) -> tuple[defaultdict[Any, list], list]: message_mappings = defaultdict(list) message_ids = [] -- cgit v1.2.3 From 02b3c8af0268239050d52db0becd856bc8ab9863 Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Mon, 26 Jul 2021 15:31:55 +0100 Subject: Make is_older_than_14d a static method --- bot/exts/utils/clean.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index 529dd9ee6..a1a9eafe4 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -42,6 +42,18 @@ class Clean(Cog): """Get currently loaded ModLog cog instance.""" return self.bot.get_cog("ModLog") + @staticmethod + def is_older_than_14d(message: Message) -> bool: + """ + Precisely checks if message is older than 14 days, bulk deletion limit. + + Inspired by how purge works internally. + Comparison on message age could possibly be less accurate which in turn would resort in problems + with message deletion if said messages are very close to the 14d mark. + """ + two_weeks_old_snowflake = int((time.time() - 14 * 24 * 60 * 60) * 1000.0 - 1420070400000) << 22 + return message.id < two_weeks_old_snowflake + async def _delete_messages_individually(self, messages: List[Message]) -> None: for message in messages: # Ensure that deletion was not canceled @@ -105,17 +117,6 @@ class Clean(Cog): return message_mappings, message_ids - def is_older_than_14d(self, message: Message) -> bool: - """ - Precisely checks if message is older than 14 days, bulk deletion limit. - - Inspired by how purge works internally. - Comparison on message age could possibly be less accurate which in turn would resort in problems - with message deletion if said messages are very close to the 14d mark. - """ - two_weeks_old_snowflake = int((time.time() - 14 * 24 * 60 * 60) * 1000.0 - 1420070400000) << 22 - return message.id < two_weeks_old_snowflake - async def _clean_messages( self, amount: int, -- cgit v1.2.3 From 0cc135de21c8fe8a85b2c42b95b04779b3af7baa Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Wed, 28 Jul 2021 17:21:36 +0100 Subject: Return empty containers if clean is cancelled --- bot/exts/utils/clean.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index a1a9eafe4..3aabe42f7 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -100,8 +100,8 @@ class Clean(Cog): async for message in channel.history(limit=amount): if not self.cleaning: - # Cleaning was canceled - return (message_mappings, message_ids) + # Cleaning was canceled, return empty containers + return defaultdict(list), [] if until_message: -- cgit v1.2.3 From 770528c70ff38b739c963c88b89ec6401d687d16 Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Wed, 28 Jul 2021 17:22:24 +0100 Subject: simplify range predicate for clean command --- bot/exts/utils/clean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index 3aabe42f7..847ac5c86 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -164,7 +164,7 @@ class Clean(Cog): def predicate_range(message: Message) -> bool: """Check if message is older than message provided in after_message but younger than until_message.""" - return message.created_at >= after_message.created_at and message.created_at <= until_message.created_at + return after_message.created_at <= message.created_at <= until_message.created_at # Is this an acceptable amount of messages to clean? if amount > CleanMessages.message_limit: -- cgit v1.2.3 From ed352272a67224182178bbd5583746053ec912a6 Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Wed, 28 Jul 2021 17:59:04 +0100 Subject: Rely on error handler for sending input errors to user --- bot/exts/utils/clean.py | 54 +++++++++---------------------------------------- 1 file changed, 9 insertions(+), 45 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index 847ac5c86..7514c7a64 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -1,5 +1,4 @@ import logging -import random import re import time from collections import defaultdict @@ -8,10 +7,11 @@ from typing import Any, Callable, DefaultDict, Iterable, List, Optional, Tuple from discord import Colour, Embed, Message, NotFound, TextChannel, User, errors from discord.ext import commands from discord.ext.commands import Cog, Context, group, has_any_role +from discord.ext.commands.errors import BadArgument, MaxConcurrencyReached, MissingRequiredArgument from bot.bot import Bot from bot.constants import ( - Channels, CleanMessages, Colours, Event, Icons, MODERATION_ROLES, NEGATIVE_REPLIES + Channels, CleanMessages, Colours, Event, Icons, MODERATION_ROLES ) from bot.exts.moderation.modlog import ModLog from bot.utils.channel import is_mod_channel @@ -168,55 +168,24 @@ class Clean(Cog): # Is this an acceptable amount of messages to clean? if amount > CleanMessages.message_limit: - embed = Embed( - color=Colour(Colours.soft_red), - title=random.choice(NEGATIVE_REPLIES), - description=f"You cannot clean more than {CleanMessages.message_limit} messages." - ) - await ctx.send(embed=embed) - return + raise BadArgument(f"You cannot clean more than {CleanMessages.message_limit} messages.") if after_message: - # Ensure that until_message is specified. if not until_message: - embed = Embed( - color=Colour(Colours.soft_red), - title=random.choice(NEGATIVE_REPLIES), - description="`until_message` must be specified if `after_message` is specified." - ) - await ctx.send(embed=embed) - return + raise MissingRequiredArgument("`until_message` must be specified if `after_message` is specified.") - # Check if the messages are not in same channel + # Messages are not in same channel if after_message.channel != until_message.channel: - embed = Embed( - color=Colour(Colours.soft_red), - title=random.choice(NEGATIVE_REPLIES), - description="You cannot do range clean across several channel." - ) - await ctx.send(embed=embed) - return + raise BadArgument("You cannot do range clean across several channel.") # Ensure that after_message is younger than until_message if after_message.created_at >= until_message.created_at: - embed = Embed( - color=Colour(Colours.soft_red), - title=random.choice(NEGATIVE_REPLIES), - description="`after` message must be younger than `until` message" - ) - await ctx.send(embed=embed) - return + raise BadArgument("`after` message must be younger than `until` message") # Are we already performing a clean? if self.cleaning: - embed = Embed( - color=Colour(Colours.soft_red), - title=random.choice(NEGATIVE_REPLIES), - description="Please wait for the currently ongoing clean operation to complete." - ) - await ctx.send(embed=embed) - return + raise MaxConcurrencyReached("Please wait for the currently ongoing clean operation to complete.") # Set up the correct predicate if bots_only: @@ -305,12 +274,7 @@ class Clean(Cog): log_url = await self.mod_log.upload_log(log_messages, ctx.author.id) else: # Can't build an embed, nothing to clean! - embed = Embed( - color=Colour(Colours.soft_red), - description="No matching messages could be found." - ) - await ctx.send(embed=embed, delete_after=10) - return + raise BadArgument("No matching messages could be found.") # Build the embed and send it target_channels = ", ".join(channel.mention for channel in channels) -- cgit v1.2.3 From b2e76ddc6f4d3ccd327f48d9333eb977ddfb72d2 Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Wed, 28 Jul 2021 18:04:13 +0100 Subject: Fix references to kwarg after renaming in clean command --- bot/exts/utils/clean.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py index 7514c7a64..25582165a 100644 --- a/bot/exts/utils/clean.py +++ b/bot/exts/utils/clean.py @@ -216,12 +216,12 @@ class Clean(Cog): self.cleaning = True if use_cache: - message_mappings, message_ids = self._get_messages_from_cache(amount=amount, predicate=predicate) + message_mappings, message_ids = self._get_messages_from_cache(amount=amount, to_delete=predicate) else: message_mappings, message_ids = await self._get_messages_from_channels( amount=amount, channels=channels, - predicate=predicate, + to_delete=predicate, until_message=until_message ) -- cgit v1.2.3 From 97aa87a2d3d262cfa06e922fa93889cc26b7c2cb Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 27 Aug 2021 14:49:29 +0300 Subject: Moved clean cog to moderation ext The cog is moderation related and all commands are exclusive to moderators. --- bot/exts/moderation/clean.py | 388 +++++++++++++++++++++++++++++++++++++++++++ bot/exts/utils/clean.py | 388 ------------------------------------------- 2 files changed, 388 insertions(+), 388 deletions(-) create mode 100644 bot/exts/moderation/clean.py delete mode 100644 bot/exts/utils/clean.py diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py new file mode 100644 index 000000000..25582165a --- /dev/null +++ b/bot/exts/moderation/clean.py @@ -0,0 +1,388 @@ +import logging +import re +import time +from collections import defaultdict +from typing import Any, Callable, DefaultDict, Iterable, List, Optional, Tuple + +from discord import Colour, Embed, Message, NotFound, TextChannel, User, errors +from discord.ext import commands +from discord.ext.commands import Cog, Context, group, has_any_role +from discord.ext.commands.errors import BadArgument, MaxConcurrencyReached, MissingRequiredArgument + +from bot.bot import Bot +from bot.constants import ( + Channels, CleanMessages, Colours, Event, Icons, MODERATION_ROLES +) +from bot.exts.moderation.modlog import ModLog +from bot.utils.channel import is_mod_channel + +log = logging.getLogger(__name__) + +# Type alias for checks +Predicate = Callable[[Message], bool] + + +class Clean(Cog): + """ + A cog that allows messages to be deleted in bulk, while applying various filters. + + You can delete messages sent by a specific user, messages sent by bots, all messages, or messages that match a + specific regular expression. + + The deleted messages are saved and uploaded to the database via an API endpoint, and a URL is returned which can be + used to view the messages in the Discord dark theme style. + """ + + def __init__(self, bot: Bot): + self.bot = bot + self.cleaning = False + + @property + def mod_log(self) -> ModLog: + """Get currently loaded ModLog cog instance.""" + return self.bot.get_cog("ModLog") + + @staticmethod + def is_older_than_14d(message: Message) -> bool: + """ + Precisely checks if message is older than 14 days, bulk deletion limit. + + Inspired by how purge works internally. + Comparison on message age could possibly be less accurate which in turn would resort in problems + with message deletion if said messages are very close to the 14d mark. + """ + two_weeks_old_snowflake = int((time.time() - 14 * 24 * 60 * 60) * 1000.0 - 1420070400000) << 22 + return message.id < two_weeks_old_snowflake + + async def _delete_messages_individually(self, messages: List[Message]) -> None: + for message in messages: + # Ensure that deletion was not canceled + if not self.cleaning: + return + try: + await message.delete() + except NotFound: + # Message doesn't exist or was already deleted + continue + + def _get_messages_from_cache(self, amount: int, to_delete: Predicate) -> Tuple[DefaultDict, List[int]]: + """Helper function for getting messages from the cache.""" + message_mappings = defaultdict(list) + message_ids = [] + for message in self.bot.cached_messages: + if not self.cleaning: + # Cleaning was canceled + return (message_mappings, message_ids) + + if to_delete(message): + message_mappings[message.channel].append(message) + message_ids.append(message.id) + + if len(message_ids) == amount: + # We've got the requested amount of messages + return message_mappings, message_ids + + # Amount exceeds amount of messages matching the check + return message_mappings, message_ids + + async def _get_messages_from_channels( + self, + amount: int, + channels: Iterable[TextChannel], + to_delete: Predicate, + until_message: Optional[Message] = None + ) -> tuple[defaultdict[Any, list], list]: + message_mappings = defaultdict(list) + message_ids = [] + + for channel in channels: + + async for message in channel.history(limit=amount): + + if not self.cleaning: + # Cleaning was canceled, return empty containers + return defaultdict(list), [] + + if until_message: + + # We could use ID's here however in case if the message we are looking for gets deleted, + # We won't have a way to figure that out thus checking for datetime should be more reliable + if message.created_at < until_message.created_at: + # Means we have found the message until which we were supposed to be deleting. + break + + if to_delete(message): + message_mappings[message.channel].append(message) + message_ids.append(message.id) + + return message_mappings, message_ids + + async def _clean_messages( + self, + amount: int, + ctx: Context, + channels: Iterable[TextChannel], + bots_only: bool = False, + use_cache: bool = False, + user: User = None, + regex: Optional[str] = None, + until_message: Optional[Message] = None, + after_message: Optional[Message] = None, + ) -> None: + """A helper function that does the actual message cleaning.""" + def predicate_bots_only(message: Message) -> bool: + """Return True if the message was sent by a bot.""" + return message.author.bot + + def predicate_specific_user(message: Message) -> bool: + """Return True if the message was sent by the user provided in the _clean_messages call.""" + return message.author == user + + def predicate_regex(message: Message) -> bool: + """Check if the regex provided in _clean_messages matches the message content or any embed attributes.""" + content = [message.content] + + # Add the content for all embed attributes + for embed in message.embeds: + content.append(embed.title) + content.append(embed.description) + content.append(embed.footer.text) + content.append(embed.author.name) + for field in embed.fields: + content.append(field.name) + content.append(field.value) + + # Get rid of empty attributes and turn it into a string + content = [attr for attr in content if attr] + content = "\n".join(content) + + # Now let's see if there's a regex match + if not content: + return False + else: + return bool(re.search(regex.lower(), content.lower())) + + def predicate_range(message: Message) -> bool: + """Check if message is older than message provided in after_message but younger than until_message.""" + return after_message.created_at <= message.created_at <= until_message.created_at + + # Is this an acceptable amount of messages to clean? + if amount > CleanMessages.message_limit: + raise BadArgument(f"You cannot clean more than {CleanMessages.message_limit} messages.") + + if after_message: + # Ensure that until_message is specified. + if not until_message: + raise MissingRequiredArgument("`until_message` must be specified if `after_message` is specified.") + + # Messages are not in same channel + if after_message.channel != until_message.channel: + raise BadArgument("You cannot do range clean across several channel.") + + # Ensure that after_message is younger than until_message + if after_message.created_at >= until_message.created_at: + raise BadArgument("`after` message must be younger than `until` message") + + # Are we already performing a clean? + if self.cleaning: + raise MaxConcurrencyReached("Please wait for the currently ongoing clean operation to complete.") + + # Set up the correct predicate + if bots_only: + predicate = predicate_bots_only # Delete messages from bots + elif user: + predicate = predicate_specific_user # Delete messages from specific user + elif regex: + predicate = predicate_regex # Delete messages that match regex + elif after_message: + predicate = predicate_range # Delete messages older than specific message + else: + predicate = lambda m: True # Delete all messages # noqa: E731 + + # Default to using the invoking context's channel + if not channels: + channels = [ctx.channel] + + if not is_mod_channel(ctx.channel): + + # Delete the invocation first + self.mod_log.ignore(Event.message_delete, ctx.message.id) + try: + await ctx.message.delete() + except errors.NotFound: + # Invocation message has already been deleted + log.info("Tried to delete invocation message, but it was already deleted.") + + self.cleaning = True + + if use_cache: + message_mappings, message_ids = self._get_messages_from_cache(amount=amount, to_delete=predicate) + else: + message_mappings, message_ids = await self._get_messages_from_channels( + amount=amount, + channels=channels, + to_delete=predicate, + until_message=until_message + ) + + if not self.cleaning: + # Means that the cleaning was canceled + return + + # Now let's delete the actual messages with purge. + self.mod_log.ignore(Event.message_delete, *message_ids) + + for channel, messages in message_mappings.items(): + + to_delete = [] + + for current_index, message in enumerate(messages): + + if not self.cleaning: + # Means that the cleaning was canceled + return + + if self.is_older_than_14d(message): + # further messages are too old to be deleted in bulk + await self._delete_messages_individually(messages[current_index:]) + if not self.cleaning: + # Means that deletion was canceled while deleting the individual messages + return + break + + to_delete.append(message) + + if len(to_delete) == 100: + # we can only delete up to 100 messages in a bulk + await channel.delete_messages(to_delete) + to_delete.clear() + + if len(to_delete) > 0: + # deleting any leftover messages if there are any + await channel.delete_messages(to_delete) + + self.cleaning = False + + log_messages = [] + + for messages in message_mappings.values(): + log_messages.extend(messages) + + if log_messages: + # Reverse the list to restore chronological order + log_messages = reversed(log_messages) + log_url = await self.mod_log.upload_log(log_messages, ctx.author.id) + else: + # Can't build an embed, nothing to clean! + raise BadArgument("No matching messages could be found.") + + # Build the embed and send it + target_channels = ", ".join(channel.mention for channel in channels) + + message = ( + f"**{len(message_ids)}** messages deleted in {target_channels} by " + f"{ctx.author.mention}\n\n" + f"A log of the deleted messages can be found [here]({log_url})." + ) + + await self.mod_log.send_log_message( + icon_url=Icons.message_bulk_delete, + colour=Colour(Colours.soft_red), + title="Bulk message delete", + text=message, + channel_id=Channels.mod_log, + ) + + @group(invoke_without_command=True, name="clean", aliases=["clear", "purge"]) + @has_any_role(*MODERATION_ROLES) + async def clean_group(self, ctx: Context) -> None: + """Commands for cleaning messages in channels.""" + await ctx.send_help(ctx.command) + + @clean_group.command(name="user", aliases=["users"]) + @has_any_role(*MODERATION_ROLES) + async def clean_user( + self, + ctx: Context, + user: User, + amount: Optional[int] = 10, + channels: commands.Greedy[TextChannel] = None + ) -> None: + """Delete messages posted by the provided user, stop cleaning after traversing `amount` messages.""" + use_cache = not channels + await self._clean_messages(amount, ctx, user=user, channels=channels, use_cache=use_cache) + + @clean_group.command(name="all", aliases=["everything"]) + @has_any_role(*MODERATION_ROLES) + async def clean_all( + self, + ctx: Context, + amount: Optional[int] = 10, + channels: commands.Greedy[TextChannel] = None + ) -> None: + """Delete all messages, regardless of poster, stop cleaning after traversing `amount` messages.""" + await self._clean_messages(amount, ctx, channels=channels) + + @clean_group.command(name="bots", aliases=["bot"]) + @has_any_role(*MODERATION_ROLES) + async def clean_bots( + self, + ctx: Context, + amount: Optional[int] = 10, + channels: commands.Greedy[TextChannel] = None + ) -> None: + """Delete all messages posted by a bot, stop cleaning after traversing `amount` messages.""" + await self._clean_messages(amount, ctx, bots_only=True, channels=channels) + + @clean_group.command(name="regex", aliases=["word", "expression"]) + @has_any_role(*MODERATION_ROLES) + async def clean_regex( + self, + ctx: Context, + regex: str, + amount: Optional[int] = 10, + channels: commands.Greedy[TextChannel] = None + ) -> None: + """Delete all messages that match a certain regex, stop cleaning after traversing `amount` messages.""" + use_cache = not channels + await self._clean_messages(amount, ctx, regex=regex, channels=channels, use_cache=use_cache) + + @clean_group.command(name="until") + @has_any_role(*MODERATION_ROLES) + async def clean_until(self, ctx: Context, message: Message) -> None: + """Delete all messages until certain message, stop cleaning after hitting the `message`.""" + await self._clean_messages( + CleanMessages.message_limit, + ctx, + channels=[message.channel], + until_message=message + ) + + @clean_group.command(name="from-to", aliases=["after-until", "between"]) + @has_any_role(*MODERATION_ROLES) + async def clean_from_to(self, ctx: Context, after_message: Message, until_message: Message) -> None: + """Delete all messages within range of messages.""" + await self._clean_messages( + CleanMessages.message_limit, + ctx, + channels=[until_message.channel], + until_message=until_message, + after_message=after_message, + ) + + @clean_group.command(name="stop", aliases=["cancel", "abort"]) + @has_any_role(*MODERATION_ROLES) + async def clean_cancel(self, ctx: Context) -> None: + """If there is an ongoing cleaning process, attempt to immediately cancel it.""" + self.cleaning = False + + embed = Embed( + color=Colour.blurple(), + description="Clean interrupted." + ) + await ctx.send(embed=embed, delete_after=10) + + +def setup(bot: Bot) -> None: + """Load the Clean cog.""" + bot.add_cog(Clean(bot)) diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py deleted file mode 100644 index 25582165a..000000000 --- a/bot/exts/utils/clean.py +++ /dev/null @@ -1,388 +0,0 @@ -import logging -import re -import time -from collections import defaultdict -from typing import Any, Callable, DefaultDict, Iterable, List, Optional, Tuple - -from discord import Colour, Embed, Message, NotFound, TextChannel, User, errors -from discord.ext import commands -from discord.ext.commands import Cog, Context, group, has_any_role -from discord.ext.commands.errors import BadArgument, MaxConcurrencyReached, MissingRequiredArgument - -from bot.bot import Bot -from bot.constants import ( - Channels, CleanMessages, Colours, Event, Icons, MODERATION_ROLES -) -from bot.exts.moderation.modlog import ModLog -from bot.utils.channel import is_mod_channel - -log = logging.getLogger(__name__) - -# Type alias for checks -Predicate = Callable[[Message], bool] - - -class Clean(Cog): - """ - A cog that allows messages to be deleted in bulk, while applying various filters. - - You can delete messages sent by a specific user, messages sent by bots, all messages, or messages that match a - specific regular expression. - - The deleted messages are saved and uploaded to the database via an API endpoint, and a URL is returned which can be - used to view the messages in the Discord dark theme style. - """ - - def __init__(self, bot: Bot): - self.bot = bot - self.cleaning = False - - @property - def mod_log(self) -> ModLog: - """Get currently loaded ModLog cog instance.""" - return self.bot.get_cog("ModLog") - - @staticmethod - def is_older_than_14d(message: Message) -> bool: - """ - Precisely checks if message is older than 14 days, bulk deletion limit. - - Inspired by how purge works internally. - Comparison on message age could possibly be less accurate which in turn would resort in problems - with message deletion if said messages are very close to the 14d mark. - """ - two_weeks_old_snowflake = int((time.time() - 14 * 24 * 60 * 60) * 1000.0 - 1420070400000) << 22 - return message.id < two_weeks_old_snowflake - - async def _delete_messages_individually(self, messages: List[Message]) -> None: - for message in messages: - # Ensure that deletion was not canceled - if not self.cleaning: - return - try: - await message.delete() - except NotFound: - # Message doesn't exist or was already deleted - continue - - def _get_messages_from_cache(self, amount: int, to_delete: Predicate) -> Tuple[DefaultDict, List[int]]: - """Helper function for getting messages from the cache.""" - message_mappings = defaultdict(list) - message_ids = [] - for message in self.bot.cached_messages: - if not self.cleaning: - # Cleaning was canceled - return (message_mappings, message_ids) - - if to_delete(message): - message_mappings[message.channel].append(message) - message_ids.append(message.id) - - if len(message_ids) == amount: - # We've got the requested amount of messages - return message_mappings, message_ids - - # Amount exceeds amount of messages matching the check - return message_mappings, message_ids - - async def _get_messages_from_channels( - self, - amount: int, - channels: Iterable[TextChannel], - to_delete: Predicate, - until_message: Optional[Message] = None - ) -> tuple[defaultdict[Any, list], list]: - message_mappings = defaultdict(list) - message_ids = [] - - for channel in channels: - - async for message in channel.history(limit=amount): - - if not self.cleaning: - # Cleaning was canceled, return empty containers - return defaultdict(list), [] - - if until_message: - - # We could use ID's here however in case if the message we are looking for gets deleted, - # We won't have a way to figure that out thus checking for datetime should be more reliable - if message.created_at < until_message.created_at: - # Means we have found the message until which we were supposed to be deleting. - break - - if to_delete(message): - message_mappings[message.channel].append(message) - message_ids.append(message.id) - - return message_mappings, message_ids - - async def _clean_messages( - self, - amount: int, - ctx: Context, - channels: Iterable[TextChannel], - bots_only: bool = False, - use_cache: bool = False, - user: User = None, - regex: Optional[str] = None, - until_message: Optional[Message] = None, - after_message: Optional[Message] = None, - ) -> None: - """A helper function that does the actual message cleaning.""" - def predicate_bots_only(message: Message) -> bool: - """Return True if the message was sent by a bot.""" - return message.author.bot - - def predicate_specific_user(message: Message) -> bool: - """Return True if the message was sent by the user provided in the _clean_messages call.""" - return message.author == user - - def predicate_regex(message: Message) -> bool: - """Check if the regex provided in _clean_messages matches the message content or any embed attributes.""" - content = [message.content] - - # Add the content for all embed attributes - for embed in message.embeds: - content.append(embed.title) - content.append(embed.description) - content.append(embed.footer.text) - content.append(embed.author.name) - for field in embed.fields: - content.append(field.name) - content.append(field.value) - - # Get rid of empty attributes and turn it into a string - content = [attr for attr in content if attr] - content = "\n".join(content) - - # Now let's see if there's a regex match - if not content: - return False - else: - return bool(re.search(regex.lower(), content.lower())) - - def predicate_range(message: Message) -> bool: - """Check if message is older than message provided in after_message but younger than until_message.""" - return after_message.created_at <= message.created_at <= until_message.created_at - - # Is this an acceptable amount of messages to clean? - if amount > CleanMessages.message_limit: - raise BadArgument(f"You cannot clean more than {CleanMessages.message_limit} messages.") - - if after_message: - # Ensure that until_message is specified. - if not until_message: - raise MissingRequiredArgument("`until_message` must be specified if `after_message` is specified.") - - # Messages are not in same channel - if after_message.channel != until_message.channel: - raise BadArgument("You cannot do range clean across several channel.") - - # Ensure that after_message is younger than until_message - if after_message.created_at >= until_message.created_at: - raise BadArgument("`after` message must be younger than `until` message") - - # Are we already performing a clean? - if self.cleaning: - raise MaxConcurrencyReached("Please wait for the currently ongoing clean operation to complete.") - - # Set up the correct predicate - if bots_only: - predicate = predicate_bots_only # Delete messages from bots - elif user: - predicate = predicate_specific_user # Delete messages from specific user - elif regex: - predicate = predicate_regex # Delete messages that match regex - elif after_message: - predicate = predicate_range # Delete messages older than specific message - else: - predicate = lambda m: True # Delete all messages # noqa: E731 - - # Default to using the invoking context's channel - if not channels: - channels = [ctx.channel] - - if not is_mod_channel(ctx.channel): - - # Delete the invocation first - self.mod_log.ignore(Event.message_delete, ctx.message.id) - try: - await ctx.message.delete() - except errors.NotFound: - # Invocation message has already been deleted - log.info("Tried to delete invocation message, but it was already deleted.") - - self.cleaning = True - - if use_cache: - message_mappings, message_ids = self._get_messages_from_cache(amount=amount, to_delete=predicate) - else: - message_mappings, message_ids = await self._get_messages_from_channels( - amount=amount, - channels=channels, - to_delete=predicate, - until_message=until_message - ) - - if not self.cleaning: - # Means that the cleaning was canceled - return - - # Now let's delete the actual messages with purge. - self.mod_log.ignore(Event.message_delete, *message_ids) - - for channel, messages in message_mappings.items(): - - to_delete = [] - - for current_index, message in enumerate(messages): - - if not self.cleaning: - # Means that the cleaning was canceled - return - - if self.is_older_than_14d(message): - # further messages are too old to be deleted in bulk - await self._delete_messages_individually(messages[current_index:]) - if not self.cleaning: - # Means that deletion was canceled while deleting the individual messages - return - break - - to_delete.append(message) - - if len(to_delete) == 100: - # we can only delete up to 100 messages in a bulk - await channel.delete_messages(to_delete) - to_delete.clear() - - if len(to_delete) > 0: - # deleting any leftover messages if there are any - await channel.delete_messages(to_delete) - - self.cleaning = False - - log_messages = [] - - for messages in message_mappings.values(): - log_messages.extend(messages) - - if log_messages: - # Reverse the list to restore chronological order - log_messages = reversed(log_messages) - log_url = await self.mod_log.upload_log(log_messages, ctx.author.id) - else: - # Can't build an embed, nothing to clean! - raise BadArgument("No matching messages could be found.") - - # Build the embed and send it - target_channels = ", ".join(channel.mention for channel in channels) - - message = ( - f"**{len(message_ids)}** messages deleted in {target_channels} by " - f"{ctx.author.mention}\n\n" - f"A log of the deleted messages can be found [here]({log_url})." - ) - - await self.mod_log.send_log_message( - icon_url=Icons.message_bulk_delete, - colour=Colour(Colours.soft_red), - title="Bulk message delete", - text=message, - channel_id=Channels.mod_log, - ) - - @group(invoke_without_command=True, name="clean", aliases=["clear", "purge"]) - @has_any_role(*MODERATION_ROLES) - async def clean_group(self, ctx: Context) -> None: - """Commands for cleaning messages in channels.""" - await ctx.send_help(ctx.command) - - @clean_group.command(name="user", aliases=["users"]) - @has_any_role(*MODERATION_ROLES) - async def clean_user( - self, - ctx: Context, - user: User, - amount: Optional[int] = 10, - channels: commands.Greedy[TextChannel] = None - ) -> None: - """Delete messages posted by the provided user, stop cleaning after traversing `amount` messages.""" - use_cache = not channels - await self._clean_messages(amount, ctx, user=user, channels=channels, use_cache=use_cache) - - @clean_group.command(name="all", aliases=["everything"]) - @has_any_role(*MODERATION_ROLES) - async def clean_all( - self, - ctx: Context, - amount: Optional[int] = 10, - channels: commands.Greedy[TextChannel] = None - ) -> None: - """Delete all messages, regardless of poster, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, channels=channels) - - @clean_group.command(name="bots", aliases=["bot"]) - @has_any_role(*MODERATION_ROLES) - async def clean_bots( - self, - ctx: Context, - amount: Optional[int] = 10, - channels: commands.Greedy[TextChannel] = None - ) -> None: - """Delete all messages posted by a bot, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, bots_only=True, channels=channels) - - @clean_group.command(name="regex", aliases=["word", "expression"]) - @has_any_role(*MODERATION_ROLES) - async def clean_regex( - self, - ctx: Context, - regex: str, - amount: Optional[int] = 10, - channels: commands.Greedy[TextChannel] = None - ) -> None: - """Delete all messages that match a certain regex, stop cleaning after traversing `amount` messages.""" - use_cache = not channels - await self._clean_messages(amount, ctx, regex=regex, channels=channels, use_cache=use_cache) - - @clean_group.command(name="until") - @has_any_role(*MODERATION_ROLES) - async def clean_until(self, ctx: Context, message: Message) -> None: - """Delete all messages until certain message, stop cleaning after hitting the `message`.""" - await self._clean_messages( - CleanMessages.message_limit, - ctx, - channels=[message.channel], - until_message=message - ) - - @clean_group.command(name="from-to", aliases=["after-until", "between"]) - @has_any_role(*MODERATION_ROLES) - async def clean_from_to(self, ctx: Context, after_message: Message, until_message: Message) -> None: - """Delete all messages within range of messages.""" - await self._clean_messages( - CleanMessages.message_limit, - ctx, - channels=[until_message.channel], - until_message=until_message, - after_message=after_message, - ) - - @clean_group.command(name="stop", aliases=["cancel", "abort"]) - @has_any_role(*MODERATION_ROLES) - async def clean_cancel(self, ctx: Context) -> None: - """If there is an ongoing cleaning process, attempt to immediately cancel it.""" - self.cleaning = False - - embed = Embed( - color=Colour.blurple(), - description="Clean interrupted." - ) - await ctx.send(embed=embed, delete_after=10) - - -def setup(bot: Bot) -> None: - """Load the Clean cog.""" - bot.add_cog(Clean(bot)) -- cgit v1.2.3 From 2b5a5311110f52328651e5a19a186f4e3552ee84 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 27 Aug 2021 17:21:18 +0300 Subject: Move clean logging to a helper function --- bot/exts/moderation/clean.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 25582165a..e198dde9c 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -2,6 +2,7 @@ import logging import re import time from collections import defaultdict +from itertools import chain from typing import Any, Callable, DefaultDict, Iterable, List, Optional, Tuple from discord import Colour, Embed, Message, NotFound, TextChannel, User, errors @@ -263,25 +264,24 @@ class Clean(Cog): self.cleaning = False - log_messages = [] + await self._log_clean(list(chain.from_iterable(message_mappings.values())), channels, ctx.author) - for messages in message_mappings.values(): - log_messages.extend(messages) - - if log_messages: - # Reverse the list to restore chronological order - log_messages = reversed(log_messages) - log_url = await self.mod_log.upload_log(log_messages, ctx.author.id) - else: + async def _log_clean(self, messages: list[Message], channels: Iterable[TextChannel], invoker: User) -> None: + """Log the deleted messages to the modlog.""" + if not messages: # Can't build an embed, nothing to clean! raise BadArgument("No matching messages could be found.") + # Reverse the list to restore chronological order + log_messages = reversed(messages) + log_url = await self.mod_log.upload_log(log_messages, invoker.id) + # Build the embed and send it target_channels = ", ".join(channel.mention for channel in channels) message = ( - f"**{len(message_ids)}** messages deleted in {target_channels} by " - f"{ctx.author.mention}\n\n" + f"**{len(messages)}** messages deleted in {target_channels} by " + f"{invoker.mention}\n\n" f"A log of the deleted messages can be found [here]({log_url})." ) -- cgit v1.2.3 From 841a148f45bfe265815eac1aa9f22d84e332f548 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 27 Aug 2021 17:31:54 +0300 Subject: Move setting cleaning flag to correct line Between the concurrency check and setting the cleaning flag to True there was an await statement, which could potentially cause race conditions.The setting of the flag was moved to right below the concurrency check. --- bot/exts/moderation/clean.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index e198dde9c..2e3f9ac77 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -187,6 +187,7 @@ class Clean(Cog): # Are we already performing a clean? if self.cleaning: raise MaxConcurrencyReached("Please wait for the currently ongoing clean operation to complete.") + self.cleaning = True # Set up the correct predicate if bots_only: @@ -214,8 +215,6 @@ class Clean(Cog): # Invocation message has already been deleted log.info("Tried to delete invocation message, but it was already deleted.") - self.cleaning = True - if use_cache: message_mappings, message_ids = self._get_messages_from_cache(amount=amount, to_delete=predicate) else: -- cgit v1.2.3 From 8aeec5fc96bc5dcb8db1dbdbad870ade55ef5540 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 27 Aug 2021 17:33:07 +0300 Subject: Correct logging comment --- bot/exts/moderation/clean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 2e3f9ac77..19f64e0e7 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -271,7 +271,7 @@ class Clean(Cog): # Can't build an embed, nothing to clean! raise BadArgument("No matching messages could be found.") - # Reverse the list to restore chronological order + # Reverse the list to have reverse chronological order log_messages = reversed(messages) log_url = await self.mod_log.upload_log(log_messages, invoker.id) -- cgit v1.2.3 From 33e05017bfb0530a736fe1473f5e2b3c275f18f0 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 27 Aug 2021 17:37:09 +0300 Subject: Change `from-to` primary name to `between` --- bot/exts/moderation/clean.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 19f64e0e7..007aba317 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -357,9 +357,9 @@ class Clean(Cog): until_message=message ) - @clean_group.command(name="from-to", aliases=["after-until", "between"]) + @clean_group.command(name="between", aliases=["after-until", "from-to"]) @has_any_role(*MODERATION_ROLES) - async def clean_from_to(self, ctx: Context, after_message: Message, until_message: Message) -> None: + async def clean_between(self, ctx: Context, after_message: Message, until_message: Message) -> None: """Delete all messages within range of messages.""" await self._clean_messages( CleanMessages.message_limit, -- cgit v1.2.3 From f9d2e6919ab746b046c510daab2133e4b53bda6d Mon Sep 17 00:00:00 2001 From: mbaruh Date: Fri, 27 Aug 2021 17:52:13 +0300 Subject: Don't delete clean cancel embed in mod channel --- bot/exts/moderation/clean.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 007aba317..504ecccd1 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -379,7 +379,10 @@ class Clean(Cog): color=Colour.blurple(), description="Clean interrupted." ) - await ctx.send(embed=embed, delete_after=10) + delete_after = 10 + if is_mod_channel(ctx.channel): + delete_after = None + await ctx.send(embed=embed, delete_after=delete_after) def setup(bot: Bot) -> None: -- cgit v1.2.3 From d81b550594d02f25fcc310eb22cda5bd930fd197 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 28 Aug 2021 17:12:34 +0300 Subject: Change cache usage The cache is used only when all channels are used, as before. Unlike before, using all channels requires using "*" in the channels argument. Before all channels would be used if use_cache was set to True. Using all channels uses the cache by default. To traverse every single text channel in the server, setting use_cache to False is required in the command. --- bot/exts/moderation/clean.py | 64 +++++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 504ecccd1..15a48ea75 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -3,11 +3,11 @@ import re import time from collections import defaultdict from itertools import chain -from typing import Any, Callable, DefaultDict, Iterable, List, Optional, Tuple +from typing import Any, Callable, DefaultDict, Iterable, List, Literal, Optional, TYPE_CHECKING, Tuple, Union from discord import Colour, Embed, Message, NotFound, TextChannel, User, errors -from discord.ext import commands -from discord.ext.commands import Cog, Context, group, has_any_role +from discord.ext.commands import Cog, Context, Converter, group, has_any_role +from discord.ext.commands.converter import TextChannelConverter from discord.ext.commands.errors import BadArgument, MaxConcurrencyReached, MissingRequiredArgument from bot.bot import Bot @@ -23,6 +23,22 @@ log = logging.getLogger(__name__) Predicate = Callable[[Message], bool] +class CleanChannels(Converter): + """A converter that turns the given string to a list of channels to clean, or the literal `*` for all channels.""" + + _channel_converter = TextChannelConverter() + + async def convert(self, ctx: Context, argument: str) -> Union[Literal["*"], list[TextChannel]]: + """Converts a string to a list of channels to clean, or the literal `*` for all channels.""" + if argument == "*": + return "*" + return [await self._channel_converter.convert(ctx, channel) for channel in argument.split()] + + +if TYPE_CHECKING: + CleanChannels = Union[Literal["*"], list[TextChannel]] # noqa: F811 + + class Clean(Cog): """ A cog that allows messages to be deleted in bulk, while applying various filters. @@ -122,13 +138,13 @@ class Clean(Cog): self, amount: int, ctx: Context, - channels: Iterable[TextChannel], + channels: CleanChannels, bots_only: bool = False, - use_cache: bool = False, user: User = None, regex: Optional[str] = None, until_message: Optional[Message] = None, after_message: Optional[Message] = None, + use_cache: Optional[bool] = True ) -> None: """A helper function that does the actual message cleaning.""" def predicate_bots_only(message: Message) -> bool: @@ -215,12 +231,15 @@ class Clean(Cog): # Invocation message has already been deleted log.info("Tried to delete invocation message, but it was already deleted.") - if use_cache: + if channels == "*" and use_cache: message_mappings, message_ids = self._get_messages_from_cache(amount=amount, to_delete=predicate) else: + deletion_channels = channels + if channels == "*": + deletion_channels = [channel for channel in ctx.guild.channels if isinstance(channel, TextChannel)] message_mappings, message_ids = await self._get_messages_from_channels( amount=amount, - channels=channels, + channels=deletion_channels, to_delete=predicate, until_message=until_message ) @@ -265,7 +284,7 @@ class Clean(Cog): await self._log_clean(list(chain.from_iterable(message_mappings.values())), channels, ctx.author) - async def _log_clean(self, messages: list[Message], channels: Iterable[TextChannel], invoker: User) -> None: + async def _log_clean(self, messages: list[Message], channels: CleanChannels, invoker: User) -> None: """Log the deleted messages to the modlog.""" if not messages: # Can't build an embed, nothing to clean! @@ -276,7 +295,10 @@ class Clean(Cog): log_url = await self.mod_log.upload_log(log_messages, invoker.id) # Build the embed and send it - target_channels = ", ".join(channel.mention for channel in channels) + if channels == "*": + target_channels = "all channels" + else: + target_channels = ", ".join(channel.mention for channel in channels) message = ( f"**{len(messages)}** messages deleted in {target_channels} by " @@ -305,10 +327,11 @@ class Clean(Cog): ctx: Context, user: User, amount: Optional[int] = 10, - channels: commands.Greedy[TextChannel] = None + use_cache: Optional[bool] = True, + *, + channels: Optional[CleanChannels] = None ) -> None: """Delete messages posted by the provided user, stop cleaning after traversing `amount` messages.""" - use_cache = not channels await self._clean_messages(amount, ctx, user=user, channels=channels, use_cache=use_cache) @clean_group.command(name="all", aliases=["everything"]) @@ -317,10 +340,12 @@ class Clean(Cog): self, ctx: Context, amount: Optional[int] = 10, - channels: commands.Greedy[TextChannel] = None + use_cache: Optional[bool] = True, + *, + channels: Optional[CleanChannels] = None ) -> None: """Delete all messages, regardless of poster, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, channels=channels) + await self._clean_messages(amount, ctx, channels=channels, use_cache=use_cache) @clean_group.command(name="bots", aliases=["bot"]) @has_any_role(*MODERATION_ROLES) @@ -328,22 +353,25 @@ class Clean(Cog): self, ctx: Context, amount: Optional[int] = 10, - channels: commands.Greedy[TextChannel] = None + use_cache: Optional[bool] = True, + *, + channels: Optional[CleanChannels] = None ) -> None: """Delete all messages posted by a bot, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, bots_only=True, channels=channels) + await self._clean_messages(amount, ctx, bots_only=True, channels=channels, use_cache=use_cache) - @clean_group.command(name="regex", aliases=["word", "expression"]) + @clean_group.command(name="regex", aliases=["word", "expression", "pattern"]) @has_any_role(*MODERATION_ROLES) async def clean_regex( self, ctx: Context, regex: str, amount: Optional[int] = 10, - channels: commands.Greedy[TextChannel] = None + use_cache: Optional[bool] = True, + *, + channels: Optional[CleanChannels] = None ) -> None: """Delete all messages that match a certain regex, stop cleaning after traversing `amount` messages.""" - use_cache = not channels await self._clean_messages(amount, ctx, regex=regex, channels=channels, use_cache=use_cache) @clean_group.command(name="until") -- cgit v1.2.3 From 4334988a664bbb516760a6046a2e8106e9777eab Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 28 Aug 2021 17:30:14 +0300 Subject: Rename "amount" argument to "traverse" This name as been confusing moderators for a while now. "amount" sounds like this is the amount of messages that the bot will try to delete, and keep going until it reaches that number. In reality it's the amount of latest messages per channel the bot will traverse. Hopefully the new name conveys that better. --- bot/exts/moderation/clean.py | 48 ++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 15a48ea75..5954672fe 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -82,7 +82,7 @@ class Clean(Cog): # Message doesn't exist or was already deleted continue - def _get_messages_from_cache(self, amount: int, to_delete: Predicate) -> Tuple[DefaultDict, List[int]]: + def _get_messages_from_cache(self, traverse: int, to_delete: Predicate) -> Tuple[DefaultDict, List[int]]: """Helper function for getting messages from the cache.""" message_mappings = defaultdict(list) message_ids = [] @@ -95,16 +95,16 @@ class Clean(Cog): message_mappings[message.channel].append(message) message_ids.append(message.id) - if len(message_ids) == amount: - # We've got the requested amount of messages + if len(message_ids) == traverse: + # We traversed the requested amount of messages. return message_mappings, message_ids - # Amount exceeds amount of messages matching the check + # There are fewer messages in the cache than the number requested to traverse. return message_mappings, message_ids async def _get_messages_from_channels( self, - amount: int, + traverse: int, channels: Iterable[TextChannel], to_delete: Predicate, until_message: Optional[Message] = None @@ -114,7 +114,7 @@ class Clean(Cog): for channel in channels: - async for message in channel.history(limit=amount): + async for message in channel.history(limit=traverse): if not self.cleaning: # Cleaning was canceled, return empty containers @@ -136,7 +136,7 @@ class Clean(Cog): async def _clean_messages( self, - amount: int, + traverse: int, ctx: Context, channels: CleanChannels, bots_only: bool = False, @@ -183,9 +183,9 @@ class Clean(Cog): """Check if message is older than message provided in after_message but younger than until_message.""" return after_message.created_at <= message.created_at <= until_message.created_at - # Is this an acceptable amount of messages to clean? - if amount > CleanMessages.message_limit: - raise BadArgument(f"You cannot clean more than {CleanMessages.message_limit} messages.") + # Is this an acceptable amount of messages to traverse? + if traverse > CleanMessages.message_limit: + raise BadArgument(f"You cannot traverse more than {CleanMessages.message_limit} messages.") if after_message: # Ensure that until_message is specified. @@ -232,13 +232,13 @@ class Clean(Cog): log.info("Tried to delete invocation message, but it was already deleted.") if channels == "*" and use_cache: - message_mappings, message_ids = self._get_messages_from_cache(amount=amount, to_delete=predicate) + message_mappings, message_ids = self._get_messages_from_cache(traverse=traverse, to_delete=predicate) else: deletion_channels = channels if channels == "*": deletion_channels = [channel for channel in ctx.guild.channels if isinstance(channel, TextChannel)] message_mappings, message_ids = await self._get_messages_from_channels( - amount=amount, + traverse=traverse, channels=deletion_channels, to_delete=predicate, until_message=until_message @@ -326,39 +326,39 @@ class Clean(Cog): self, ctx: Context, user: User, - amount: Optional[int] = 10, + traverse: Optional[int] = 10, use_cache: Optional[bool] = True, *, channels: Optional[CleanChannels] = None ) -> None: - """Delete messages posted by the provided user, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, user=user, channels=channels, use_cache=use_cache) + """Delete messages posted by the provided user, stop cleaning after traversing `traverse` messages.""" + await self._clean_messages(traverse, ctx, user=user, channels=channels, use_cache=use_cache) @clean_group.command(name="all", aliases=["everything"]) @has_any_role(*MODERATION_ROLES) async def clean_all( self, ctx: Context, - amount: Optional[int] = 10, + traverse: Optional[int] = 10, use_cache: Optional[bool] = True, *, channels: Optional[CleanChannels] = None ) -> None: - """Delete all messages, regardless of poster, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, channels=channels, use_cache=use_cache) + """Delete all messages, regardless of poster, stop cleaning after traversing `traverse` messages.""" + await self._clean_messages(traverse, ctx, channels=channels, use_cache=use_cache) @clean_group.command(name="bots", aliases=["bot"]) @has_any_role(*MODERATION_ROLES) async def clean_bots( self, ctx: Context, - amount: Optional[int] = 10, + traverse: Optional[int] = 10, use_cache: Optional[bool] = True, *, channels: Optional[CleanChannels] = None ) -> None: - """Delete all messages posted by a bot, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, bots_only=True, channels=channels, use_cache=use_cache) + """Delete all messages posted by a bot, stop cleaning after traversing `traverse` messages.""" + await self._clean_messages(traverse, ctx, bots_only=True, channels=channels, use_cache=use_cache) @clean_group.command(name="regex", aliases=["word", "expression", "pattern"]) @has_any_role(*MODERATION_ROLES) @@ -366,13 +366,13 @@ class Clean(Cog): self, ctx: Context, regex: str, - amount: Optional[int] = 10, + traverse: Optional[int] = 10, use_cache: Optional[bool] = True, *, channels: Optional[CleanChannels] = None ) -> None: - """Delete all messages that match a certain regex, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, regex=regex, channels=channels, use_cache=use_cache) + """Delete all messages that match a certain regex, stop cleaning after traversing `traverse` messages.""" + await self._clean_messages(traverse, ctx, regex=regex, channels=channels, use_cache=use_cache) @clean_group.command(name="until") @has_any_role(*MODERATION_ROLES) -- cgit v1.2.3 From be9bce46ab5479d3fdbb8e7baa26f1ad947685f6 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 28 Aug 2021 23:09:20 +0300 Subject: Refactor code, correct logging This commit further splits the bulky _clean_messages function, and all its helper functions are grouped together in the same region. Additionally, this commit fixes logging by logging only the messages that have been successfully deleted, before being possibly interrupted by the cancel command. --- bot/exts/moderation/clean.py | 229 +++++++++++++++++++++++++------------------ 1 file changed, 133 insertions(+), 96 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 5954672fe..455d28faa 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -2,7 +2,6 @@ import logging import re import time from collections import defaultdict -from itertools import chain from typing import Any, Callable, DefaultDict, Iterable, List, Literal, Optional, TYPE_CHECKING, Tuple, Union from discord import Colour, Embed, Message, NotFound, TextChannel, User, errors @@ -59,28 +58,35 @@ class Clean(Cog): """Get currently loaded ModLog cog instance.""" return self.bot.get_cog("ModLog") + # region: Helper functions + @staticmethod - def is_older_than_14d(message: Message) -> bool: - """ - Precisely checks if message is older than 14 days, bulk deletion limit. + def _validate_input( + traverse: int, + channels: CleanChannels, + bots_only: bool, + user: User, + until_message: Message, + after_message: Message, + use_cache: bool + ) -> None: + """Raise errors if an argument value or a combination of values is invalid.""" + # Is this an acceptable amount of messages to traverse? + if traverse > CleanMessages.message_limit: + raise BadArgument(f"You cannot traverse more than {CleanMessages.message_limit} messages.") - Inspired by how purge works internally. - Comparison on message age could possibly be less accurate which in turn would resort in problems - with message deletion if said messages are very close to the 14d mark. - """ - two_weeks_old_snowflake = int((time.time() - 14 * 24 * 60 * 60) * 1000.0 - 1420070400000) << 22 - return message.id < two_weeks_old_snowflake + if after_message: + # Ensure that until_message is specified. + if not until_message: + raise MissingRequiredArgument("`until_message` must be specified if `after_message` is specified.") - async def _delete_messages_individually(self, messages: List[Message]) -> None: - for message in messages: - # Ensure that deletion was not canceled - if not self.cleaning: - return - try: - await message.delete() - except NotFound: - # Message doesn't exist or was already deleted - continue + # Messages are not in same channel + if after_message.channel != until_message.channel: + raise BadArgument("You cannot do range clean across several channel.") + + # Ensure that after_message is younger than until_message + if after_message.created_at >= until_message.created_at: + raise BadArgument("`after` message must be younger than `until` message") def _get_messages_from_cache(self, traverse: int, to_delete: Predicate) -> Tuple[DefaultDict, List[int]]: """Helper function for getting messages from the cache.""" @@ -134,6 +140,107 @@ class Clean(Cog): return message_mappings, message_ids + @staticmethod + def is_older_than_14d(message: Message) -> bool: + """ + Precisely checks if message is older than 14 days, bulk deletion limit. + + Inspired by how purge works internally. + Comparison on message age could possibly be less accurate which in turn would resort in problems + with message deletion if said messages are very close to the 14d mark. + """ + two_weeks_old_snowflake = int((time.time() - 14 * 24 * 60 * 60) * 1000.0 - 1420070400000) << 22 + return message.id < two_weeks_old_snowflake + + async def _delete_messages_individually(self, messages: List[Message]) -> list[Message]: + """Delete each message in the list unless cleaning is cancelled. Return the deleted messages.""" + deleted = [] + for message in messages: + # Ensure that deletion was not canceled + if not self.cleaning: + return deleted + try: + await message.delete() + except NotFound: + # Message doesn't exist or was already deleted + continue + else: + deleted.append(message) + return deleted + + async def _delete_found(self, message_mappings: dict[TextChannel, list[Message]]) -> list[Message]: + """ + Delete the detected messages. + + Deletion is made in bulk per channel for messages less than 14d old. + The function returns the deleted messages. + If cleaning was cancelled in the middle, return messages already deleted. + """ + deleted = [] + for channel, messages in message_mappings.items(): + to_delete = [] + + for current_index, message in enumerate(messages): + if not self.cleaning: + # Means that the cleaning was canceled + return deleted + + if self.is_older_than_14d(message): + # further messages are too old to be deleted in bulk + deleted_remaining = await self._delete_messages_individually(messages[current_index:]) + deleted.extend(deleted_remaining) + if not self.cleaning: + # Means that deletion was canceled while deleting the individual messages + return deleted + break + + to_delete.append(message) + + if len(to_delete) == 100: + # we can only delete up to 100 messages in a bulk + await channel.delete_messages(to_delete) + deleted.extend(to_delete) + to_delete.clear() + + if len(to_delete) > 0: + # deleting any leftover messages if there are any + await channel.delete_messages(to_delete) + deleted.extend(to_delete) + + return deleted + + async def _log_clean(self, messages: list[Message], channels: CleanChannels, invoker: User) -> None: + """Log the deleted messages to the modlog.""" + if not messages: + # Can't build an embed, nothing to clean! + raise BadArgument("No matching messages could be found.") + + # Reverse the list to have reverse chronological order + log_messages = reversed(messages) + log_url = await self.mod_log.upload_log(log_messages, invoker.id) + + # Build the embed and send it + if channels == "*": + target_channels = "all channels" + else: + target_channels = ", ".join(channel.mention for channel in channels) + + message = ( + f"**{len(messages)}** messages deleted in {target_channels} by " + f"{invoker.mention}\n\n" + f"A log of the deleted messages can be found [here]({log_url})." + ) + + await self.mod_log.send_log_message( + icon_url=Icons.message_bulk_delete, + colour=Colour(Colours.soft_red), + title="Bulk message delete", + text=message, + channel_id=Channels.mod_log, + ) + + # endregion + async def _clean_messages( self, traverse: int, @@ -183,22 +290,7 @@ class Clean(Cog): """Check if message is older than message provided in after_message but younger than until_message.""" return after_message.created_at <= message.created_at <= until_message.created_at - # Is this an acceptable amount of messages to traverse? - if traverse > CleanMessages.message_limit: - raise BadArgument(f"You cannot traverse more than {CleanMessages.message_limit} messages.") - - if after_message: - # Ensure that until_message is specified. - if not until_message: - raise MissingRequiredArgument("`until_message` must be specified if `after_message` is specified.") - - # Messages are not in same channel - if after_message.channel != until_message.channel: - raise BadArgument("You cannot do range clean across several channel.") - - # Ensure that after_message is younger than until_message - if after_message.created_at >= until_message.created_at: - raise BadArgument("`after` message must be younger than `until` message") + self._validate_input(traverse, channels, bots_only, user, until_message, after_message, use_cache) # Are we already performing a clean? if self.cleaning: @@ -250,69 +342,12 @@ class Clean(Cog): # Now let's delete the actual messages with purge. self.mod_log.ignore(Event.message_delete, *message_ids) - - for channel, messages in message_mappings.items(): - - to_delete = [] - - for current_index, message in enumerate(messages): - - if not self.cleaning: - # Means that the cleaning was canceled - return - - if self.is_older_than_14d(message): - # further messages are too old to be deleted in bulk - await self._delete_messages_individually(messages[current_index:]) - if not self.cleaning: - # Means that deletion was canceled while deleting the individual messages - return - break - - to_delete.append(message) - - if len(to_delete) == 100: - # we can only delete up to 100 messages in a bulk - await channel.delete_messages(to_delete) - to_delete.clear() - - if len(to_delete) > 0: - # deleting any leftover messages if there are any - await channel.delete_messages(to_delete) - + deleted_messages = await self._delete_found(message_mappings) self.cleaning = False - await self._log_clean(list(chain.from_iterable(message_mappings.values())), channels, ctx.author) - - async def _log_clean(self, messages: list[Message], channels: CleanChannels, invoker: User) -> None: - """Log the deleted messages to the modlog.""" - if not messages: - # Can't build an embed, nothing to clean! - raise BadArgument("No matching messages could be found.") - - # Reverse the list to have reverse chronological order - log_messages = reversed(messages) - log_url = await self.mod_log.upload_log(log_messages, invoker.id) + await self._log_clean(deleted_messages, channels, ctx.author) - # Build the embed and send it - if channels == "*": - target_channels = "all channels" - else: - target_channels = ", ".join(channel.mention for channel in channels) - - message = ( - f"**{len(messages)}** messages deleted in {target_channels} by " - f"{invoker.mention}\n\n" - f"A log of the deleted messages can be found [here]({log_url})." - ) - - await self.mod_log.send_log_message( - icon_url=Icons.message_bulk_delete, - colour=Colour(Colours.soft_red), - title="Bulk message delete", - text=message, - channel_id=Channels.mod_log, - ) + # region: Commands @group(invoke_without_command=True, name="clean", aliases=["clear", "purge"]) @has_any_role(*MODERATION_ROLES) @@ -412,6 +447,8 @@ class Clean(Cog): delete_after = None await ctx.send(embed=embed, delete_after=delete_after) + # endregion + def setup(bot: Bot) -> None: """Load the Clean cog.""" -- cgit v1.2.3 From 675630a620afe9dee4772bc659a16289be9665d7 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 28 Aug 2021 23:22:28 +0300 Subject: Add checkmark after command completes in mod channels --- bot/exts/moderation/clean.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 455d28faa..f8526b1b9 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -11,7 +11,7 @@ from discord.ext.commands.errors import BadArgument, MaxConcurrencyReached, Miss from bot.bot import Bot from bot.constants import ( - Channels, CleanMessages, Colours, Event, Icons, MODERATION_ROLES + Channels, CleanMessages, Colours, Emojis, Event, Icons, MODERATION_ROLES ) from bot.exts.moderation.modlog import ModLog from bot.utils.channel import is_mod_channel @@ -347,6 +347,9 @@ class Clean(Cog): await self._log_clean(deleted_messages, channels, ctx.author) + if is_mod_channel(ctx.channel): + await ctx.message.add_reaction(Emojis.check_mark) + # region: Commands @group(invoke_without_command=True, name="clean", aliases=["clear", "purge"]) -- cgit v1.2.3 From 7b0cb52bc05261200a03428a51a48813eb3ccf0b Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 28 Aug 2021 23:33:32 +0300 Subject: Send message when no messages found This commit changes the clean command to send a message instead of raising BadArgument when no messages are found. Not finding messages is not an error, and doesn't necessarily require the help embed to spring up, just that the parameters might need tweaking. --- bot/exts/moderation/clean.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index f8526b1b9..1d323fa0b 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -209,15 +209,17 @@ class Clean(Cog): return deleted - async def _log_clean(self, messages: list[Message], channels: CleanChannels, invoker: User) -> None: - """Log the deleted messages to the modlog.""" + async def _log_clean(self, messages: list[Message], channels: CleanChannels, ctx: Context) -> bool: + """Log the deleted messages to the modlog. Return True if logging was successful.""" if not messages: # Can't build an embed, nothing to clean! - raise BadArgument("No matching messages could be found.") + delete_after = None if is_mod_channel(ctx.channel) else 5 + await ctx.send(":x: No matching messages could be found.", delete_after=delete_after) + return False # Reverse the list to have reverse chronological order log_messages = reversed(messages) - log_url = await self.mod_log.upload_log(log_messages, invoker.id) + log_url = await self.mod_log.upload_log(log_messages, ctx.author.id) # Build the embed and send it if channels == "*": @@ -227,7 +229,7 @@ class Clean(Cog): message = ( f"**{len(messages)}** messages deleted in {target_channels} by " - f"{invoker.mention}\n\n" + f"{ctx.author.mention}\n\n" f"A log of the deleted messages can be found [here]({log_url})." ) @@ -239,6 +241,8 @@ class Clean(Cog): channel_id=Channels.mod_log, ) + return True + # endregion async def _clean_messages( @@ -345,9 +349,9 @@ class Clean(Cog): deleted_messages = await self._delete_found(message_mappings) self.cleaning = False - await self._log_clean(deleted_messages, channels, ctx.author) + logged = await self._log_clean(deleted_messages, channels, ctx) - if is_mod_channel(ctx.channel): + if logged and is_mod_channel(ctx.channel): await ctx.message.add_reaction(Emojis.check_mark) # region: Commands -- cgit v1.2.3 From 13308200ff62784832ba9f9084b69cd3a214b966 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 29 Aug 2021 02:10:43 +0300 Subject: `until` and `between` overhaul - The two subcommands can now accept a time delta and an ISO date time in addition to messages. - The two limits are now exclusive. Meaning cleaning until a message will not delete that message. - Added a separate predicate for the `until` case, as the combination of that command and cache usage would result in incorrect behavior. Additionally, deleting from cache now correctly traverses only `traverse` messages, rather than trying to delete `traverse` messages. --- bot/converters.py | 19 ++++++ bot/exts/moderation/clean.py | 145 +++++++++++++++++++++++++++---------------- 2 files changed, 111 insertions(+), 53 deletions(-) diff --git a/bot/converters.py b/bot/converters.py index 0118cc48a..546f6e8f4 100644 --- a/bot/converters.py +++ b/bot/converters.py @@ -388,6 +388,24 @@ class Duration(DurationDelta): raise BadArgument(f"`{duration}` results in a datetime outside the supported range.") +class Age(DurationDelta): + """Convert duration strings into UTC datetime.datetime objects.""" + + async def convert(self, ctx: Context, duration: str) -> datetime: + """ + Converts a `duration` string to a datetime object that's `duration` in the past. + + The converter supports the same symbols for each unit of time as its parent class. + """ + delta = await super().convert(ctx, duration) + now = datetime.utcnow() + + try: + return now - delta + except (ValueError, OverflowError): + raise BadArgument(f"`{duration}` results in a datetime outside the supported range.") + + class OffTopicName(Converter): """A converter that ensures an added off-topic name is valid.""" @@ -554,6 +572,7 @@ if t.TYPE_CHECKING: SourceConverter = SourceType # noqa: F811 DurationDelta = relativedelta # noqa: F811 Duration = datetime # noqa: F811 + Age = datetime # noqa: F811 OffTopicName = str # noqa: F811 ISODateTime = datetime # noqa: F811 HushDurationConverter = int # noqa: F811 diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 1d323fa0b..90f7f3e03 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -2,17 +2,20 @@ import logging import re import time from collections import defaultdict +from datetime import datetime +from itertools import islice from typing import Any, Callable, DefaultDict, Iterable, List, Literal, Optional, TYPE_CHECKING, Tuple, Union from discord import Colour, Embed, Message, NotFound, TextChannel, User, errors from discord.ext.commands import Cog, Context, Converter, group, has_any_role from discord.ext.commands.converter import TextChannelConverter -from discord.ext.commands.errors import BadArgument, MaxConcurrencyReached, MissingRequiredArgument +from discord.ext.commands.errors import BadArgument, MaxConcurrencyReached from bot.bot import Bot from bot.constants import ( Channels, CleanMessages, Colours, Emojis, Event, Icons, MODERATION_ROLES ) +from bot.converters import Age, ISODateTime from bot.exts.moderation.modlog import ModLog from bot.utils.channel import is_mod_channel @@ -21,6 +24,8 @@ log = logging.getLogger(__name__) # Type alias for checks Predicate = Callable[[Message], bool] +CleanLimit = Union[Message, Age, ISODateTime] + class CleanChannels(Converter): """A converter that turns the given string to a list of channels to clean, or the literal `*` for all channels.""" @@ -66,46 +71,40 @@ class Clean(Cog): channels: CleanChannels, bots_only: bool, user: User, - until_message: Message, - after_message: Message, + first_limit: CleanLimit, + second_limit: CleanLimit, use_cache: bool ) -> None: """Raise errors if an argument value or a combination of values is invalid.""" # Is this an acceptable amount of messages to traverse? if traverse > CleanMessages.message_limit: - raise BadArgument(f"You cannot traverse more than {CleanMessages.message_limit} messages.") + raise BadArgument(f"Cannot traverse more than {CleanMessages.message_limit} messages.") - if after_message: - # Ensure that until_message is specified. - if not until_message: - raise MissingRequiredArgument("`until_message` must be specified if `after_message` is specified.") + if (isinstance(first_limit, Message) or isinstance(first_limit, Message)) and channels: + raise BadArgument("Both a message limit and channels specified.") - # Messages are not in same channel - if after_message.channel != until_message.channel: - raise BadArgument("You cannot do range clean across several channel.") + if isinstance(first_limit, Message) and isinstance(second_limit, Message): + # Messages are not in same channel. + if first_limit.channel != second_limit.channel: + raise BadArgument("Message limits are in different channels.") - # Ensure that after_message is younger than until_message - if after_message.created_at >= until_message.created_at: - raise BadArgument("`after` message must be younger than `until` message") + # This is an implementation error rather than user error. + if second_limit and not first_limit: + raise ValueError("Second limit specified without the first.") def _get_messages_from_cache(self, traverse: int, to_delete: Predicate) -> Tuple[DefaultDict, List[int]]: """Helper function for getting messages from the cache.""" message_mappings = defaultdict(list) message_ids = [] - for message in self.bot.cached_messages: + for message in islice(self.bot.cached_messages, traverse): if not self.cleaning: # Cleaning was canceled - return (message_mappings, message_ids) + return message_mappings, message_ids if to_delete(message): message_mappings[message.channel].append(message) message_ids.append(message.id) - if len(message_ids) == traverse: - # We traversed the requested amount of messages. - return message_mappings, message_ids - - # There are fewer messages in the cache than the number requested to traverse. return message_mappings, message_ids async def _get_messages_from_channels( @@ -113,27 +112,19 @@ class Clean(Cog): traverse: int, channels: Iterable[TextChannel], to_delete: Predicate, - until_message: Optional[Message] = None + before: Optional[datetime] = None, + after: Optional[datetime] = None ) -> tuple[defaultdict[Any, list], list]: message_mappings = defaultdict(list) message_ids = [] for channel in channels: - - async for message in channel.history(limit=traverse): + async for message in channel.history(limit=traverse, before=before, after=after): if not self.cleaning: - # Cleaning was canceled, return empty containers + # Cleaning was canceled, return empty containers. return defaultdict(list), [] - if until_message: - - # We could use ID's here however in case if the message we are looking for gets deleted, - # We won't have a way to figure that out thus checking for datetime should be more reliable - if message.created_at < until_message.created_at: - # Means we have found the message until which we were supposed to be deleting. - break - if to_delete(message): message_mappings[message.channel].append(message) message_ids.append(message.id) @@ -253,8 +244,8 @@ class Clean(Cog): bots_only: bool = False, user: User = None, regex: Optional[str] = None, - until_message: Optional[Message] = None, - after_message: Optional[Message] = None, + first_limit: Optional[CleanLimit] = None, + second_limit: Optional[CleanLimit] = None, use_cache: Optional[bool] = True ) -> None: """A helper function that does the actual message cleaning.""" @@ -291,10 +282,14 @@ class Clean(Cog): return bool(re.search(regex.lower(), content.lower())) def predicate_range(message: Message) -> bool: - """Check if message is older than message provided in after_message but younger than until_message.""" - return after_message.created_at <= message.created_at <= until_message.created_at + """Check if the message age is between the two limits.""" + return first_limit <= message.created_at <= second_limit - self._validate_input(traverse, channels, bots_only, user, until_message, after_message, use_cache) + def predicate_after(message: Message) -> bool: + """Check if the message is older than the first limit.""" + return message.created_at >= first_limit + + self._validate_input(traverse, channels, bots_only, user, first_limit, second_limit, use_cache) # Are we already performing a clean? if self.cleaning: @@ -308,17 +303,31 @@ class Clean(Cog): predicate = predicate_specific_user # Delete messages from specific user elif regex: predicate = predicate_regex # Delete messages that match regex - elif after_message: - predicate = predicate_range # Delete messages older than specific message + elif second_limit: + predicate = predicate_range # Delete messages in the specified age range + elif first_limit: + predicate = predicate_after # Delete messages older than specific message else: predicate = lambda m: True # Delete all messages # noqa: E731 - # Default to using the invoking context's channel + # Default to using the invoking context's channel or the channel of the message limit(s). if not channels: - channels = [ctx.channel] + # At this point second_limit is guaranteed to not exist, be a datetime, or a message in the same channel. + if isinstance(first_limit, Message): + channels = [first_limit.channel] + elif isinstance(second_limit, Message): + channels = [second_limit.channel] + else: + channels = [ctx.channel] - if not is_mod_channel(ctx.channel): + if isinstance(first_limit, Message): + first_limit = first_limit.created_at + if isinstance(second_limit, Message): + second_limit = second_limit.created_at + if first_limit and second_limit: + first_limit, second_limit = sorted([first_limit, second_limit]) + if not is_mod_channel(ctx.channel): # Delete the invocation first self.mod_log.ignore(Event.message_delete, ctx.message.id) try: @@ -337,7 +346,8 @@ class Clean(Cog): traverse=traverse, channels=deletion_channels, to_delete=predicate, - until_message=until_message + before=second_limit, + after=first_limit # Remember first is the earlier datetime. ) if not self.cleaning: @@ -418,25 +428,54 @@ class Clean(Cog): @clean_group.command(name="until") @has_any_role(*MODERATION_ROLES) - async def clean_until(self, ctx: Context, message: Message) -> None: - """Delete all messages until certain message, stop cleaning after hitting the `message`.""" + async def clean_until( + self, + ctx: Context, + until: CleanLimit, + use_cache: Optional[bool] = True, + *, + channels: Optional[CleanChannels] = None) -> None: + """ + Delete all messages until a certain limit. + + A limit can be either a message, and ISO date-time string, or a time delta. + If a message is specified, `channels` cannot be specified. + """ await self._clean_messages( CleanMessages.message_limit, ctx, - channels=[message.channel], - until_message=message + channels=channels, + first_limit=until, + use_cache=use_cache ) @clean_group.command(name="between", aliases=["after-until", "from-to"]) @has_any_role(*MODERATION_ROLES) - async def clean_between(self, ctx: Context, after_message: Message, until_message: Message) -> None: - """Delete all messages within range of messages.""" + async def clean_between( + self, + ctx: Context, + first_limit: CleanLimit, + second_limit: CleanLimit, + use_cache: Optional[bool] = True, + *, + channels: Optional[CleanChannels] = None + ) -> None: + """ + Delete all messages within range. + + The range is specified through two limits. + A limit can be either a message, and ISO date-time string, or a time delta. + + If two messages are specified, they both must be in the same channel. + If a message is specified, `channels` cannot be specified. + """ await self._clean_messages( CleanMessages.message_limit, ctx, - channels=[until_message.channel], - until_message=until_message, - after_message=after_message, + channels=channels, + first_limit=first_limit, + second_limit=second_limit, + use_cache=use_cache ) @clean_group.command(name="stop", aliases=["cancel", "abort"]) -- cgit v1.2.3 From 9f124b9eefd24bd1e3bc7210361fe927e8f9eeba Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 29 Aug 2021 13:47:37 +0300 Subject: Restrict until and between to a single channel The subcommands should stay simple and answer the most common use cases. Deleting all messages within a time range across many channels seems esoteric and gives just more room for mistakes. --- bot/exts/moderation/clean.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 90f7f3e03..6c7f3c22d 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -432,21 +432,19 @@ class Clean(Cog): self, ctx: Context, until: CleanLimit, - use_cache: Optional[bool] = True, - *, - channels: Optional[CleanChannels] = None) -> None: + channel: Optional[TextChannel] = None + ) -> None: """ Delete all messages until a certain limit. A limit can be either a message, and ISO date-time string, or a time delta. - If a message is specified, `channels` cannot be specified. + If a message is specified, `channel` cannot be specified. """ await self._clean_messages( CleanMessages.message_limit, ctx, - channels=channels, + channels=[channel] if channel else None, first_limit=until, - use_cache=use_cache ) @clean_group.command(name="between", aliases=["after-until", "from-to"]) @@ -456,9 +454,7 @@ class Clean(Cog): ctx: Context, first_limit: CleanLimit, second_limit: CleanLimit, - use_cache: Optional[bool] = True, - *, - channels: Optional[CleanChannels] = None + channel: Optional[TextChannel] = None ) -> None: """ Delete all messages within range. @@ -467,15 +463,14 @@ class Clean(Cog): A limit can be either a message, and ISO date-time string, or a time delta. If two messages are specified, they both must be in the same channel. - If a message is specified, `channels` cannot be specified. + If a message is specified, `channel` cannot be specified. """ await self._clean_messages( CleanMessages.message_limit, ctx, - channels=channels, + channels=[channel] if channel else None, first_limit=first_limit, second_limit=second_limit, - use_cache=use_cache ) @clean_group.command(name="stop", aliases=["cancel", "abort"]) -- cgit v1.2.3 From ec8f06312d756325fff31d7735ea56465440ed57 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 29 Aug 2021 13:53:06 +0300 Subject: Use a cog-wide role check --- bot/exts/moderation/clean.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 6c7f3c22d..950c0c82e 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -367,13 +367,11 @@ class Clean(Cog): # region: Commands @group(invoke_without_command=True, name="clean", aliases=["clear", "purge"]) - @has_any_role(*MODERATION_ROLES) async def clean_group(self, ctx: Context) -> None: """Commands for cleaning messages in channels.""" await ctx.send_help(ctx.command) @clean_group.command(name="user", aliases=["users"]) - @has_any_role(*MODERATION_ROLES) async def clean_user( self, ctx: Context, @@ -387,7 +385,6 @@ class Clean(Cog): await self._clean_messages(traverse, ctx, user=user, channels=channels, use_cache=use_cache) @clean_group.command(name="all", aliases=["everything"]) - @has_any_role(*MODERATION_ROLES) async def clean_all( self, ctx: Context, @@ -400,7 +397,6 @@ class Clean(Cog): await self._clean_messages(traverse, ctx, channels=channels, use_cache=use_cache) @clean_group.command(name="bots", aliases=["bot"]) - @has_any_role(*MODERATION_ROLES) async def clean_bots( self, ctx: Context, @@ -413,7 +409,6 @@ class Clean(Cog): await self._clean_messages(traverse, ctx, bots_only=True, channels=channels, use_cache=use_cache) @clean_group.command(name="regex", aliases=["word", "expression", "pattern"]) - @has_any_role(*MODERATION_ROLES) async def clean_regex( self, ctx: Context, @@ -427,7 +422,6 @@ class Clean(Cog): await self._clean_messages(traverse, ctx, regex=regex, channels=channels, use_cache=use_cache) @clean_group.command(name="until") - @has_any_role(*MODERATION_ROLES) async def clean_until( self, ctx: Context, @@ -448,7 +442,6 @@ class Clean(Cog): ) @clean_group.command(name="between", aliases=["after-until", "from-to"]) - @has_any_role(*MODERATION_ROLES) async def clean_between( self, ctx: Context, @@ -474,7 +467,6 @@ class Clean(Cog): ) @clean_group.command(name="stop", aliases=["cancel", "abort"]) - @has_any_role(*MODERATION_ROLES) async def clean_cancel(self, ctx: Context) -> None: """If there is an ongoing cleaning process, attempt to immediately cancel it.""" self.cleaning = False @@ -490,6 +482,10 @@ class Clean(Cog): # endregion + async def cog_check(self, ctx: Context) -> bool: + """Only allow moderators to invoke the commands in this cog.""" + return await has_any_role(*MODERATION_ROLES).predicate(ctx) + def setup(bot: Bot) -> None: """Load the Clean cog.""" -- cgit v1.2.3 From fee4c0c8be7d0e7d0bcb8358bb11255feb3f66b8 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 29 Aug 2021 15:06:50 +0300 Subject: Handle reacted message being deleted --- bot/exts/moderation/clean.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 950c0c82e..6fb33c692 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -2,6 +2,7 @@ import logging import re import time from collections import defaultdict +from contextlib import suppress from datetime import datetime from itertools import islice from typing import Any, Callable, DefaultDict, Iterable, List, Literal, Optional, TYPE_CHECKING, Tuple, Union @@ -362,7 +363,8 @@ class Clean(Cog): logged = await self._log_clean(deleted_messages, channels, ctx) if logged and is_mod_channel(ctx.channel): - await ctx.message.add_reaction(Emojis.check_mark) + with suppress(NotFound): # Can happen if the invoker deleted their own messages. + await ctx.message.add_reaction(Emojis.check_mark) # region: Commands -- cgit v1.2.3 From ab155fb20ea77c4c7ab60e6368b76733662b93d7 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sun, 29 Aug 2021 17:47:27 +0300 Subject: Added master command The subcommands are kept simple and with few arguments, as they deal with most cases and their usage shouldn't be cumbersome. However we might to clean by criteria of several functionalities offered by the subcommands, for example delete a specific user's messages but only those that contain a certain pattern. For this reason the top-level command can now accept all arguments available in any of the subcommands, and will combine the criteria. Because the channels list has to be the last argument in order to accept either a list of channel or "*", I had to force a specific pattern in the regex argument for it to not consume the first channel specified. The regex argument must now have an "r" prefix and be enclosed in single quotes. For example: r'\d+'. For patterns with spaces the whole thing still needs to be enclosed in double quotes. For consistency the `clean regex` subcommand was changed to require the same. --- bot/exts/moderation/clean.py | 230 +++++++++++++++++++++++++++++-------------- 1 file changed, 156 insertions(+), 74 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 6fb33c692..bf018e8aa 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -8,7 +8,7 @@ from itertools import islice from typing import Any, Callable, DefaultDict, Iterable, List, Literal, Optional, TYPE_CHECKING, Tuple, Union from discord import Colour, Embed, Message, NotFound, TextChannel, User, errors -from discord.ext.commands import Cog, Context, Converter, group, has_any_role +from discord.ext.commands import Cog, Context, Converter, Greedy, group, has_any_role from discord.ext.commands.converter import TextChannelConverter from discord.ext.commands.errors import BadArgument, MaxConcurrencyReached @@ -22,6 +22,8 @@ from bot.utils.channel import is_mod_channel log = logging.getLogger(__name__) +DEFAULT_TRAVERSE = 10 + # Type alias for checks Predicate = Callable[[Message], bool] @@ -40,8 +42,17 @@ class CleanChannels(Converter): return [await self._channel_converter.convert(ctx, channel) for channel in argument.split()] +class Regex(Converter): + """A converter that takes a string in the form r'.+' and strips the 'r' prefix and the single quotes.""" + + async def convert(self, ctx: Context, argument: str) -> str: + """Strips the 'r' prefix and the enclosing single quotes from the string.""" + return re.match(r"r'(.+?)'", argument).group(1) + + if TYPE_CHECKING: CleanChannels = Union[Literal["*"], list[TextChannel]] # noqa: F811 + Regex = str # noqa: F811 class Clean(Cog): @@ -71,10 +82,9 @@ class Clean(Cog): traverse: int, channels: CleanChannels, bots_only: bool, - user: User, + users: list[User], first_limit: CleanLimit, second_limit: CleanLimit, - use_cache: bool ) -> None: """Raise errors if an argument value or a combination of values is invalid.""" # Is this an acceptable amount of messages to traverse? @@ -89,10 +99,85 @@ class Clean(Cog): if first_limit.channel != second_limit.channel: raise BadArgument("Message limits are in different channels.") + if users and bots_only: + raise BadArgument("Marked as bots only, but users were specified.") + # This is an implementation error rather than user error. if second_limit and not first_limit: raise ValueError("Second limit specified without the first.") + @staticmethod + def _build_predicate( + bots_only: bool = False, + users: list[User] = None, + regex: Optional[str] = None, + first_limit: Optional[datetime] = None, + second_limit: Optional[datetime] = None, + ) -> Predicate: + """Return the predicate that decides whether to delete a given message.""" + def predicate_bots_only(message: Message) -> bool: + """Return True if the message was sent by a bot.""" + return message.author.bot + + def predicate_specific_users(message: Message) -> bool: + """Return True if the message was sent by the user provided in the _clean_messages call.""" + return message.author in users + + def predicate_regex(message: Message) -> bool: + """Check if the regex provided in _clean_messages matches the message content or any embed attributes.""" + content = [message.content] + + # Add the content for all embed attributes + for embed in message.embeds: + content.append(embed.title) + content.append(embed.description) + content.append(embed.footer.text) + content.append(embed.author.name) + for field in embed.fields: + content.append(field.name) + content.append(field.value) + + # Get rid of empty attributes and turn it into a string + content = [attr for attr in content if attr] + content = "\n".join(content) + + # Now let's see if there's a regex match + if not content: + return False + else: + return bool(re.search(regex.lower(), content.lower())) + + def predicate_range(message: Message) -> bool: + """Check if the message age is between the two limits.""" + return first_limit <= message.created_at <= second_limit + + def predicate_after(message: Message) -> bool: + """Check if the message is older than the first limit.""" + return message.created_at >= first_limit + + predicates = [] + # Set up the correct predicate + if bots_only: + predicates.append(predicate_bots_only) # Delete messages from bots + if users: + predicates.append(predicate_specific_users) # Delete messages from specific user + if regex: + predicates.append(predicate_regex) # Delete messages that match regex + # Add up to one of the following: + if second_limit: + predicates.append(predicate_range) # Delete messages in the specified age range + elif first_limit: + predicates.append(predicate_after) # Delete messages older than specific message + + if not predicates: + predicate = lambda m: True # Delete all messages # noqa: E731 + elif len(predicates) == 1: + predicate = predicates[0] + else: + predicate = lambda m: all(pred(m) for pred in predicates) # noqa: E731 + + return predicate + def _get_messages_from_cache(self, traverse: int, to_delete: Predicate) -> Tuple[DefaultDict, List[int]]: """Helper function for getting messages from the cache.""" message_mappings = defaultdict(list) @@ -239,78 +324,24 @@ class Clean(Cog): async def _clean_messages( self, - traverse: int, ctx: Context, + traverse: int, channels: CleanChannels, bots_only: bool = False, - user: User = None, + users: list[User] = None, regex: Optional[str] = None, first_limit: Optional[CleanLimit] = None, second_limit: Optional[CleanLimit] = None, use_cache: Optional[bool] = True ) -> None: """A helper function that does the actual message cleaning.""" - def predicate_bots_only(message: Message) -> bool: - """Return True if the message was sent by a bot.""" - return message.author.bot - - def predicate_specific_user(message: Message) -> bool: - """Return True if the message was sent by the user provided in the _clean_messages call.""" - return message.author == user - - def predicate_regex(message: Message) -> bool: - """Check if the regex provided in _clean_messages matches the message content or any embed attributes.""" - content = [message.content] - - # Add the content for all embed attributes - for embed in message.embeds: - content.append(embed.title) - content.append(embed.description) - content.append(embed.footer.text) - content.append(embed.author.name) - for field in embed.fields: - content.append(field.name) - content.append(field.value) - - # Get rid of empty attributes and turn it into a string - content = [attr for attr in content if attr] - content = "\n".join(content) - - # Now let's see if there's a regex match - if not content: - return False - else: - return bool(re.search(regex.lower(), content.lower())) - - def predicate_range(message: Message) -> bool: - """Check if the message age is between the two limits.""" - return first_limit <= message.created_at <= second_limit - - def predicate_after(message: Message) -> bool: - """Check if the message is older than the first limit.""" - return message.created_at >= first_limit - - self._validate_input(traverse, channels, bots_only, user, first_limit, second_limit, use_cache) + self._validate_input(traverse, channels, bots_only, users, first_limit, second_limit) # Are we already performing a clean? if self.cleaning: raise MaxConcurrencyReached("Please wait for the currently ongoing clean operation to complete.") self.cleaning = True - # Set up the correct predicate - if bots_only: - predicate = predicate_bots_only # Delete messages from bots - elif user: - predicate = predicate_specific_user # Delete messages from specific user - elif regex: - predicate = predicate_regex # Delete messages that match regex - elif second_limit: - predicate = predicate_range # Delete messages in the specified age range - elif first_limit: - predicate = predicate_after # Delete messages older than specific message - else: - predicate = lambda m: True # Delete all messages # noqa: E731 - # Default to using the invoking context's channel or the channel of the message limit(s). if not channels: # At this point second_limit is guaranteed to not exist, be a datetime, or a message in the same channel. @@ -328,6 +359,9 @@ class Clean(Cog): if first_limit and second_limit: first_limit, second_limit = sorted([first_limit, second_limit]) + # Needs to be called after standardizing the input. + predicate = self._build_predicate(bots_only, users, regex, first_limit, second_limit) + if not is_mod_channel(ctx.channel): # Delete the invocation first self.mod_log.ignore(Event.message_delete, ctx.message.id) @@ -369,9 +403,51 @@ class Clean(Cog): # region: Commands @group(invoke_without_command=True, name="clean", aliases=["clear", "purge"]) - async def clean_group(self, ctx: Context) -> None: - """Commands for cleaning messages in channels.""" - await ctx.send_help(ctx.command) + async def clean_group( + self, + ctx: Context, + traverse: Optional[int] = None, + users: Greedy[User] = None, + first_limit: Optional[CleanLimit] = None, + second_limit: Optional[CleanLimit] = None, + use_cache: Optional[bool] = None, + bots_only: Optional[bool] = False, + regex: Optional[Regex] = None, + *, + channels: Optional[CleanChannels] = None + ) -> None: + """ + Commands for cleaning messages in channels. + + If arguments are provided, will act as a master command from which all subcommands can be derived. + `traverse`: The number of messages to look at in each channel. + `users`: A series of user mentions, ID's, or names. + `first_limit` and `second_limit`: A message, a duration delta, or an ISO datetime. + If a message is provided, cleaning will happen in that channel, and channels cannot be provided. + If only one of them is provided, acts as `clean until`. If both are provided, acts as `clean between`. + `use_cache`: Whether to use the message cache. + If not provided, will default to False unless an asterisk is used for the channels. + `bots_only`: Whether to delete only bots. If specified, users cannot be specified. + `regex`: A regex pattern the message must contain to be deleted. + The pattern must be provided with an "r" prefix and enclosed in single quotes. + If the pattern contains spaces, it still needs to be enclosed in double quotes on top of that. + `channels`: A series of channels to delete in, or an asterisk to delete from all channels. + """ + if not any([traverse, users, first_limit, second_limit, regex]): + await ctx.send_help(ctx.command) + return + + if not traverse: + if first_limit: + traverse = CleanMessages.message_limit + else: + traverse = DEFAULT_TRAVERSE + if not use_cache: + use_cache = channels == "*" + + await self._clean_messages( + ctx, traverse, channels, bots_only, users, regex, first_limit, second_limit, use_cache + ) @clean_group.command(name="user", aliases=["users"]) async def clean_user( @@ -384,44 +460,50 @@ class Clean(Cog): channels: Optional[CleanChannels] = None ) -> None: """Delete messages posted by the provided user, stop cleaning after traversing `traverse` messages.""" - await self._clean_messages(traverse, ctx, user=user, channels=channels, use_cache=use_cache) + await self._clean_messages(ctx, traverse, users=[user], channels=channels, use_cache=use_cache) @clean_group.command(name="all", aliases=["everything"]) async def clean_all( self, ctx: Context, - traverse: Optional[int] = 10, + traverse: Optional[int] = DEFAULT_TRAVERSE, use_cache: Optional[bool] = True, *, channels: Optional[CleanChannels] = None ) -> None: """Delete all messages, regardless of poster, stop cleaning after traversing `traverse` messages.""" - await self._clean_messages(traverse, ctx, channels=channels, use_cache=use_cache) + await self._clean_messages(ctx, traverse, channels=channels, use_cache=use_cache) @clean_group.command(name="bots", aliases=["bot"]) async def clean_bots( self, ctx: Context, - traverse: Optional[int] = 10, + traverse: Optional[int] = DEFAULT_TRAVERSE, use_cache: Optional[bool] = True, *, channels: Optional[CleanChannels] = None ) -> None: """Delete all messages posted by a bot, stop cleaning after traversing `traverse` messages.""" - await self._clean_messages(traverse, ctx, bots_only=True, channels=channels, use_cache=use_cache) + await self._clean_messages(ctx, traverse, bots_only=True, channels=channels, use_cache=use_cache) @clean_group.command(name="regex", aliases=["word", "expression", "pattern"]) async def clean_regex( self, ctx: Context, - regex: str, - traverse: Optional[int] = 10, + regex: Regex, + traverse: Optional[int] = DEFAULT_TRAVERSE, use_cache: Optional[bool] = True, *, channels: Optional[CleanChannels] = None ) -> None: - """Delete all messages that match a certain regex, stop cleaning after traversing `traverse` messages.""" - await self._clean_messages(traverse, ctx, regex=regex, channels=channels, use_cache=use_cache) + """ + Delete all messages that match a certain regex, stop cleaning after traversing `traverse` messages. + + The pattern must be provided with an "r" prefix and enclosed in single quotes. + If the pattern contains spaces, and still needs to be enclosed in double quotes on top of that. + For example: r'[0-9]+' + """ + await self._clean_messages(ctx, traverse, regex=regex, channels=channels, use_cache=use_cache) @clean_group.command(name="until") async def clean_until( @@ -437,8 +519,8 @@ class Clean(Cog): If a message is specified, `channel` cannot be specified. """ await self._clean_messages( - CleanMessages.message_limit, ctx, + CleanMessages.message_limit, channels=[channel] if channel else None, first_limit=until, ) @@ -461,8 +543,8 @@ class Clean(Cog): If a message is specified, `channel` cannot be specified. """ await self._clean_messages( - CleanMessages.message_limit, ctx, + CleanMessages.message_limit, channels=[channel] if channel else None, first_limit=first_limit, second_limit=second_limit, -- cgit v1.2.3 From f2fb9f3dd449d58162471525ecaccc6db7d721f0 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 31 Aug 2021 20:49:15 +0300 Subject: Disallow time range cleaning in multiple channels Cleaning in the same time range across several channels seems like an arbitrary decision. --- bot/exts/moderation/clean.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index bf018e8aa..1148b3eb5 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -91,6 +91,9 @@ class Clean(Cog): if traverse > CleanMessages.message_limit: raise BadArgument(f"Cannot traverse more than {CleanMessages.message_limit} messages.") + if first_limit and channels and (channels == "*" or len(channels) > 1): + raise BadArgument("Message or time range specified across multiple channels.") + if (isinstance(first_limit, Message) or isinstance(first_limit, Message)) and channels: raise BadArgument("Both a message limit and channels specified.") @@ -424,6 +427,7 @@ class Clean(Cog): `users`: A series of user mentions, ID's, or names. `first_limit` and `second_limit`: A message, a duration delta, or an ISO datetime. If a message is provided, cleaning will happen in that channel, and channels cannot be provided. + If a limit is provided, multiple channels cannot be provided. If only one of them is provided, acts as `clean until`. If both are provided, acts as `clean between`. `use_cache`: Whether to use the message cache. If not provided, will default to False unless an asterisk is used for the channels. -- cgit v1.2.3 From 3ccb533686d464e11bf330ac19900a9f6cfc4366 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 31 Aug 2021 22:06:53 +0300 Subject: Changed regex formatting to wrapped in backticks After discussion, backticks seems like the preferrable formatting as it also cancels Discord's formatting. Additionally removed the Optionals from the last args in the commands, to not silently ignore incorrect input. --- bot/exts/moderation/clean.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 1148b3eb5..5b64693cc 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -43,11 +43,11 @@ class CleanChannels(Converter): class Regex(Converter): - """A converter that takes a string in the form r'.+' and strips the 'r' prefix and the single quotes.""" + """A converter that takes a string in the form `.+` and returns the contents of the inline code.""" async def convert(self, ctx: Context, argument: str) -> str: - """Strips the 'r' prefix and the enclosing single quotes from the string.""" - return re.match(r"r'(.+?)'", argument).group(1) + """Strips the backticks from the string.""" + return re.fullmatch(r"`(.+?)`", argument).group(1) if TYPE_CHECKING: @@ -417,7 +417,7 @@ class Clean(Cog): bots_only: Optional[bool] = False, regex: Optional[Regex] = None, *, - channels: Optional[CleanChannels] = None + channels: CleanChannels = None ) -> None: """ Commands for cleaning messages in channels. @@ -433,11 +433,11 @@ class Clean(Cog): If not provided, will default to False unless an asterisk is used for the channels. `bots_only`: Whether to delete only bots. If specified, users cannot be specified. `regex`: A regex pattern the message must contain to be deleted. - The pattern must be provided with an "r" prefix and enclosed in single quotes. + The pattern must be provided enclosed in backticks. If the pattern contains spaces, it still needs to be enclosed in double quotes on top of that. `channels`: A series of channels to delete in, or an asterisk to delete from all channels. """ - if not any([traverse, users, first_limit, second_limit, regex]): + if not any([traverse, users, first_limit, second_limit, regex, channels]): await ctx.send_help(ctx.command) return @@ -461,7 +461,7 @@ class Clean(Cog): traverse: Optional[int] = 10, use_cache: Optional[bool] = True, *, - channels: Optional[CleanChannels] = None + channels: CleanChannels = None ) -> None: """Delete messages posted by the provided user, stop cleaning after traversing `traverse` messages.""" await self._clean_messages(ctx, traverse, users=[user], channels=channels, use_cache=use_cache) @@ -473,7 +473,7 @@ class Clean(Cog): traverse: Optional[int] = DEFAULT_TRAVERSE, use_cache: Optional[bool] = True, *, - channels: Optional[CleanChannels] = None + channels: CleanChannels = None ) -> None: """Delete all messages, regardless of poster, stop cleaning after traversing `traverse` messages.""" await self._clean_messages(ctx, traverse, channels=channels, use_cache=use_cache) @@ -485,7 +485,7 @@ class Clean(Cog): traverse: Optional[int] = DEFAULT_TRAVERSE, use_cache: Optional[bool] = True, *, - channels: Optional[CleanChannels] = None + channels: CleanChannels = None ) -> None: """Delete all messages posted by a bot, stop cleaning after traversing `traverse` messages.""" await self._clean_messages(ctx, traverse, bots_only=True, channels=channels, use_cache=use_cache) @@ -498,14 +498,14 @@ class Clean(Cog): traverse: Optional[int] = DEFAULT_TRAVERSE, use_cache: Optional[bool] = True, *, - channels: Optional[CleanChannels] = None + channels: CleanChannels = None ) -> None: """ Delete all messages that match a certain regex, stop cleaning after traversing `traverse` messages. - The pattern must be provided with an "r" prefix and enclosed in single quotes. + The pattern must be provided enclosed in backticks. If the pattern contains spaces, and still needs to be enclosed in double quotes on top of that. - For example: r'[0-9]+' + For example: `[0-9]` """ await self._clean_messages(ctx, traverse, regex=regex, channels=channels, use_cache=use_cache) @@ -514,7 +514,7 @@ class Clean(Cog): self, ctx: Context, until: CleanLimit, - channel: Optional[TextChannel] = None + channel: TextChannel = None ) -> None: """ Delete all messages until a certain limit. @@ -535,7 +535,7 @@ class Clean(Cog): ctx: Context, first_limit: CleanLimit, second_limit: CleanLimit, - channel: Optional[TextChannel] = None + channel: TextChannel = None ) -> None: """ Delete all messages within range. -- cgit v1.2.3 From f5d7a006e4a0ebc1be0fd79be76eaf3501c6521a Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 7 Sep 2021 02:13:57 +0300 Subject: Code and comments polish Co-authored-by: Shivansh-007 --- bot/exts/moderation/clean.py | 46 ++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 5b64693cc..a9f936d88 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -5,7 +5,7 @@ from collections import defaultdict from contextlib import suppress from datetime import datetime from itertools import islice -from typing import Any, Callable, DefaultDict, Iterable, List, Literal, Optional, TYPE_CHECKING, Tuple, Union +from typing import Any, Callable, DefaultDict, Iterable, Literal, Optional, TYPE_CHECKING, Union from discord import Colour, Embed, Message, NotFound, TextChannel, User, errors from discord.ext.commands import Cog, Context, Converter, Greedy, group, has_any_role @@ -57,7 +57,7 @@ if TYPE_CHECKING: class Clean(Cog): """ - A cog that allows messages to be deleted in bulk, while applying various filters. + A cog that allows messages to be deleted in bulk while applying various filters. You can delete messages sent by a specific user, messages sent by bots, all messages, or messages that match a specific regular expression. @@ -94,7 +94,7 @@ class Clean(Cog): if first_limit and channels and (channels == "*" or len(channels) > 1): raise BadArgument("Message or time range specified across multiple channels.") - if (isinstance(first_limit, Message) or isinstance(first_limit, Message)) and channels: + if (isinstance(first_limit, Message) or isinstance(second_limit, Message)) and channels: raise BadArgument("Both a message limit and channels specified.") if isinstance(first_limit, Message) and isinstance(second_limit, Message): @@ -141,8 +141,7 @@ class Clean(Cog): content.append(field.value) # Get rid of empty attributes and turn it into a string - content = [attr for attr in content if attr] - content = "\n".join(content) + content = "\n".join(attr for attr in content if attr) # Now let's see if there's a regex match if not content: @@ -173,15 +172,12 @@ class Clean(Cog): predicates.append(predicate_after) # Delete messages older than specific message if not predicates: - predicate = lambda m: True # Delete all messages # noqa: E731 - elif len(predicates) == 1: - predicate = predicates[0] - else: - predicate = lambda m: all(pred(m) for pred in predicates) # noqa: E731 - - return predicate + return lambda m: True + if len(predicates) == 1: + return predicates[0] + return lambda m: all(pred(m) for pred in predicates) - def _get_messages_from_cache(self, traverse: int, to_delete: Predicate) -> Tuple[DefaultDict, List[int]]: + def _get_messages_from_cache(self, traverse: int, to_delete: Predicate) -> tuple[DefaultDict, list[int]]: """Helper function for getting messages from the cache.""" message_mappings = defaultdict(list) message_ids = [] @@ -232,7 +228,7 @@ class Clean(Cog): two_weeks_old_snowflake = int((time.time() - 14 * 24 * 60 * 60) * 1000.0 - 1420070400000) << 22 return message.id < two_weeks_old_snowflake - async def _delete_messages_individually(self, messages: List[Message]) -> list[Message]: + async def _delete_messages_individually(self, messages: list[Message]) -> list[Message]: """Delete each message in the list unless cleaning is cancelled. Return the deleted messages.""" deleted = [] for message in messages: @@ -289,7 +285,7 @@ class Clean(Cog): return deleted - async def _log_clean(self, messages: list[Message], channels: CleanChannels, ctx: Context) -> bool: + async def _modlog_cleaned_messages(self, messages: list[Message], channels: CleanChannels, ctx: Context) -> bool: """Log the deleted messages to the modlog. Return True if logging was successful.""" if not messages: # Can't build an embed, nothing to clean! @@ -347,7 +343,7 @@ class Clean(Cog): # Default to using the invoking context's channel or the channel of the message limit(s). if not channels: - # At this point second_limit is guaranteed to not exist, be a datetime, or a message in the same channel. + # Input was validated - if first_limit is a message, second_limit won't point at a different channel. if isinstance(first_limit, Message): channels = [first_limit.channel] elif isinstance(second_limit, Message): @@ -397,7 +393,7 @@ class Clean(Cog): deleted_messages = await self._delete_found(message_mappings) self.cleaning = False - logged = await self._log_clean(deleted_messages, channels, ctx) + logged = await self._modlog_cleaned_messages(deleted_messages, channels, ctx) if logged and is_mod_channel(ctx.channel): with suppress(NotFound): # Can happen if the invoker deleted their own messages. @@ -417,25 +413,25 @@ class Clean(Cog): bots_only: Optional[bool] = False, regex: Optional[Regex] = None, *, - channels: CleanChannels = None + channels: CleanChannels = None # "Optional" with discord.py silently ignores incorrect input. ) -> None: """ Commands for cleaning messages in channels. If arguments are provided, will act as a master command from which all subcommands can be derived. - `traverse`: The number of messages to look at in each channel. - `users`: A series of user mentions, ID's, or names. - `first_limit` and `second_limit`: A message, a duration delta, or an ISO datetime. + • `traverse`: The number of messages to look at in each channel. + • `users`: A series of user mentions, ID's, or names. + • `first_limit` and `second_limit`: A message, a duration delta, or an ISO datetime. If a message is provided, cleaning will happen in that channel, and channels cannot be provided. If a limit is provided, multiple channels cannot be provided. If only one of them is provided, acts as `clean until`. If both are provided, acts as `clean between`. - `use_cache`: Whether to use the message cache. + • `use_cache`: Whether to use the message cache. If not provided, will default to False unless an asterisk is used for the channels. - `bots_only`: Whether to delete only bots. If specified, users cannot be specified. - `regex`: A regex pattern the message must contain to be deleted. + • `bots_only`: Whether to delete only bots. If specified, users cannot be specified. + • `regex`: A regex pattern the message must contain to be deleted. The pattern must be provided enclosed in backticks. If the pattern contains spaces, it still needs to be enclosed in double quotes on top of that. - `channels`: A series of channels to delete in, or an asterisk to delete from all channels. + • `channels`: A series of channels to delete in, or an asterisk to delete from all channels. """ if not any([traverse, users, first_limit, second_limit, regex, channels]): await ctx.send_help(ctx.command) -- cgit v1.2.3 From ed30eae8b29ad9863a297db541bb1b9fdaf9ab1e Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 7 Sep 2021 20:07:02 +0300 Subject: Fix regex search The regex was lowercased, even though regex patterns are case sensitive. Also adds the DOTALL flag. --- bot/exts/moderation/clean.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index a9f936d88..ca458f066 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -144,10 +144,7 @@ class Clean(Cog): content = "\n".join(attr for attr in content if attr) # Now let's see if there's a regex match - if not content: - return False - else: - return bool(re.search(regex.lower(), content.lower())) + return bool(re.search(regex, content, re.IGNORECASE + re.DOTALL)) def predicate_range(message: Message) -> bool: """Check if the message age is between the two limits.""" -- cgit v1.2.3 From c992b6eacd47b67ba731c229ac0e6ab8df63d25f Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 7 Sep 2021 21:43:10 +0300 Subject: Improve responses - Tells the user if clean cancel was attempted with no ongoing clean. - Fixes MaxConcurrencyReached call bug. There was a missing argument, and it shouldn't invoke the help embed anyway, so it's now a message. - Some code refactoring. --- bot/exts/moderation/clean.py | 55 ++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index ca458f066..7a24833fe 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -7,10 +7,10 @@ from datetime import datetime from itertools import islice from typing import Any, Callable, DefaultDict, Iterable, Literal, Optional, TYPE_CHECKING, Union -from discord import Colour, Embed, Message, NotFound, TextChannel, User, errors +from discord import Colour, Message, NotFound, TextChannel, User, errors from discord.ext.commands import Cog, Context, Converter, Greedy, group, has_any_role from discord.ext.commands.converter import TextChannelConverter -from discord.ext.commands.errors import BadArgument, MaxConcurrencyReached +from discord.ext.commands.errors import BadArgument from bot.bot import Bot from bot.constants import ( @@ -23,6 +23,7 @@ from bot.utils.channel import is_mod_channel log = logging.getLogger(__name__) DEFAULT_TRAVERSE = 10 +MESSAGE_DELETE_DELAY = 5 # Type alias for checks Predicate = Callable[[Message], bool] @@ -109,6 +110,12 @@ class Clean(Cog): if second_limit and not first_limit: raise ValueError("Second limit specified without the first.") + @staticmethod + async def _send_expiring_message(ctx: Context, content: str) -> None: + """Send `content` to the context channel. Automatically delete if it's not a mod channel.""" + delete_after = None if is_mod_channel(ctx.channel) else MESSAGE_DELETE_DELAY + await ctx.send(content, delete_after=delete_after) + @staticmethod def _build_predicate( bots_only: bool = False, @@ -174,6 +181,16 @@ class Clean(Cog): return predicates[0] return lambda m: all(pred(m) for pred in predicates) + async def _delete_invocation(self, ctx: Context) -> None: + """Delete the command invocation if it's not in a mod channel.""" + if not is_mod_channel(ctx.channel): + self.mod_log.ignore(Event.message_delete, ctx.message.id) + try: + await ctx.message.delete() + except errors.NotFound: + # Invocation message has already been deleted + log.info("Tried to delete invocation message, but it was already deleted.") + def _get_messages_from_cache(self, traverse: int, to_delete: Predicate) -> tuple[DefaultDict, list[int]]: """Helper function for getting messages from the cache.""" message_mappings = defaultdict(list) @@ -286,8 +303,7 @@ class Clean(Cog): """Log the deleted messages to the modlog. Return True if logging was successful.""" if not messages: # Can't build an embed, nothing to clean! - delete_after = None if is_mod_channel(ctx.channel) else 5 - await ctx.send(":x: No matching messages could be found.", delete_after=delete_after) + await self._send_expiring_message(ctx, ":x: No matching messages could be found.") return False # Reverse the list to have reverse chronological order @@ -335,7 +351,10 @@ class Clean(Cog): # Are we already performing a clean? if self.cleaning: - raise MaxConcurrencyReached("Please wait for the currently ongoing clean operation to complete.") + await self._send_expiring_message( + ctx, ":x: Please wait for the currently ongoing clean operation to complete." + ) + return self.cleaning = True # Default to using the invoking context's channel or the channel of the message limit(s). @@ -358,14 +377,8 @@ class Clean(Cog): # Needs to be called after standardizing the input. predicate = self._build_predicate(bots_only, users, regex, first_limit, second_limit) - if not is_mod_channel(ctx.channel): - # Delete the invocation first - self.mod_log.ignore(Event.message_delete, ctx.message.id) - try: - await ctx.message.delete() - except errors.NotFound: - # Invocation message has already been deleted - log.info("Tried to delete invocation message, but it was already deleted.") + # Delete the invocation first + await self._delete_invocation(ctx) if channels == "*" and use_cache: message_mappings, message_ids = self._get_messages_from_cache(traverse=traverse, to_delete=predicate) @@ -550,16 +563,14 @@ class Clean(Cog): @clean_group.command(name="stop", aliases=["cancel", "abort"]) async def clean_cancel(self, ctx: Context) -> None: """If there is an ongoing cleaning process, attempt to immediately cancel it.""" - self.cleaning = False + if not self.cleaning: + message = ":question: There's no cleaning going on." + else: + self.cleaning = False + message = f"{Emojis.check_mark} Clean interrupted." - embed = Embed( - color=Colour.blurple(), - description="Clean interrupted." - ) - delete_after = 10 - if is_mod_channel(ctx.channel): - delete_after = None - await ctx.send(embed=embed, delete_after=delete_after) + await self._send_expiring_message(ctx, message) + await self._delete_invocation(ctx) # endregion -- cgit v1.2.3 From 5b8e16bb9e0226e40173a84da4e103d9960b8839 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Tue, 7 Sep 2021 22:54:43 +0300 Subject: Fix delete order In case of old messages, it would delete the old messages first, and only then bulk delete the remainder, which affected logging. This commit corrects the deletion order. --- bot/exts/moderation/clean.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 7a24833fe..c90aff256 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -270,33 +270,38 @@ class Clean(Cog): for channel, messages in message_mappings.items(): to_delete = [] - for current_index, message in enumerate(messages): + delete_old = False + for current_index, message in enumerate(messages): # noqa: B007 if not self.cleaning: # Means that the cleaning was canceled return deleted if self.is_older_than_14d(message): - # further messages are too old to be deleted in bulk - deleted_remaining = await self._delete_messages_individually(messages[current_index:]) - deleted.extend(deleted_remaining) - if not self.cleaning: - # Means that deletion was canceled while deleting the individual messages - return deleted + # Further messages are too old to be deleted in bulk + delete_old = True break to_delete.append(message) if len(to_delete) == 100: - # we can only delete up to 100 messages in a bulk + # Only up to 100 messages can be deleted in a bulk await channel.delete_messages(to_delete) deleted.extend(to_delete) to_delete.clear() + if not self.cleaning: + return deleted if len(to_delete) > 0: - # deleting any leftover messages if there are any + # Deleting any leftover messages if there are any await channel.delete_messages(to_delete) deleted.extend(to_delete) + if not self.cleaning: + return deleted + if delete_old: + old_deleted = await self._delete_messages_individually(messages[current_index:]) + deleted.extend(old_deleted) + return deleted async def _modlog_cleaned_messages(self, messages: list[Message], channels: CleanChannels, ctx: Context) -> bool: -- cgit v1.2.3 From e33b4aad936ac052695a45f62bd986d13f2163b0 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Wed, 8 Sep 2021 00:18:23 +0300 Subject: Switch `users` and `traverse` in main command When providing a user ID it would clash with `traverse` which came first. --- bot/exts/moderation/clean.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index c90aff256..5f97aae22 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -420,8 +420,8 @@ class Clean(Cog): async def clean_group( self, ctx: Context, - traverse: Optional[int] = None, users: Greedy[User] = None, + traverse: Optional[int] = None, first_limit: Optional[CleanLimit] = None, second_limit: Optional[CleanLimit] = None, use_cache: Optional[bool] = None, @@ -434,8 +434,8 @@ class Clean(Cog): Commands for cleaning messages in channels. If arguments are provided, will act as a master command from which all subcommands can be derived. - • `traverse`: The number of messages to look at in each channel. • `users`: A series of user mentions, ID's, or names. + • `traverse`: The number of messages to look at in each channel. • `first_limit` and `second_limit`: A message, a duration delta, or an ISO datetime. If a message is provided, cleaning will happen in that channel, and channels cannot be provided. If a limit is provided, multiple channels cannot be provided. -- cgit v1.2.3 From d9efe01198ae6645d146be2b42c025a47d21bbf4 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Wed, 8 Sep 2021 02:05:22 +0300 Subject: Fix incorrect cache usage --- bot/exts/moderation/clean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 5f97aae22..f12550ab6 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -457,7 +457,7 @@ class Clean(Cog): traverse = CleanMessages.message_limit else: traverse = DEFAULT_TRAVERSE - if not use_cache: + if use_cache is None: use_cache = channels == "*" await self._clean_messages( -- cgit v1.2.3 From f4658f8468cbe055e67d795cc6aa4b171c8c0b0f Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 11 Sep 2021 20:15:21 +0300 Subject: Handle Regex converter errors Handle cases where there are no enclosing backticks, and where the regex pattern is invalid. --- bot/exts/moderation/clean.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index f12550ab6..af79d5a35 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -44,16 +44,21 @@ class CleanChannels(Converter): class Regex(Converter): - """A converter that takes a string in the form `.+` and returns the contents of the inline code.""" + """A converter that takes a string in the form `.+` and returns the contents of the inline code compiled.""" - async def convert(self, ctx: Context, argument: str) -> str: - """Strips the backticks from the string.""" - return re.fullmatch(r"`(.+?)`", argument).group(1) + async def convert(self, ctx: Context, argument: str) -> re.Pattern: + """Strips the backticks from the string and compiles it to a regex pattern.""" + if not (match := re.fullmatch(r"`(.+?)`", argument)): + raise BadArgument("Regex pattern missing wrapping backticks") + try: + return re.compile(match.group(1), re.IGNORECASE + re.DOTALL) + except re.error as e: + raise BadArgument(f"Regex error: {e.msg}") if TYPE_CHECKING: CleanChannels = Union[Literal["*"], list[TextChannel]] # noqa: F811 - Regex = str # noqa: F811 + Regex = re.Pattern # noqa: F811 class Clean(Cog): @@ -120,7 +125,7 @@ class Clean(Cog): def _build_predicate( bots_only: bool = False, users: list[User] = None, - regex: Optional[str] = None, + regex: Optional[re.Pattern] = None, first_limit: Optional[datetime] = None, second_limit: Optional[datetime] = None, ) -> Predicate: @@ -151,7 +156,7 @@ class Clean(Cog): content = "\n".join(attr for attr in content if attr) # Now let's see if there's a regex match - return bool(re.search(regex, content, re.IGNORECASE + re.DOTALL)) + return bool(regex.search(content)) def predicate_range(message: Message) -> bool: """Check if the message age is between the two limits.""" @@ -346,7 +351,7 @@ class Clean(Cog): channels: CleanChannels, bots_only: bool = False, users: list[User] = None, - regex: Optional[str] = None, + regex: Optional[re.Pattern] = None, first_limit: Optional[CleanLimit] = None, second_limit: Optional[CleanLimit] = None, use_cache: Optional[bool] = True -- cgit v1.2.3 From e215fb03552f822e6e702e741956ee17d07f6117 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Sat, 11 Sep 2021 20:34:50 +0300 Subject: End clean on unexpected errors Added a cog_command_error method that sets cleaning to False when a command ends on an exception. I don't have anything in mind that might cause this, but it will ensure that in any case the cog will still be usable. --- bot/exts/moderation/clean.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index af79d5a35..3fb2c2870 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -588,6 +588,10 @@ class Clean(Cog): """Only allow moderators to invoke the commands in this cog.""" return await has_any_role(*MODERATION_ROLES).predicate(ctx) + async def cog_command_error(self, ctx: Context, error: Exception) -> None: + """Safely end the cleaning operation on unexpected errors.""" + self.cleaning = False + def setup(bot: Bot) -> None: """Load the Clean cog.""" -- cgit v1.2.3 From c27226504b5d384023074eb070e37464b6c8749a Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 20 Sep 2021 23:11:01 +0300 Subject: Indentation, type-hint, and documentation fixes --- bot/exts/moderation/clean.py | 63 ++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 3fb2c2870..d5bfdb485 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -5,7 +5,7 @@ from collections import defaultdict from contextlib import suppress from datetime import datetime from itertools import islice -from typing import Any, Callable, DefaultDict, Iterable, Literal, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, Iterable, Literal, Optional, TYPE_CHECKING, Union from discord import Colour, Message, NotFound, TextChannel, User, errors from discord.ext.commands import Cog, Context, Converter, Greedy, group, has_any_role @@ -86,11 +86,11 @@ class Clean(Cog): @staticmethod def _validate_input( traverse: int, - channels: CleanChannels, + channels: Optional[CleanChannels], bots_only: bool, - users: list[User], - first_limit: CleanLimit, - second_limit: CleanLimit, + users: Optional[list[User]], + first_limit: Optional[CleanLimit], + second_limit: Optional[CleanLimit], ) -> None: """Raise errors if an argument value or a combination of values is invalid.""" # Is this an acceptable amount of messages to traverse? @@ -124,7 +124,7 @@ class Clean(Cog): @staticmethod def _build_predicate( bots_only: bool = False, - users: list[User] = None, + users: Optional[list[User]] = None, regex: Optional[re.Pattern] = None, first_limit: Optional[datetime] = None, second_limit: Optional[datetime] = None, @@ -196,7 +196,7 @@ class Clean(Cog): # Invocation message has already been deleted log.info("Tried to delete invocation message, but it was already deleted.") - def _get_messages_from_cache(self, traverse: int, to_delete: Predicate) -> tuple[DefaultDict, list[int]]: + def _get_messages_from_cache(self, traverse: int, to_delete: Predicate) -> tuple[defaultdict[Any, list], list[int]]: """Helper function for getting messages from the cache.""" message_mappings = defaultdict(list) message_ids = [] @@ -348,9 +348,9 @@ class Clean(Cog): self, ctx: Context, traverse: int, - channels: CleanChannels, + channels: Optional[CleanChannels], bots_only: bool = False, - users: list[User] = None, + users: Optional[list[User]] = None, regex: Optional[re.Pattern] = None, first_limit: Optional[CleanLimit] = None, second_limit: Optional[CleanLimit] = None, @@ -423,24 +423,25 @@ class Clean(Cog): @group(invoke_without_command=True, name="clean", aliases=["clear", "purge"]) async def clean_group( - self, - ctx: Context, - users: Greedy[User] = None, - traverse: Optional[int] = None, - first_limit: Optional[CleanLimit] = None, - second_limit: Optional[CleanLimit] = None, - use_cache: Optional[bool] = None, - bots_only: Optional[bool] = False, - regex: Optional[Regex] = None, - *, - channels: CleanChannels = None # "Optional" with discord.py silently ignores incorrect input. + self, + ctx: Context, + users: Greedy[User] = None, + traverse: Optional[int] = None, + first_limit: Optional[CleanLimit] = None, + second_limit: Optional[CleanLimit] = None, + use_cache: Optional[bool] = None, + bots_only: Optional[bool] = False, + regex: Optional[Regex] = None, + *, + channels: CleanChannels = None # "Optional" with discord.py silently ignores incorrect input. ) -> None: """ Commands for cleaning messages in channels. If arguments are provided, will act as a master command from which all subcommands can be derived. • `users`: A series of user mentions, ID's, or names. - • `traverse`: The number of messages to look at in each channel. + • `traverse`: The number of messages to look at in each channel. If using the cache, will look at the first + `traverse` messages in the cache. • `first_limit` and `second_limit`: A message, a duration delta, or an ISO datetime. If a message is provided, cleaning will happen in that channel, and channels cannot be provided. If a limit is provided, multiple channels cannot be provided. @@ -474,7 +475,7 @@ class Clean(Cog): self, ctx: Context, user: User, - traverse: Optional[int] = 10, + traverse: Optional[int] = DEFAULT_TRAVERSE, use_cache: Optional[bool] = True, *, channels: CleanChannels = None @@ -527,10 +528,10 @@ class Clean(Cog): @clean_group.command(name="until") async def clean_until( - self, - ctx: Context, - until: CleanLimit, - channel: TextChannel = None + self, + ctx: Context, + until: CleanLimit, + channel: TextChannel = None ) -> None: """ Delete all messages until a certain limit. @@ -547,11 +548,11 @@ class Clean(Cog): @clean_group.command(name="between", aliases=["after-until", "from-to"]) async def clean_between( - self, - ctx: Context, - first_limit: CleanLimit, - second_limit: CleanLimit, - channel: TextChannel = None + self, + ctx: Context, + first_limit: CleanLimit, + second_limit: CleanLimit, + channel: TextChannel = None ) -> None: """ Delete all messages within range. -- cgit v1.2.3 From acc79870d55ca257a234a9dc49bfa4a7c85da2ca Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Tue, 19 Oct 2021 11:17:31 +0000 Subject: Filtering: add autoban on specific reasons Due to the increase in typo-squatting based phishing, we want to automatically ban users sending specific domain names. For that, this commit will automatically ban any user that trigger a filter which has `[autoban]` in its reason. That's it! --- bot/exts/filters/filtering.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index 7faf063b9..ad67f3469 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -42,6 +42,10 @@ ZALGO_RE = regex.compile(rf"[\p{{NONSPACING MARK}}\p{{ENCLOSING MARK}}--[{VARIAT # Other constants. DAYS_BETWEEN_ALERTS = 3 OFFENSIVE_MSG_DELETE_TIME = timedelta(days=Filter.offensive_msg_delete_days) +AUTO_BAN_REASON = ( + "Your account seem to be compromised (%s). " + "Please appeal this ban once you have regained control of your account." +) FilterMatch = Union[re.Match, dict, bool, List[discord.Embed]] @@ -346,6 +350,26 @@ class Filtering(Cog): stats = self._add_stats(filter_name, match, msg.content) await self._send_log(filter_name, _filter, msg, stats, reason) + # If the filter reason contains `[autoban]`, we want to indeed ban + if "[autoban]" in reason.lower(): + # We create a new context from that message and make sure the staffer is the bot + # and the feeback message is sent in #mod-alert + context = await self.bot.get_context(msg) + context.author = self.bot.user + context.channel = self.bot.get_channel(Channels.mod_alerts) + + # We need to convert the user to a member if we are inside a DM channel + if msg.guild is None: + user = self.bot.get_guild(Guild.id).get_member(msg.author.id) + else: + user = msg.author + + await context.invoke( + self.bot.get_command("ban"), + user, + reason=AUTO_BAN_REASON % reason.lower().replace("[autoban]", "").strip() + ) + break # We don't want multiple filters to trigger async def _send_log( @@ -367,6 +391,10 @@ class Filtering(Cog): # Allow specific filters to override ping_everyone ping_everyone = Filter.ping_everyone and _filter.get("ping_everyone", True) + # If we are going to autoban, we don't want to ping + if "[autoban]" in reason: + ping_everyone = False + eval_msg = "using !eval " if is_eval else "" footer = f"Reason: {reason}" if reason else None message = ( -- cgit v1.2.3 From 8e4fe2e73198edb37eb1dcffb7f531a69cec34c2 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Tue, 19 Oct 2021 13:39:44 +0200 Subject: Filtering: update auto-ban message Co-authored-by: Boris Muratov <8bee278@gmail.com> --- bot/exts/filters/filtering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index ad67f3469..82bbd880e 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -43,8 +43,8 @@ ZALGO_RE = regex.compile(rf"[\p{{NONSPACING MARK}}\p{{ENCLOSING MARK}}--[{VARIAT DAYS_BETWEEN_ALERTS = 3 OFFENSIVE_MSG_DELETE_TIME = timedelta(days=Filter.offensive_msg_delete_days) AUTO_BAN_REASON = ( - "Your account seem to be compromised (%s). " - "Please appeal this ban once you have regained control of your account." + "Your account seems to be compromised (%s). " + "You're welcome to appeal this ban once you have regained control of your account." ) FilterMatch = Union[re.Match, dict, bool, List[discord.Embed]] -- cgit v1.2.3 From cf6f11488523db9c25d396a6c70b4a60604e6306 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Tue, 19 Oct 2021 11:43:37 +0000 Subject: Filtering: do not try to convert to a member --- bot/exts/filters/filtering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index 82bbd880e..11d0f038b 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -358,9 +358,9 @@ class Filtering(Cog): context.author = self.bot.user context.channel = self.bot.get_channel(Channels.mod_alerts) - # We need to convert the user to a member if we are inside a DM channel + # We try to convert the user to a member if we are inside a DM channel if msg.guild is None: - user = self.bot.get_guild(Guild.id).get_member(msg.author.id) + user = self.bot.get_guild(Guild.id).get_member(msg.author.id) or msg.author else: user = msg.author -- cgit v1.2.3 From 9216e7e341e014528a4fdfaad3bd1a198b8fba02 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Tue, 19 Oct 2021 12:20:25 +0000 Subject: Filtering: make autoban temp --- bot/exts/filters/filtering.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index 11d0f038b..15c12d27c 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -46,6 +46,7 @@ AUTO_BAN_REASON = ( "Your account seems to be compromised (%s). " "You're welcome to appeal this ban once you have regained control of your account." ) +AUTO_BAN_DURATION = timedelta(days=4) FilterMatch = Union[re.Match, dict, bool, List[discord.Embed]] @@ -358,15 +359,10 @@ class Filtering(Cog): context.author = self.bot.user context.channel = self.bot.get_channel(Channels.mod_alerts) - # We try to convert the user to a member if we are inside a DM channel - if msg.guild is None: - user = self.bot.get_guild(Guild.id).get_member(msg.author.id) or msg.author - else: - user = msg.author - await context.invoke( - self.bot.get_command("ban"), - user, + self.bot.get_command("tempban"), + msg.author, + datetime.now() + AUTO_BAN_DURATION, reason=AUTO_BAN_REASON % reason.lower().replace("[autoban]", "").strip() ) -- cgit v1.2.3 From fc47cf4888b3589019ebafe037996f505c4a8b30 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Fri, 22 Oct 2021 00:12:16 +0400 Subject: Bumps Pip Licences Updates the pip-licences version to fix a breaking bug in the currently pinned version. Signed-off-by: Hassan Abouelela --- poetry.lock | 181 ++++++++++++++++++++++++++++++--------------------------- pyproject.toml | 2 +- 2 files changed, 95 insertions(+), 88 deletions(-) diff --git a/poetry.lock b/poetry.lock index 16c599bd1..d91941d45 100644 --- a/poetry.lock +++ b/poetry.lock @@ -722,7 +722,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pycares" -version = "4.0.0" +version = "4.1.2" description = "Python interface for c-ares" category = "main" optional = false @@ -902,7 +902,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [[package]] name = "rapidfuzz" -version = "1.7.1" +version = "1.8.0" description = "rapid fuzzy string matching" category = "main" optional = false @@ -1114,7 +1114,7 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "3.9.*" -content-hash = "e37923739c35ef349d57e324579acfe304cc7e6fc20ddc54205fc89f171ae94f" +content-hash = "da321f13297501e62dd1eb362eccb586ea1a9c21ddb395e11a91b93a2f92e9d4" [metadata.files] aio-pika = [ @@ -1471,6 +1471,8 @@ lxml = [ {file = "lxml-4.6.3-cp27-cp27m-win_amd64.whl", hash = "sha256:8157dadbb09a34a6bd95a50690595e1fa0af1a99445e2744110e3dca7831c4ee"}, {file = "lxml-4.6.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7728e05c35412ba36d3e9795ae8995e3c86958179c9770e65558ec3fdfd3724f"}, {file = "lxml-4.6.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:4bff24dfeea62f2e56f5bab929b4428ae6caba2d1eea0c2d6eb618e30a71e6d4"}, + {file = "lxml-4.6.3-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:64812391546a18896adaa86c77c59a4998f33c24788cadc35789e55b727a37f4"}, + {file = "lxml-4.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c1a40c06fd5ba37ad39caa0b3144eb3772e813b5fb5b084198a985431c2f1e8d"}, {file = "lxml-4.6.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:74f7d8d439b18fa4c385f3f5dfd11144bb87c1da034a466c5b5577d23a1d9b51"}, {file = "lxml-4.6.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f90ba11136bfdd25cae3951af8da2e95121c9b9b93727b1b896e3fa105b2f586"}, {file = "lxml-4.6.3-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:4c61b3a0db43a1607d6264166b230438f85bfed02e8cff20c22e564d0faff354"}, @@ -1674,39 +1676,37 @@ py = [ {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, ] pycares = [ - {file = "pycares-4.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:db5a533111a3cfd481e7e4fb2bf8bef69f4fa100339803e0504dd5aecafb96a5"}, - {file = "pycares-4.0.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:fdff88393c25016f417770d82678423fc7a56995abb2df3d2a1e55725db6977d"}, - {file = "pycares-4.0.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0aa97f900a7ffb259be77d640006585e2a907b0cd4edeee0e85cf16605995d5a"}, - {file = "pycares-4.0.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:a34b0e3e693dceb60b8a1169668d606c75cb100ceba0a2df53c234a0eb067fbc"}, - {file = "pycares-4.0.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:7661d6bbd51a337e7373cb356efa8be9b4655fda484e068f9455e939aec8d54e"}, - {file = "pycares-4.0.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:57315b8eb8fdbc56b3ad4932bc4b17132bb7c7fd2bd590f7fb84b6b522098aa9"}, - {file = "pycares-4.0.0-cp36-cp36m-win32.whl", hash = "sha256:dca9dc58845a9d083f302732a3130c68ded845ad5d463865d464e53c75a3dd45"}, - {file = "pycares-4.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:c95c964d5dd307e104b44b193095c67bb6b10c9eda1ffe7d44ab7a9e84c476d9"}, - {file = "pycares-4.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:26e67e4f81c80a5955dcf6193f3d9bee3c491fc0056299b383b84d792252fba4"}, - {file = "pycares-4.0.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:cd3011ffd5e1ad55880f7256791dbab9c43ebeda260474a968f19cd0319e1aef"}, - {file = "pycares-4.0.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1b959dd5921d207d759d421eece1b60416df33a7f862465739d5f2c363c2f523"}, - {file = "pycares-4.0.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:6f258c1b74c048a9501a25f732f11b401564005e5e3c18f1ca6cad0c3dc0fb19"}, - {file = "pycares-4.0.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:b17ef48729786e62b574c6431f675f4cb02b27691b49e7428a605a50cd59c072"}, - {file = "pycares-4.0.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:82b3259cb590ddd107a6d2dc52da2a2e9a986bf242e893d58c786af2f8191047"}, - {file = "pycares-4.0.0-cp37-cp37m-win32.whl", hash = "sha256:4876fc790ae32832ae270c4a010a1a77e12ddf8d8e6ad70ad0b0a9d506c985f7"}, - {file = "pycares-4.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f60c04c5561b1ddf85ca4e626943cc09d7fb684e1adb22abb632095415a40fd7"}, - {file = "pycares-4.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:615406013cdcd1b445e5d1a551d276c6200b3abe77e534f8a7f7e1551208d14f"}, - {file = "pycares-4.0.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:6580aef5d1b29a88c3d72fe73c691eacfd454f86e74d3fdd18f4bad8e8def98b"}, - {file = "pycares-4.0.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8ebb3ba0485f66cae8eed7ce3e9ed6f2c0bfd5e7319d5d0fbbb511064f17e1d4"}, - {file = "pycares-4.0.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:c5362b7690ca481440f6b98395ac6df06aa50518ccb183c560464d1e5e2ab5d4"}, - {file = "pycares-4.0.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:eb60be66accc9a9ea1018b591a1f5800cba83491d07e9acc8c56bc6e6607ab54"}, - {file = "pycares-4.0.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:44896d6e191a6b5a914dbe3aa7c748481bf6ad19a9df33c1e76f8f2dc33fc8f0"}, - {file = "pycares-4.0.0-cp38-cp38-win32.whl", hash = "sha256:09b28fc7bc2cc05f7f69bf1636ddf46086e0a1837b62961e2092fcb40477320d"}, - {file = "pycares-4.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:d4a5081e232c1d181883dcac4675807f3a6cf33911c4173fbea00c0523687ed4"}, - {file = "pycares-4.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:103353577a6266a53e71bfee4cf83825f1401fefa60f0fb8bdec35f13be6a5f2"}, - {file = "pycares-4.0.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:ad6caf580ee69806fc6534be93ddbb6e99bf94296d79ab351c37b2992b17abfd"}, - {file = "pycares-4.0.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:3d5e50c95849f6905d2a9dbf02ed03f82580173e3c5604a39e2ad054185631f1"}, - {file = "pycares-4.0.0-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:53bc4f181b19576499b02cea4b45391e8dcbe30abd4cd01492f66bfc15615a13"}, - {file = "pycares-4.0.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:d52f9c725d2a826d5ffa37681eb07ffb996bfe21788590ef257664a3898fc0b5"}, - {file = "pycares-4.0.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:3c7fb8d34ee11971c39acfaf98d0fac66725385ccef3bfe1b174c92b210e1aa4"}, - {file = "pycares-4.0.0-cp39-cp39-win32.whl", hash = "sha256:e9773e07684a55f54657df05237267611a77b294ec3bacb5f851c4ffca38a465"}, - {file = "pycares-4.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:38e54037f36c149146ff15f17a4a963fbdd0f9871d4a21cd94ff9f368140f57e"}, - {file = "pycares-4.0.0.tar.gz", hash = "sha256:d0154fc5753b088758fbec9bc137e1b24bb84fc0c6a09725c8bac25a342311cd"}, + {file = "pycares-4.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:71b99b9e041ae3356b859822c511f286f84c8889ec9ed1fbf6ac30fb4da13e4c"}, + {file = "pycares-4.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c000942f5fc64e6e046aa61aa53b629b576ba11607d108909727c3c8f211a157"}, + {file = "pycares-4.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:b0e50ddc78252f2e2b6b5f2c73e5b2449dfb6bea7a5a0e21dfd1e2bcc9e17382"}, + {file = "pycares-4.1.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6831e963a910b0a8cbdd2750ffcdf5f2bb0edb3f53ca69ff18484de2cc3807c4"}, + {file = "pycares-4.1.2-cp310-cp310-win32.whl", hash = "sha256:ad7b28e1b6bc68edd3d678373fa3af84e39d287090434f25055d21b4716b2fc6"}, + {file = "pycares-4.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:27a6f09dbfb69bb79609724c0f90dfaa7c215876a7cd9f12d585574d1f922112"}, + {file = "pycares-4.1.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e5a060f5fa90ae245aa99a4a8ad13ec39c2340400de037c7e8d27b081e1a3c64"}, + {file = "pycares-4.1.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:056330275dea42b7199494047a745e1d9785d39fb8c4cd469dca043532240b80"}, + {file = "pycares-4.1.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0aa897543a786daba74ec5e19638bd38b2b432d179a0e248eac1e62de5756207"}, + {file = "pycares-4.1.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cbceaa9b2c416aa931627466d3240aecfc905c292c842252e3d77b8630072505"}, + {file = "pycares-4.1.2-cp36-cp36m-win32.whl", hash = "sha256:112e1385c451069112d6b5ea1f9c378544f3c6b89882ff964e9a64be3336d7e4"}, + {file = "pycares-4.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:c6680f7fdc0f1163e8f6c2a11d11b9a0b524a61000d2a71f9ccd410f154fb171"}, + {file = "pycares-4.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58a41a2baabcd95266db776c510d349d417919407f03510fc87ac7488730d913"}, + {file = "pycares-4.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a810d01c9a426ee8b0f36969c2aef5fb966712be9d7e466920beb328cd9cefa3"}, + {file = "pycares-4.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:b266cec81dcea2c3efbbd3dda00af8d7eb0693ae9e47e8706518334b21f27d4a"}, + {file = "pycares-4.1.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8319afe4838e09df267c421ca93da408f770b945ec6217dda72f1f6a493e37e4"}, + {file = "pycares-4.1.2-cp37-cp37m-win32.whl", hash = "sha256:4d5da840aa0d9b15fa51107f09270c563a348cb77b14ae9653d0bbdbe326fcc2"}, + {file = "pycares-4.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:5632f21d92cc0225ba5ff906e4e5dec415ef0b3df322c461d138190681cd5d89"}, + {file = "pycares-4.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8fd1ff17a26bb004f0f6bb902ba7dddd810059096ae0cc3b45e4f5be46315d19"}, + {file = "pycares-4.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:439799be4b7576e907139a7f9b3c8a01b90d3e38af4af9cd1fc6c1ee9a42b9e6"}, + {file = "pycares-4.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:40079ed58efa91747c50aac4edf8ecc7e570132ab57dc0a4030eb0d016a6cab8"}, + {file = "pycares-4.1.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e190471a015f8225fa38069617192e06122771cce2b169ac7a60bfdbd3d4ab2"}, + {file = "pycares-4.1.2-cp38-cp38-win32.whl", hash = "sha256:2b837315ed08c7df009b67725fe1f50489e99de9089f58ec1b243dc612f172aa"}, + {file = "pycares-4.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:c7eba3c8354b730a54d23237d0b6445a2f68570fa68d0848887da23a3f3b71f3"}, + {file = "pycares-4.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2f5f84fe9f83eab9cd68544b165b74ba6e3412d029cc9ab20098d9c332869fc5"}, + {file = "pycares-4.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569eef8597b5e02b1bc4644b9f272160304d8c9985357d7ecfcd054da97c0771"}, + {file = "pycares-4.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e1489aa25d14dbf7176110ead937c01176ed5a0ebefd3b092bbd6b202241814c"}, + {file = "pycares-4.1.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:dc942692fca0e27081b7bb414bb971d34609c80df5e953f6d0c62ecc8019acd9"}, + {file = "pycares-4.1.2-cp39-cp39-win32.whl", hash = "sha256:ed71dc4290d9c3353945965604ef1f6a4de631733e9819a7ebc747220b27e641"}, + {file = "pycares-4.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:ec00f3594ee775665167b1a1630edceefb1b1283af9ac57480dba2fb6fd6c360"}, + {file = "pycares-4.1.2.tar.gz", hash = "sha256:03490be0e7b51a0c8073f877bec347eff31003f64f57d9518d419d9369452837"}, ] pycodestyle = [ {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"}, @@ -1792,57 +1792,64 @@ pyyaml = [ {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, ] rapidfuzz = [ - {file = "rapidfuzz-1.7.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1ca9888e867aed2bb8d51571270e5f8393d718bb189fe1a7c0b047b8fd72bad3"}, - {file = "rapidfuzz-1.7.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:f336cd32a2a72eb9d7694618c9065ef3a2af330ab7e54bc0ec69d3b2eb08080e"}, - {file = "rapidfuzz-1.7.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:76124767ac3d3213a1aad989f80b156b225defef8addc825a5b631d3164c3213"}, - {file = "rapidfuzz-1.7.1-cp27-cp27m-win32.whl", hash = "sha256:c1090deb95e5369fff47c223c0ed3472644efc56817e288ebeaaa34822a1235c"}, - {file = "rapidfuzz-1.7.1-cp27-cp27m-win_amd64.whl", hash = "sha256:83f94c89e8f16679e0def3c7afa6c9ba477d837fd01250d6a1e3fea12267ce24"}, - {file = "rapidfuzz-1.7.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:cdd5962bd009b1457e280b5619d312cd6305b5b8afeff6c27869f98fee839c36"}, - {file = "rapidfuzz-1.7.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:2940960e212b66f00fc58f9b4a13e6f80221141dcbaee9c51f97e0a1f30ff1ab"}, - {file = "rapidfuzz-1.7.1-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:5ed4304a91043d27b92fe9af5eb87d1586548da6d03cbda5bbc98b00fee227cb"}, - {file = "rapidfuzz-1.7.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:be18495bd84bf2bd3e888270a3cd4dea868ff4b9b8ec6e540f0e195cda554140"}, - {file = "rapidfuzz-1.7.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:d5779e6f548b6f3edfbdfbeeda4158286684dcb2bae3515ce68c510ea48e1b4d"}, - {file = "rapidfuzz-1.7.1-cp35-cp35m-win32.whl", hash = "sha256:80d780c4f6da08eb6801489df54fdbdc5ef2b882bd73f9585ef6e0cf09f1690d"}, - {file = "rapidfuzz-1.7.1-cp35-cp35m-win_amd64.whl", hash = "sha256:3b205c63b8606c2b8595ba8403a8c3ebd39de9f7f44631a2f651f3efe106ae9a"}, - {file = "rapidfuzz-1.7.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8f96588a8a7d021debb4c60d82b15a80995daa99159bbeddd8a37f68f75ee06c"}, - {file = "rapidfuzz-1.7.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b8139116a937691dde17f27aafe774647808339305f4683b3a6d9bae6518aa2a"}, - {file = "rapidfuzz-1.7.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba574801c8410cc1f2d690ef65f898f6a660bba22ec8213e0f34dd0f0590bc71"}, - {file = "rapidfuzz-1.7.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2d5194e3cb638af0cc7c02daa61cef07e332fd3f790ec113006302131be9afa6"}, - {file = "rapidfuzz-1.7.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dd9d8eaae888b966422cbcba954390a63b4933d8c513ea0056fd6e42d421d08"}, - {file = "rapidfuzz-1.7.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3725c61b9cf57b6b7a765b92046e7d9e5ccce845835b523954b410a70dc32692"}, - {file = "rapidfuzz-1.7.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e417961e5ca450d6c7448accc5a7e4e9ab0dd3c63729f76215d5e672785920fc"}, - {file = "rapidfuzz-1.7.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:26d756284c8c6274b5d558e759415bfb4016fcdf168159b34702c346875d8cc0"}, - {file = "rapidfuzz-1.7.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4887766f0dcc5df43fe4315df4b3c642829e06dc60d5bcb5e682fb76657e8ed1"}, - {file = "rapidfuzz-1.7.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ec0a29671d59998b97998b757ab1c636dd3b7721eda41746ae897abe709681a9"}, - {file = "rapidfuzz-1.7.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dff55750fecd8c0f07bc199e48427c86873be2d0e6a3a80df98972847287f5d3"}, - {file = "rapidfuzz-1.7.1-cp37-cp37m-win32.whl", hash = "sha256:e113f741bb18b0ddd14d714d80ce9c6d5322724f3023b920708e82491e7aef28"}, - {file = "rapidfuzz-1.7.1-cp37-cp37m-win_amd64.whl", hash = "sha256:ef20654be0aed240ee44c98ce02639c37422adc3e144d28c4b6d3da043d9fd20"}, - {file = "rapidfuzz-1.7.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:9e27eb57745a4d2a390b056f6f490b712c2f54250c5d2c794dd76062065a8aef"}, - {file = "rapidfuzz-1.7.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:de2b0ebb67ee0b78973141dba91f574a325a3425664dbdbad37fd7aca7b28cab"}, - {file = "rapidfuzz-1.7.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:88c65d91dcd3c0595112d16555536c60ac5bcab1a43e517e155a242a39525057"}, - {file = "rapidfuzz-1.7.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:afd525a9b593cc1099f0210e116bcb4d9fc5585728d7bd929e6a4133dacd2d59"}, - {file = "rapidfuzz-1.7.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e6d77f104a8d67c01ae4248ced6f0d4ef05e63931afdf49c20decf962318877f"}, - {file = "rapidfuzz-1.7.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7db9d6ad0ab80e9e0f66f157b8e31b1d04ce5fa767b936ca1c212b98092572b1"}, - {file = "rapidfuzz-1.7.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0195c57f4beea0e7691594f59faf62a4be3c818c1955a8b9b712f37adc479d2d"}, - {file = "rapidfuzz-1.7.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9ffca8c8b74d12cd36c051e9befa7c4eb2d34624ce71f22dbfc659af15bf4a1e"}, - {file = "rapidfuzz-1.7.1-cp38-cp38-win32.whl", hash = "sha256:234cb75aa1e21cabad6a8c0718f84e2bfafdd4756b5232d5739545f97e343e59"}, - {file = "rapidfuzz-1.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:058977e93ab736071fcd8828fc6289ec026e9ca4a19f2a0967f9260e63910da8"}, - {file = "rapidfuzz-1.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9d02bb0724326826b1884cc9b9d9fd97ac352c18213f45e465a39ef069a33115"}, - {file = "rapidfuzz-1.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:212d6fa5b824aaa49a921c81d7cdc1d079b3545a30563ae14dc88e17918e76bf"}, - {file = "rapidfuzz-1.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a0cd8117deba10e2a1d6dccb6ff44a4c737adda3048dc45860c5f53cf64db14f"}, - {file = "rapidfuzz-1.7.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:61faa47b6b5d5a0cbe9fa6369df44d3f9435c4cccdb4d38d9de437f18b69dc4d"}, - {file = "rapidfuzz-1.7.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1daa756be52a7ee60d553ba667cda3a188ee811c92a9c21df43a4cdadb1eb8ca"}, - {file = "rapidfuzz-1.7.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c98ac10782dadf507e922963c8b8456a79151b4f10dbb08cfc86c1572db366dc"}, - {file = "rapidfuzz-1.7.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:358d80061ca107df6c3e1f67fa7af0f94a62827cb9c44ac09a16e78b38f7c3d5"}, - {file = "rapidfuzz-1.7.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a5f90fc31d54fcd74a97d175892555786a8214a3cff43077463915b8a45a191d"}, - {file = "rapidfuzz-1.7.1-cp39-cp39-win32.whl", hash = "sha256:55dffdcdccea6f077a4f09164039411f01f621633be5883c58ceaf94f007a688"}, - {file = "rapidfuzz-1.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:d712a7f680d2074b587650f81865ca838c04fcc6b77c9d2d742de0853aaa24ce"}, - {file = "rapidfuzz-1.7.1-pp27-pypy_73-macosx_10_9_x86_64.whl", hash = "sha256:729d73a8db5a2b444a19d4aa2be009b2e628d207d7c754f6d280e3c6a59b94cb"}, - {file = "rapidfuzz-1.7.1-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:a1cabbc645395b6175cad79164d9ec621866a004b476e44cac534020b9f6bddb"}, - {file = "rapidfuzz-1.7.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ae697294f456f7f76e5bd30db5a65e8b855e7e09f9a65e144efa1e2c5009553c"}, - {file = "rapidfuzz-1.7.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e8ae51c1cf1f034f15216fec2e1eef658c8b3a9cbdcc1a053cc7133ede9d616d"}, - {file = "rapidfuzz-1.7.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:dccc072f2a0eeb98d46a79427ef793836ebc5184b1fe544b34607be10705ddc3"}, - {file = "rapidfuzz-1.7.1.tar.gz", hash = "sha256:99495c679174b2a02641f7dc2364a208135cacca77fc4825a86efbfe1e23b0ff"}, + {file = "rapidfuzz-1.8.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:91f094562c683802e6c972bce27a692dad70d6cd1114e626b29d990c3704c653"}, + {file = "rapidfuzz-1.8.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:4a20682121e245cf5ad2dbdd771360763ea11b77520632a1034c4bb9ad1e854c"}, + {file = "rapidfuzz-1.8.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:8810e75d8f9c4453bbd6209c372bf97514359b0b5efff555caf85b15f8a9d862"}, + {file = "rapidfuzz-1.8.0-cp27-cp27m-win32.whl", hash = "sha256:00cf713d843735b5958d87294f08b05c653a593ced7c4120be34f5d26d7a320a"}, + {file = "rapidfuzz-1.8.0-cp27-cp27m-win_amd64.whl", hash = "sha256:2baca64e23a623e077f57e5470de21af2765af15aa1088676eb2d475e664eed0"}, + {file = "rapidfuzz-1.8.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:9bf7a6c61bacedd84023be356e057e1d209dd6997cfaa3c1cee77aa21d642f88"}, + {file = "rapidfuzz-1.8.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:61b6434e3341ca5158ecb371b1ceb4c1f6110563a72d28bdce4eb2a084493e47"}, + {file = "rapidfuzz-1.8.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e425e690383f6cf308e8c2e8d630fa9596f67d233344efd8fae11e70a9f5635f"}, + {file = "rapidfuzz-1.8.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:93db5e693b76d616b09df27ca5c79e0dda169af7f1b8f5ab3262826d981e37e2"}, + {file = "rapidfuzz-1.8.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a8c4f76ed1c8a65892d98dc2913027c9acdb219d18f3a441cfa427a32861af9"}, + {file = "rapidfuzz-1.8.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:71e217fd30901214cc96c0c15057278bafb7072aa9b2be4c97459c1fedf3e731"}, + {file = "rapidfuzz-1.8.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d579dd447b8e851462e79054b68f94b66b09df8b3abb2aa5ca07fe00912ef5e8"}, + {file = "rapidfuzz-1.8.0-cp310-cp310-win32.whl", hash = "sha256:5808064555273496dcd594d659bd28ee8d399149dd31575321034424455dc955"}, + {file = "rapidfuzz-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:798fef1671ca66c78b47802228e9583f7ab32b99bdfe3984ebb1f96e93e38b5f"}, + {file = "rapidfuzz-1.8.0-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:c9e0ed210831f5c73533bf11099ea7897db491e76c3443bef281d9c1c67d7f3a"}, + {file = "rapidfuzz-1.8.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:c819bb19eb615a31ddc9cb8248a285bf04f58158b53ce096451178631f99b652"}, + {file = "rapidfuzz-1.8.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:942ee45564f28ef70320d1229f02dc998bd93e3519c1f3a80f33ce144b51039c"}, + {file = "rapidfuzz-1.8.0-cp35-cp35m-win32.whl", hash = "sha256:7e6ae2e5a3bc9acc51e118f25d32b8efcd431c5d8deb408336dd2ed0f21d087c"}, + {file = "rapidfuzz-1.8.0-cp35-cp35m-win_amd64.whl", hash = "sha256:98901fba67c89ad2506f3946642cf6eb8f489592fb7eb307ebdf8bdb0c4e97f9"}, + {file = "rapidfuzz-1.8.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:705e1686f406a0c77ef323cdb7369b7cf9e68f2abfcb83ff5f1e0a5b21f5a534"}, + {file = "rapidfuzz-1.8.0-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:da0c5fe5fdbbd74206c1778af6b8c5ff8dfbe2dd04ae12bbe96642b358acefce"}, + {file = "rapidfuzz-1.8.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:535253bc9224215131ae450aad6c9f7ef1b24f15c685045eab2b52511268bd06"}, + {file = "rapidfuzz-1.8.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:acdad83f07d886705fce164b0d1f4e3b56788a205602ed3a7fc8b10ceaf05fbf"}, + {file = "rapidfuzz-1.8.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35097f649831f8375d6c65a237deccac3aceb573aa7fae1e5d3fa942e89de1c8"}, + {file = "rapidfuzz-1.8.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6f4db142e5b4b44314166a90e11603220db659bd2f9c23dd5db402c13eac8eb7"}, + {file = "rapidfuzz-1.8.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:19a3f55f27411d68360540484874beda0b428b062596d5f0f141663ef0738bfd"}, + {file = "rapidfuzz-1.8.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:22b4c1a7f6fe29bd8dae49f7d5ab085dc42c3964f1a78b6dca22fdf83b5c9bfa"}, + {file = "rapidfuzz-1.8.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8bfb2fbc147904b78d5c510ee75dc8704b606e956df23f33a9e89abc03f45c3"}, + {file = "rapidfuzz-1.8.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e6dc5111ebfed2c4f2e4d120a9b280ea13ea4fbb60b6915dd239817b4fc092ed"}, + {file = "rapidfuzz-1.8.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db5ee2457d97cb967ffe08446a8c595c03fe747fdc2e145266713f9c516d1c4a"}, + {file = "rapidfuzz-1.8.0-cp37-cp37m-win32.whl", hash = "sha256:12c1b78cc15fc26f555a4bf66088d5afb6354b5a5aa149a123f01a15af6c411b"}, + {file = "rapidfuzz-1.8.0-cp37-cp37m-win_amd64.whl", hash = "sha256:693e9579048d8db4ff020715dd6f25aa315fd6445bc94e7400d7a94a227dad27"}, + {file = "rapidfuzz-1.8.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b4fe19df3edcf7de359448b872aec08e6592b4ca2d3df4d8ee57b5812d68bebf"}, + {file = "rapidfuzz-1.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f3670b9df0e1f479637cad1577afca7766a02775dc08c14837cf495c82861d7c"}, + {file = "rapidfuzz-1.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:61d118f36eb942649b0db344f7b7a19ad7e9b5749d831788187eb03b57ce1bfa"}, + {file = "rapidfuzz-1.8.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fce3a2c8a1d10da12aff4a0d367624e8ae9e15c1b84a5144843681d39be0c355"}, + {file = "rapidfuzz-1.8.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1577ef26e3647ccc4cc9754c34ffaa731639779f4d7779e91a761c72adac093e"}, + {file = "rapidfuzz-1.8.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fec9b7e60fde51990c3b48fc1aa9dba9ac3acaf78f623dbb645a6fe21a9654e"}, + {file = "rapidfuzz-1.8.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b954469d93858bc8b48129bc63fd644382a4df5f3fb1b4b290f48eac1d00a2da"}, + {file = "rapidfuzz-1.8.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:190ba709069a7e5a6b39b7c8bc413a08cfa7f1f4defec5d974c4128b510e0234"}, + {file = "rapidfuzz-1.8.0-cp38-cp38-win32.whl", hash = "sha256:97b2d13d6323649b43d1b113681e4013ba230bd6e9827cc832dcebee447d7250"}, + {file = "rapidfuzz-1.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:81c3091209b75f6611efe2af18834180946d4ce28f41ca8d44fce816187840d2"}, + {file = "rapidfuzz-1.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d610afa33e92aa0481a514ffda3ec51ca5df3c684c1c1c795307589c62025931"}, + {file = "rapidfuzz-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d976f33ca6b5fabbb095c0a662f5b86baf706184fc24c7f125d4ddb54b8bf036"}, + {file = "rapidfuzz-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0f5ca7bca2af598d4ddcf5b93b64b50654a9ff684e6f18d865f6e13fee442b3e"}, + {file = "rapidfuzz-1.8.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6dc2aac5ea6b0306dcd28a6d1a89d35ed2c6ac426f2673ee1b92cf3f1d0fd5cd"}, + {file = "rapidfuzz-1.8.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f145c9831c0454a696a3136a6380ea4e01434e9cc2f2bc10d032864c16d1d0e5"}, + {file = "rapidfuzz-1.8.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4ce53291575b56c9d45add73ea013f43bafcea55eee9d5139aa759918d7685f"}, + {file = "rapidfuzz-1.8.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de5773a39c00a0f23cfc5da9e0e5fd0fb512b0ebe23dc7289a38e1f9a4b5cefc"}, + {file = "rapidfuzz-1.8.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87a802e55792bfbe192e2d557f38867dbe3671b49b3d5ecd873859c7460746ba"}, + {file = "rapidfuzz-1.8.0-cp39-cp39-win32.whl", hash = "sha256:9391abf1121df831316222f28cea37397a0f72bd7978f3be6e7da29a7821e4e5"}, + {file = "rapidfuzz-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:9eeca1b436042b5523dcf314f5822b1131597898c1d967f140d1917541a8a3d1"}, + {file = "rapidfuzz-1.8.0-pp27-pypy_73-macosx_10_9_x86_64.whl", hash = "sha256:a01f2495aca479b49d3b3a8863d6ba9bea2043447a1ced74ae5ec5270059cbc1"}, + {file = "rapidfuzz-1.8.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:b7d4b1a5d16817f8cdb34365c7b58ae22d5cf1b3207720bb2fa0b55968bdb034"}, + {file = "rapidfuzz-1.8.0-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c738d0d7f1744646d48d19b4c775926082bcefebd2460f45ca383a0e882f5672"}, + {file = "rapidfuzz-1.8.0-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0fb9c6078c17c12b52e66b7d0a2a1674f6bbbdc6a76e454c8479b95147018123"}, + {file = "rapidfuzz-1.8.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:1482b385d83670eb069577c9667f72b41eec4f005aee32f1a4ff4e71e88afde2"}, + {file = "rapidfuzz-1.8.0.tar.gz", hash = "sha256:83fff37acf0367314879231264169dcbc5e7de969a94f4b82055d06a7fddab9a"}, ] redis = [ {file = "redis-3.5.3-py2.py3-none-any.whl", hash = "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24"}, diff --git a/pyproject.toml b/pyproject.toml index e227ffaa6..563bf4a27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ flake8-isort = "~=4.0" pep8-naming = "~=0.9" pre-commit = "~=2.1" taskipy = "~=1.7.0" -pip-licenses = "~=3.5.2" +pip-licenses = "~=3.5.3" python-dotenv = "~=0.17.1" pytest = "~=6.2.4" pytest-cov = "~=2.12.1" -- cgit v1.2.3 From b912cc7c7e3c52e502ec7c558975dcf558a1b418 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Fri, 22 Oct 2021 13:08:13 +0000 Subject: Modlog: explicitly write thread names It seems like there could be some caching issue with threads causing to appear as deleted channels. Beside, we also want to keep the name of deleted threads around. --- bot/exts/moderation/modlog.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/bot/exts/moderation/modlog.py b/bot/exts/moderation/modlog.py index 9d1ae6853..c6752d0f9 100644 --- a/bot/exts/moderation/modlog.py +++ b/bot/exts/moderation/modlog.py @@ -800,7 +800,10 @@ class ModLog(Cog, name="ModLog"): icon, colour, f"Thread {action}", - f"Thread {after.mention} (`{after.id}`) from {after.parent.mention} (`{after.parent.id}`) was {action}" + ( + f"Thread {after.mention} ({after.name}, `{after.id}`) from {after.parent.mention} " + f"(`{after.parent.id}`) was {action}" + ) ) @Cog.listener() @@ -810,7 +813,10 @@ class ModLog(Cog, name="ModLog"): Icons.hash_red, Colours.soft_red, "Thread deleted", - f"Thread {thread.mention} (`{thread.id}`) from {thread.parent.mention} (`{thread.parent.id}`) deleted" + ( + f"Thread {thread.mention} ({thread.name}, `{thread.id}`) from {thread.parent.mention} " + f"(`{thread.parent.id}`) deleted" + ) ) @Cog.listener() @@ -825,7 +831,10 @@ class ModLog(Cog, name="ModLog"): Icons.hash_green, Colours.soft_green, "Thread created", - f"Thread {thread.mention} (`{thread.id}`) from {thread.parent.mention} (`{thread.parent.id}`) created" + ( + f"Thread {thread.mention} ({thread.name}, `{thread.id}`) from {thread.parent.mention} " + f"(`{thread.parent.id}`) created" + ) ) @Cog.listener() -- cgit v1.2.3 From 913b1d5644b57fd900474b8d9bc271e24ea729de Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Fri, 22 Oct 2021 21:52:09 +0300 Subject: Move to timezone aware datetimes (#1895) * Move to timezone aware datetimes With the shift of the discord.py library to timezone aware datetimes, this commit changes datetimes throughout the bot to be in the UTC timezone accordingly. This has several advantages: - There's no need to discard the TZ every time the datetime of a Discord object is fetched. - Using TZ aware datetimes reduces the likelihood of silently adding bugs into the codebase (can't compare an aware datetime with a naive one). - Our DB already stores datetimes in UTC, but we've been discarding the TZ so far whenever we read from it. Specific places in the codebase continue using naive datetimes, mainly for UI purposes (for examples embed footers use naive datetimes to display local time). * Improve ISODateTime converter documentation Co-authored-by: Kieran Siek --- bot/converters.py | 19 ++++++----- bot/exts/filters/antispam.py | 15 ++++----- bot/exts/filters/filtering.py | 19 ++++++----- bot/exts/fun/off_topic_names.py | 7 ++-- bot/exts/moderation/defcon.py | 8 +++-- bot/exts/moderation/infraction/_scheduler.py | 12 +++---- bot/exts/moderation/infraction/management.py | 2 +- bot/exts/moderation/modlog.py | 8 ++--- bot/exts/moderation/modpings.py | 5 +-- bot/exts/moderation/voice_gate.py | 6 ++-- bot/exts/recruitment/talentpool/_review.py | 7 ++-- bot/exts/utils/internal.py | 6 ++-- bot/exts/utils/ping.py | 5 ++- bot/exts/utils/reminders.py | 10 +++--- bot/monkey_patches.py | 7 ++-- bot/utils/checks.py | 3 +- bot/utils/time.py | 19 ++++++----- tests/bot/test_converters.py | 50 ++++++++++++++-------------- tests/bot/utils/test_time.py | 27 +++++++++------ 19 files changed, 122 insertions(+), 113 deletions(-) diff --git a/bot/converters.py b/bot/converters.py index dd02f6ae6..f50acb9c6 100644 --- a/bot/converters.py +++ b/bot/converters.py @@ -2,7 +2,7 @@ from __future__ import annotations import re import typing as t -from datetime import datetime +from datetime import datetime, timezone from ssl import CertificateError import dateutil.parser @@ -11,7 +11,7 @@ import discord from aiohttp import ClientConnectorError from dateutil.relativedelta import relativedelta from discord.ext.commands import BadArgument, Bot, Context, Converter, IDConverter, MemberConverter, UserConverter -from discord.utils import DISCORD_EPOCH, escape_markdown, snowflake_time +from discord.utils import escape_markdown, snowflake_time from bot import exts from bot.api import ResponseCodeError @@ -28,7 +28,7 @@ if t.TYPE_CHECKING: log = get_logger(__name__) -DISCORD_EPOCH_DT = datetime.utcfromtimestamp(DISCORD_EPOCH / 1000) +DISCORD_EPOCH_DT = snowflake_time(0) RE_USER_MENTION = re.compile(r"<@!?([0-9]+)>$") @@ -273,14 +273,14 @@ class Snowflake(IDConverter): snowflake = int(arg) try: - time = snowflake_time(snowflake).replace(tzinfo=None) + time = snowflake_time(snowflake) except (OverflowError, OSError) as e: # Not sure if this can ever even happen, but let's be safe. raise BadArgument(f"{error}: {e}") if time < DISCORD_EPOCH_DT: raise BadArgument(f"{error}: timestamp is before the Discord epoch.") - elif (datetime.utcnow() - time).days < -1: + elif (datetime.now(timezone.utc) - time).days < -1: raise BadArgument(f"{error}: timestamp is too far into the future.") return snowflake @@ -387,7 +387,7 @@ class Duration(DurationDelta): The converter supports the same symbols for each unit of time as its parent class. """ delta = await super().convert(ctx, duration) - now = datetime.utcnow() + now = datetime.now(timezone.utc) try: return now + delta @@ -443,8 +443,8 @@ class ISODateTime(Converter): The converter is flexible in the formats it accepts, as it uses the `isoparse` method of `dateutil.parser`. In general, it accepts datetime strings that start with a date, optionally followed by a time. Specifying a timezone offset in the datetime string is - supported, but the `datetime` object will be converted to UTC and will be returned without - `tzinfo` as a timezone-unaware `datetime` object. + supported, but the `datetime` object will be converted to UTC. If no timezone is specified, the datetime will + be assumed to be in UTC already. In all cases, the returned object will have the UTC timezone. See: https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.isoparse @@ -470,7 +470,8 @@ class ISODateTime(Converter): if dt.tzinfo: dt = dt.astimezone(dateutil.tz.UTC) - dt = dt.replace(tzinfo=None) + else: # Without a timezone, assume it represents UTC. + dt = dt.replace(tzinfo=dateutil.tz.UTC) return dt diff --git a/bot/exts/filters/antispam.py b/bot/exts/filters/antispam.py index 37ac70508..ddfd11231 100644 --- a/bot/exts/filters/antispam.py +++ b/bot/exts/filters/antispam.py @@ -2,11 +2,12 @@ import asyncio from collections import defaultdict from collections.abc import Mapping from dataclasses import dataclass, field -from datetime import datetime, timedelta +from datetime import timedelta from itertools import takewhile from operator import attrgetter, itemgetter from typing import Dict, Iterable, List, Set +import arrow from discord import Colour, Member, Message, NotFound, Object, TextChannel from discord.ext.commands import Cog @@ -177,21 +178,17 @@ class AntiSpam(Cog): self.cache.append(message) - earliest_relevant_at = datetime.utcnow() - timedelta(seconds=self.max_interval) - relevant_messages = list( - takewhile(lambda msg: msg.created_at.replace(tzinfo=None) > earliest_relevant_at, self.cache) - ) + earliest_relevant_at = arrow.utcnow() - timedelta(seconds=self.max_interval) + relevant_messages = list(takewhile(lambda msg: msg.created_at > earliest_relevant_at, self.cache)) for rule_name in AntiSpamConfig.rules: rule_config = AntiSpamConfig.rules[rule_name] rule_function = RULE_FUNCTION_MAPPING[rule_name] # Create a list of messages that were sent in the interval that the rule cares about. - latest_interesting_stamp = datetime.utcnow() - timedelta(seconds=rule_config['interval']) + latest_interesting_stamp = arrow.utcnow() - timedelta(seconds=rule_config['interval']) messages_for_rule = list( - takewhile( - lambda msg: msg.created_at.replace(tzinfo=None) > latest_interesting_stamp, relevant_messages - ) + takewhile(lambda msg: msg.created_at > latest_interesting_stamp, relevant_messages) ) result = await rule_function(message, messages_for_rule, rule_config) diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index a151db1f0..6df78f550 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -1,9 +1,10 @@ import asyncio import re -from datetime import datetime, timedelta +from datetime import timedelta from typing import Any, Dict, List, Mapping, NamedTuple, Optional, Tuple, Union -import dateutil +import arrow +import dateutil.parser import discord.errors import regex from async_rediscache import RedisCache @@ -192,8 +193,8 @@ class Filtering(Cog): async def check_send_alert(self, member: Member) -> bool: """When there is less than 3 days after last alert, return `False`, otherwise `True`.""" if last_alert := await self.name_alerts.get(member.id): - last_alert = datetime.utcfromtimestamp(last_alert) - if datetime.utcnow() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert: + last_alert = arrow.get(last_alert) + if arrow.utcnow() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert: log.trace(f"Last alert was too recent for {member}'s nickname.") return False @@ -227,7 +228,7 @@ class Filtering(Cog): ) # Update time when alert sent - await self.name_alerts.set(member.id, datetime.utcnow().timestamp()) + await self.name_alerts.set(member.id, arrow.utcnow().timestamp()) async def filter_eval(self, result: str, msg: Message) -> bool: """ @@ -603,7 +604,7 @@ class Filtering(Cog): def schedule_msg_delete(self, msg: dict) -> None: """Delete an offensive message once its deletion date is reached.""" - delete_at = dateutil.parser.isoparse(msg['delete_date']).replace(tzinfo=None) + delete_at = dateutil.parser.isoparse(msg['delete_date']) self.scheduler.schedule_at(delete_at, msg['id'], self.delete_offensive_msg(msg)) async def reschedule_offensive_msg_deletion(self) -> None: @@ -611,17 +612,17 @@ class Filtering(Cog): await self.bot.wait_until_ready() response = await self.bot.api_client.get('bot/offensive-messages',) - now = datetime.utcnow() + now = arrow.utcnow() for msg in response: - delete_at = dateutil.parser.isoparse(msg['delete_date']).replace(tzinfo=None) + delete_at = dateutil.parser.isoparse(msg['delete_date']) if delete_at < now: await self.delete_offensive_msg(msg) else: self.schedule_msg_delete(msg) - async def delete_offensive_msg(self, msg: Mapping[str, str]) -> None: + async def delete_offensive_msg(self, msg: Mapping[str, int]) -> None: """Delete an offensive message, and then delete it from the db.""" try: channel = self.bot.get_channel(msg['channel_id']) diff --git a/bot/exts/fun/off_topic_names.py b/bot/exts/fun/off_topic_names.py index 427667c66..7df1d172d 100644 --- a/bot/exts/fun/off_topic_names.py +++ b/bot/exts/fun/off_topic_names.py @@ -1,6 +1,7 @@ import difflib -from datetime import datetime, timedelta +from datetime import timedelta +import arrow from discord import Colour, Embed from discord.ext.commands import Cog, Context, group, has_any_role from discord.utils import sleep_until @@ -22,9 +23,9 @@ async def update_names(bot: Bot) -> None: while True: # Since we truncate the compute timedelta to seconds, we add one second to ensure # we go past midnight in the `seconds_to_sleep` set below. - today_at_midnight = datetime.utcnow().replace(microsecond=0, second=0, minute=0, hour=0) + today_at_midnight = arrow.utcnow().replace(microsecond=0, second=0, minute=0, hour=0) next_midnight = today_at_midnight + timedelta(days=1) - await sleep_until(next_midnight) + await sleep_until(next_midnight.datetime) try: channel_0_name, channel_1_name, channel_2_name = await bot.api_client.get( diff --git a/bot/exts/moderation/defcon.py b/bot/exts/moderation/defcon.py index 80ba10112..822a87b61 100644 --- a/bot/exts/moderation/defcon.py +++ b/bot/exts/moderation/defcon.py @@ -4,6 +4,7 @@ from datetime import datetime from enum import Enum from typing import Optional, Union +import arrow from aioredis import RedisError from async_rediscache import RedisCache from dateutil.relativedelta import relativedelta @@ -109,9 +110,9 @@ class Defcon(Cog): async def on_member_join(self, member: Member) -> None: """Check newly joining users to see if they meet the account age threshold.""" if self.threshold: - now = datetime.utcnow() + now = arrow.utcnow() - if now - member.created_at.replace(tzinfo=None) < relativedelta_to_timedelta(self.threshold): + if now - member.created_at < relativedelta_to_timedelta(self.threshold): log.info(f"Rejecting user {member}: Account is too new") message_sent = False @@ -254,7 +255,8 @@ class Defcon(Cog): expiry_message = "" if expiry: - expiry_message = f" for the next {humanize_delta(relativedelta(expiry, datetime.utcnow()), max_units=2)}" + activity_duration = relativedelta(expiry, arrow.utcnow().datetime) + expiry_message = f" for the next {humanize_delta(activity_duration, max_units=2)}" if self.threshold: channel_message = ( diff --git a/bot/exts/moderation/infraction/_scheduler.py b/bot/exts/moderation/infraction/_scheduler.py index d4e96b10b..74a987808 100644 --- a/bot/exts/moderation/infraction/_scheduler.py +++ b/bot/exts/moderation/infraction/_scheduler.py @@ -1,9 +1,9 @@ import textwrap import typing as t from abc import abstractmethod -from datetime import datetime from gettext import ngettext +import arrow import dateutil.parser import discord from discord.ext.commands import Context @@ -67,7 +67,7 @@ class InfractionScheduler: # We make sure to fire this if to_schedule: next_reschedule_point = max( - dateutil.parser.isoparse(infr["expires_at"]).replace(tzinfo=None) for infr in to_schedule + dateutil.parser.isoparse(infr["expires_at"]) for infr in to_schedule ) log.trace("Will reschedule remaining infractions at %s", next_reschedule_point) @@ -83,8 +83,8 @@ class InfractionScheduler: """Reapply an infraction if it's still active or deactivate it if less than 60 sec left.""" if infraction["expires_at"] is not None: # Calculate the time remaining, in seconds, for the mute. - expiry = dateutil.parser.isoparse(infraction["expires_at"]).replace(tzinfo=None) - delta = (expiry - datetime.utcnow()).total_seconds() + expiry = dateutil.parser.isoparse(infraction["expires_at"]) + delta = (expiry - arrow.utcnow()).total_seconds() else: # If the infraction is permanent, it is not possible to get the time remaining. delta = None @@ -382,7 +382,7 @@ class InfractionScheduler: log.info(f"Marking infraction #{id_} as inactive (expired).") - expiry = dateutil.parser.isoparse(expiry).replace(tzinfo=None) if expiry else None + expiry = dateutil.parser.isoparse(expiry) if expiry else None created = time.format_infraction_with_duration(inserted_at, expiry) log_content = None @@ -503,5 +503,5 @@ class InfractionScheduler: At the time of expiration, the infraction is marked as inactive on the website and the expiration task is cancelled. """ - expiry = dateutil.parser.isoparse(infraction["expires_at"]).replace(tzinfo=None) + expiry = dateutil.parser.isoparse(infraction["expires_at"]) self.scheduler.schedule_at(expiry, infraction["id"], self.deactivate_infraction(infraction)) diff --git a/bot/exts/moderation/infraction/management.py b/bot/exts/moderation/infraction/management.py index b1c8b64dc..96c818c47 100644 --- a/bot/exts/moderation/infraction/management.py +++ b/bot/exts/moderation/infraction/management.py @@ -315,7 +315,7 @@ class ModManagement(commands.Cog): duration = "*Permanent*" else: date_from = datetime.fromtimestamp(float(time.DISCORD_TIMESTAMP_REGEX.match(created).group(1))) - date_to = dateutil.parser.isoparse(expires_at).replace(tzinfo=None) + date_to = dateutil.parser.isoparse(expires_at) duration = humanize_delta(relativedelta(date_to, date_from)) lines = textwrap.dedent(f""" diff --git a/bot/exts/moderation/modlog.py b/bot/exts/moderation/modlog.py index c6752d0f9..6fcf43d8a 100644 --- a/bot/exts/moderation/modlog.py +++ b/bot/exts/moderation/modlog.py @@ -2,7 +2,7 @@ import asyncio import difflib import itertools import typing as t -from datetime import datetime +from datetime import datetime, timezone from itertools import zip_longest import discord @@ -58,7 +58,7 @@ class ModLog(Cog, name="ModLog"): 'bot/deleted-messages', json={ 'actor': actor_id, - 'creation': datetime.utcnow().isoformat(), + 'creation': datetime.now(timezone.utc).isoformat(), 'deletedmessage_set': [ { 'id': message.id, @@ -404,8 +404,8 @@ class ModLog(Cog, name="ModLog"): if member.guild.id != GuildConstant.id: return - now = datetime.utcnow() - difference = abs(relativedelta(now, member.created_at.replace(tzinfo=None))) + now = datetime.now(timezone.utc) + difference = abs(relativedelta(now, member.created_at)) message = format_user(member) + "\n\n**Account age:** " + humanize_delta(difference) diff --git a/bot/exts/moderation/modpings.py b/bot/exts/moderation/modpings.py index a7ccb8162..f67d8f662 100644 --- a/bot/exts/moderation/modpings.py +++ b/bot/exts/moderation/modpings.py @@ -1,5 +1,6 @@ import datetime +import arrow from async_rediscache import RedisCache from dateutil.parser import isoparse from discord import Embed, Member @@ -57,7 +58,7 @@ class ModPings(Cog): if mod.id not in pings_off: await self.reapply_role(mod) else: - expiry = isoparse(pings_off[mod.id]).replace(tzinfo=None) + expiry = isoparse(pings_off[mod.id]) self._role_scheduler.schedule_at(expiry, mod.id, self.reapply_role(mod)) async def reapply_role(self, mod: Member) -> None: @@ -92,7 +93,7 @@ class ModPings(Cog): The duration cannot be longer than 30 days. """ - delta = duration - datetime.datetime.utcnow() + delta = duration - arrow.utcnow() if delta > datetime.timedelta(days=30): await ctx.send(":x: Cannot remove the role for longer than 30 days.") return diff --git a/bot/exts/moderation/voice_gate.py b/bot/exts/moderation/voice_gate.py index 8fdc7c76b..31799ec73 100644 --- a/bot/exts/moderation/voice_gate.py +++ b/bot/exts/moderation/voice_gate.py @@ -1,7 +1,8 @@ import asyncio from contextlib import suppress -from datetime import datetime, timedelta +from datetime import timedelta +import arrow import discord from async_rediscache import RedisCache from discord import Colour, Member, VoiceState @@ -166,8 +167,7 @@ class VoiceGate(Cog): checks = { "joined_at": ( - ctx.author.joined_at.replace(tzinfo=None) > datetime.utcnow() - - timedelta(days=GateConf.minimum_days_member) + ctx.author.joined_at > arrow.utcnow() - timedelta(days=GateConf.minimum_days_member) ), "total_messages": data["total_messages"] < GateConf.minimum_messages, "voice_banned": data["voice_banned"], diff --git a/bot/exts/recruitment/talentpool/_review.py b/bot/exts/recruitment/talentpool/_review.py index dcf73c2cb..d880c524c 100644 --- a/bot/exts/recruitment/talentpool/_review.py +++ b/bot/exts/recruitment/talentpool/_review.py @@ -8,6 +8,7 @@ from collections import Counter from datetime import datetime, timedelta from typing import List, Optional, Union +import arrow from dateutil.parser import isoparse from discord import Embed, Emoji, Member, Message, NoMoreItems, PartialMessage, TextChannel from discord.ext.commands import Context @@ -68,11 +69,11 @@ class Reviewer: log.trace(f"Scheduling review of user with ID {user_id}") user_data = self._pool.cache.get(user_id) - inserted_at = isoparse(user_data['inserted_at']).replace(tzinfo=None) + inserted_at = isoparse(user_data['inserted_at']) review_at = inserted_at + timedelta(days=MAX_DAYS_IN_POOL) # If it's over a day overdue, it's probably an old nomination and shouldn't be automatically reviewed. - if datetime.utcnow() - review_at < timedelta(days=1): + if arrow.utcnow() - review_at < timedelta(days=1): self._review_scheduler.schedule_at(review_at, user_id, self.post_review(user_id, update_database=True)) async def post_review(self, user_id: int, update_database: bool) -> None: @@ -347,7 +348,7 @@ class Reviewer: nomination_times = f"{num_entries} times" if num_entries > 1 else "once" rejection_times = f"{len(history)} times" if len(history) > 1 else "once" - end_time = time_since(isoparse(history[0]['ended_at']).replace(tzinfo=None)) + end_time = time_since(isoparse(history[0]['ended_at'])) review = ( f"They were nominated **{nomination_times}** before" diff --git a/bot/exts/utils/internal.py b/bot/exts/utils/internal.py index 96664929b..165b5917d 100644 --- a/bot/exts/utils/internal.py +++ b/bot/exts/utils/internal.py @@ -5,10 +5,10 @@ import re import textwrap import traceback from collections import Counter -from datetime import datetime from io import StringIO from typing import Any, Optional, Tuple +import arrow import discord from discord.ext.commands import Cog, Context, group, has_any_role, is_owner @@ -29,7 +29,7 @@ class Internal(Cog): self.ln = 0 self.stdout = StringIO() - self.socket_since = datetime.utcnow() + self.socket_since = arrow.utcnow() self.socket_event_total = 0 self.socket_events = Counter() @@ -236,7 +236,7 @@ async def func(): # (None,) -> Any @has_any_role(Roles.admins, Roles.owners, Roles.core_developers) async def socketstats(self, ctx: Context) -> None: """Fetch information on the socket events received from Discord.""" - running_s = (datetime.utcnow() - self.socket_since).total_seconds() + running_s = (arrow.utcnow() - self.socket_since).total_seconds() per_s = self.socket_event_total / running_s diff --git a/bot/exts/utils/ping.py b/bot/exts/utils/ping.py index 43d371d87..9fb5b7b8f 100644 --- a/bot/exts/utils/ping.py +++ b/bot/exts/utils/ping.py @@ -1,5 +1,4 @@ -from datetime import datetime - +import arrow from aiohttp import client_exceptions from discord import Embed from discord.ext import commands @@ -32,7 +31,7 @@ class Latency(commands.Cog): """ # datetime.datetime objects do not have the "milliseconds" attribute. # It must be converted to seconds before converting to milliseconds. - bot_ping = (datetime.utcnow() - ctx.message.created_at.replace(tzinfo=None)).total_seconds() * 1000 + bot_ping = (arrow.utcnow() - ctx.message.created_at).total_seconds() * 1000 if bot_ping <= 0: bot_ping = "Your clock is out of sync, could not calculate ping." else: diff --git a/bot/exts/utils/reminders.py b/bot/exts/utils/reminders.py index 3cb9307a9..3dbcc4513 100644 --- a/bot/exts/utils/reminders.py +++ b/bot/exts/utils/reminders.py @@ -1,7 +1,7 @@ import random import textwrap import typing as t -from datetime import datetime +from datetime import datetime, timezone from operator import itemgetter import discord @@ -52,14 +52,14 @@ class Reminders(Cog): params={'active': 'true'} ) - now = datetime.utcnow() + now = datetime.now(timezone.utc) for reminder in response: is_valid, *_ = self.ensure_valid_reminder(reminder) if not is_valid: continue - remind_at = isoparse(reminder['expiration']).replace(tzinfo=None) + remind_at = isoparse(reminder['expiration']) # If the reminder is already overdue ... if remind_at < now: @@ -144,7 +144,7 @@ class Reminders(Cog): def schedule_reminder(self, reminder: dict) -> None: """A coroutine which sends the reminder once the time is reached, and cancels the running task.""" - reminder_datetime = isoparse(reminder['expiration']).replace(tzinfo=None) + reminder_datetime = isoparse(reminder['expiration']) self.scheduler.schedule_at(reminder_datetime, reminder["id"], self.send_reminder(reminder)) async def _edit_reminder(self, reminder_id: int, payload: dict) -> dict: @@ -333,7 +333,7 @@ class Reminders(Cog): for content, remind_at, id_, mentions in reminders: # Parse and humanize the time, make it pretty :D - remind_datetime = isoparse(remind_at).replace(tzinfo=None) + remind_datetime = isoparse(remind_at) time = discord_timestamp(remind_datetime, TimestampFormats.RELATIVE) mentions = ", ".join([ diff --git a/bot/monkey_patches.py b/bot/monkey_patches.py index e56a19da2..23482f7c3 100644 --- a/bot/monkey_patches.py +++ b/bot/monkey_patches.py @@ -1,5 +1,6 @@ -from datetime import datetime, timedelta +from datetime import timedelta +import arrow from discord import Forbidden, http from discord.ext import commands @@ -38,13 +39,13 @@ def patch_typing() -> None: async def honeybadger_type(self, channel_id: int) -> None: # noqa: ANN001 nonlocal last_403 - if last_403 and (datetime.utcnow() - last_403) < timedelta(minutes=5): + if last_403 and (arrow.utcnow() - last_403) < timedelta(minutes=5): log.warning("Not sending typing event, we got a 403 less than 5 minutes ago.") return try: await original(self, channel_id) except Forbidden: - last_403 = datetime.utcnow() + last_403 = arrow.utcnow() log.warning("Got a 403 from typing event!") pass diff --git a/bot/utils/checks.py b/bot/utils/checks.py index e7f2cfbda..188285684 100644 --- a/bot/utils/checks.py +++ b/bot/utils/checks.py @@ -1,4 +1,3 @@ -import datetime from typing import Callable, Container, Iterable, Optional, Union from discord.ext.commands import ( @@ -137,7 +136,7 @@ def cooldown_with_role_bypass(rate: int, per: float, type: BucketType = BucketTy return # cooldown logic, taken from discord.py internals - current = ctx.message.created_at.replace(tzinfo=datetime.timezone.utc).timestamp() + current = ctx.message.created_at.timestamp() bucket = buckets.get_bucket(ctx.message) retry_after = bucket.update_rate_limit(current) if retry_after: diff --git a/bot/utils/time.py b/bot/utils/time.py index 8cf7d623b..eaa9b72e9 100644 --- a/bot/utils/time.py +++ b/bot/utils/time.py @@ -3,6 +3,7 @@ import re from enum import Enum from typing import Optional, Union +import arrow import dateutil.parser from dateutil.relativedelta import relativedelta @@ -67,9 +68,9 @@ def discord_timestamp(timestamp: ValidTimestamp, format: TimestampFormats = Time # Convert each possible timestamp class to an integer. if isinstance(timestamp, datetime.datetime): - timestamp = (timestamp.replace(tzinfo=None) - datetime.datetime.utcfromtimestamp(0)).total_seconds() + timestamp = (timestamp - arrow.get(0)).total_seconds() elif isinstance(timestamp, datetime.date): - timestamp = (timestamp - datetime.date.fromtimestamp(0)).total_seconds() + timestamp = (timestamp - arrow.get(0)).total_seconds() elif isinstance(timestamp, datetime.timedelta): timestamp = timestamp.total_seconds() elif isinstance(timestamp, relativedelta): @@ -124,7 +125,7 @@ def humanize_delta(delta: relativedelta, precision: str = "seconds", max_units: def get_time_delta(time_string: str) -> str: """Returns the time in human-readable time delta format.""" - date_time = dateutil.parser.isoparse(time_string).replace(tzinfo=None) + date_time = dateutil.parser.isoparse(time_string) time_delta = time_since(date_time) return time_delta @@ -157,7 +158,7 @@ def parse_duration_string(duration: str) -> Optional[relativedelta]: def relativedelta_to_timedelta(delta: relativedelta) -> datetime.timedelta: """Converts a relativedelta object to a timedelta object.""" - utcnow = datetime.datetime.utcnow() + utcnow = arrow.utcnow() return utcnow + delta - utcnow @@ -196,8 +197,8 @@ def format_infraction_with_duration( date_to_formatted = format_infraction(date_to) - date_from = date_from or datetime.datetime.utcnow() - date_to = dateutil.parser.isoparse(date_to).replace(tzinfo=None, microsecond=0) + date_from = date_from or datetime.datetime.now(datetime.timezone.utc) + date_to = dateutil.parser.isoparse(date_to).replace(microsecond=0) delta = relativedelta(date_to, date_from) if absolute: @@ -215,15 +216,15 @@ def until_expiration( """ Get the remaining time until infraction's expiration, in a discord timestamp. - Returns a human-readable version of the remaining duration between datetime.utcnow() and an expiry. + Returns a human-readable version of the remaining duration between arrow.utcnow() and an expiry. Similar to time_since, except that this function doesn't error on a null input and return null if the expiry is in the paste """ if not expiry: return None - now = datetime.datetime.utcnow() - since = dateutil.parser.isoparse(expiry).replace(tzinfo=None, microsecond=0) + now = arrow.utcnow() + since = dateutil.parser.isoparse(expiry).replace(microsecond=0) if since < now: return None diff --git a/tests/bot/test_converters.py b/tests/bot/test_converters.py index ef6c8e19e..988b3857b 100644 --- a/tests/bot/test_converters.py +++ b/tests/bot/test_converters.py @@ -1,6 +1,6 @@ -import datetime import re import unittest +from datetime import MAXYEAR, datetime, timezone from unittest.mock import MagicMock, patch from dateutil.relativedelta import relativedelta @@ -17,7 +17,7 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): cls.context = MagicMock cls.context.author = 'bob' - cls.fixed_utc_now = datetime.datetime.fromisoformat('2019-01-01T00:00:00') + cls.fixed_utc_now = datetime.fromisoformat('2019-01-01T00:00:00+00:00') async def test_tag_name_converter_for_invalid(self): """TagNameConverter should raise the correct exception for invalid tag names.""" @@ -111,7 +111,7 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): expected_datetime = self.fixed_utc_now + relativedelta(**duration_dict) with patch('bot.converters.datetime') as mock_datetime: - mock_datetime.utcnow.return_value = self.fixed_utc_now + mock_datetime.now.return_value = self.fixed_utc_now with self.subTest(duration=duration, duration_dict=duration_dict): converted_datetime = await converter.convert(self.context, duration) @@ -157,52 +157,53 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): async def test_duration_converter_out_of_range(self, mock_datetime): """Duration converter should raise BadArgument if datetime raises a ValueError.""" mock_datetime.__add__.side_effect = ValueError - mock_datetime.utcnow.return_value = mock_datetime + mock_datetime.now.return_value = mock_datetime - duration = f"{datetime.MAXYEAR}y" + duration = f"{MAXYEAR}y" exception_message = f"`{duration}` results in a datetime outside the supported range." with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): await Duration().convert(self.context, duration) async def test_isodatetime_converter_for_valid(self): """ISODateTime converter returns correct datetime for valid datetime string.""" + utc = timezone.utc test_values = ( # `YYYY-mm-ddTHH:MM:SSZ` | `YYYY-mm-dd HH:MM:SSZ` - ('2019-09-02T02:03:05Z', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02 02:03:05Z', datetime.datetime(2019, 9, 2, 2, 3, 5)), + ('2019-09-02T02:03:05Z', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02 02:03:05Z', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), # `YYYY-mm-ddTHH:MM:SS±HH:MM` | `YYYY-mm-dd HH:MM:SS±HH:MM` - ('2019-09-02T03:18:05+01:15', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02 03:18:05+01:15', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02T00:48:05-01:15', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02 00:48:05-01:15', datetime.datetime(2019, 9, 2, 2, 3, 5)), + ('2019-09-02T03:18:05+01:15', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02 03:18:05+01:15', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02T00:48:05-01:15', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02 00:48:05-01:15', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), # `YYYY-mm-ddTHH:MM:SS±HHMM` | `YYYY-mm-dd HH:MM:SS±HHMM` - ('2019-09-02T03:18:05+0115', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02 03:18:05+0115', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02T00:48:05-0115', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02 00:48:05-0115', datetime.datetime(2019, 9, 2, 2, 3, 5)), + ('2019-09-02T03:18:05+0115', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02 03:18:05+0115', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02T00:48:05-0115', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02 00:48:05-0115', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), # `YYYY-mm-ddTHH:MM:SS±HH` | `YYYY-mm-dd HH:MM:SS±HH` - ('2019-09-02 03:03:05+01', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02T01:03:05-01', datetime.datetime(2019, 9, 2, 2, 3, 5)), + ('2019-09-02 03:03:05+01', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02T01:03:05-01', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), # `YYYY-mm-ddTHH:MM:SS` | `YYYY-mm-dd HH:MM:SS` - ('2019-09-02T02:03:05', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02 02:03:05', datetime.datetime(2019, 9, 2, 2, 3, 5)), + ('2019-09-02T02:03:05', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02 02:03:05', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), # `YYYY-mm-ddTHH:MM` | `YYYY-mm-dd HH:MM` - ('2019-11-12T09:15', datetime.datetime(2019, 11, 12, 9, 15)), - ('2019-11-12 09:15', datetime.datetime(2019, 11, 12, 9, 15)), + ('2019-11-12T09:15', datetime(2019, 11, 12, 9, 15, tzinfo=utc)), + ('2019-11-12 09:15', datetime(2019, 11, 12, 9, 15, tzinfo=utc)), # `YYYY-mm-dd` - ('2019-04-01', datetime.datetime(2019, 4, 1)), + ('2019-04-01', datetime(2019, 4, 1, tzinfo=utc)), # `YYYY-mm` - ('2019-02-01', datetime.datetime(2019, 2, 1)), + ('2019-02-01', datetime(2019, 2, 1, tzinfo=utc)), # `YYYY` - ('2025', datetime.datetime(2025, 1, 1)), + ('2025', datetime(2025, 1, 1, tzinfo=utc)), ) converter = ISODateTime() @@ -210,7 +211,6 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): for datetime_string, expected_dt in test_values: with self.subTest(datetime_string=datetime_string, expected_dt=expected_dt): converted_dt = await converter.convert(self.context, datetime_string) - self.assertIsNone(converted_dt.tzinfo) self.assertEqual(converted_dt, expected_dt) async def test_isodatetime_converter_for_invalid(self): diff --git a/tests/bot/utils/test_time.py b/tests/bot/utils/test_time.py index 8edffd1c9..a3dcbfc0a 100644 --- a/tests/bot/utils/test_time.py +++ b/tests/bot/utils/test_time.py @@ -72,9 +72,9 @@ class TimeTests(unittest.TestCase): def test_format_infraction_with_duration_custom_units(self): """format_infraction_with_duration should work for custom max_units.""" test_cases = ( - ('3000-12-12T00:01:00Z', datetime(3000, 12, 11, 12, 5, 5), 6, + ('3000-12-12T00:01:00Z', datetime(3000, 12, 11, 12, 5, 5, tzinfo=timezone.utc), 6, ' (11 hours, 55 minutes and 55 seconds)'), - ('3000-11-23T20:09:00Z', datetime(3000, 4, 25, 20, 15), 20, + ('3000-11-23T20:09:00Z', datetime(3000, 4, 25, 20, 15, tzinfo=timezone.utc), 20, ' (6 months, 28 days, 23 hours and 54 minutes)') ) @@ -84,16 +84,21 @@ class TimeTests(unittest.TestCase): def test_format_infraction_with_duration_normal_usage(self): """format_infraction_with_duration should work for normal usage, across various durations.""" + utc = timezone.utc test_cases = ( - ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 0, 5), 2, ' (12 hours and 55 seconds)'), - ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 0, 5), 1, ' (12 hours)'), - ('2019-12-12T00:00:00Z', datetime(2019, 12, 11, 23, 59), 2, ' (1 minute)'), - ('2019-11-23T20:09:00Z', datetime(2019, 11, 15, 20, 15), 2, ' (7 days and 23 hours)'), - ('2019-11-23T20:09:00Z', datetime(2019, 4, 25, 20, 15), 2, ' (6 months and 28 days)'), - ('2019-11-23T20:58:00Z', datetime(2019, 11, 23, 20, 53), 2, ' (5 minutes)'), - ('2019-11-24T00:00:00Z', datetime(2019, 11, 23, 23, 59, 0), 2, ' (1 minute)'), - ('2019-11-23T23:59:00Z', datetime(2017, 7, 21, 23, 0), 2, ' (2 years and 4 months)'), - ('2019-11-23T23:59:00Z', datetime(2019, 11, 23, 23, 49, 5), 2, + ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 0, 5, tzinfo=utc), 2, + ' (12 hours and 55 seconds)'), + ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 0, 5, tzinfo=utc), 1, ' (12 hours)'), + ('2019-12-12T00:00:00Z', datetime(2019, 12, 11, 23, 59, tzinfo=utc), 2, ' (1 minute)'), + ('2019-11-23T20:09:00Z', datetime(2019, 11, 15, 20, 15, tzinfo=utc), 2, + ' (7 days and 23 hours)'), + ('2019-11-23T20:09:00Z', datetime(2019, 4, 25, 20, 15, tzinfo=utc), 2, + ' (6 months and 28 days)'), + ('2019-11-23T20:58:00Z', datetime(2019, 11, 23, 20, 53, tzinfo=utc), 2, ' (5 minutes)'), + ('2019-11-24T00:00:00Z', datetime(2019, 11, 23, 23, 59, 0, tzinfo=utc), 2, ' (1 minute)'), + ('2019-11-23T23:59:00Z', datetime(2017, 7, 21, 23, 0, tzinfo=utc), 2, + ' (2 years and 4 months)'), + ('2019-11-23T23:59:00Z', datetime(2019, 11, 23, 23, 49, 5, tzinfo=utc), 2, ' (9 minutes and 55 seconds)'), (None, datetime(2019, 11, 23, 23, 49, 5), 2, None), ) -- cgit v1.2.3 From bb0018048ea7ec83abe7a34df263aa0ebb29fecd Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Fri, 22 Oct 2021 21:45:52 +0100 Subject: Use Arrow.fromtimestamp to get an aware datetime Fixes #1905 Fixes BOT-1P9 datetime.fromtimestamp returned an naive datetime, so when comparing to the aware datetime from dateutil.parser.isoparse, it would raise an error. --- bot/exts/moderation/infraction/management.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/exts/moderation/infraction/management.py b/bot/exts/moderation/infraction/management.py index 96c818c47..eaaa1e00b 100644 --- a/bot/exts/moderation/infraction/management.py +++ b/bot/exts/moderation/infraction/management.py @@ -1,9 +1,9 @@ import textwrap import typing as t -from datetime import datetime import dateutil.parser import discord +from arrow import Arrow from dateutil.relativedelta import relativedelta from discord.ext import commands from discord.ext.commands import Context @@ -314,7 +314,7 @@ class ModManagement(commands.Cog): if expires_at is None: duration = "*Permanent*" else: - date_from = datetime.fromtimestamp(float(time.DISCORD_TIMESTAMP_REGEX.match(created).group(1))) + date_from = Arrow.fromtimestamp(float(time.DISCORD_TIMESTAMP_REGEX.match(created).group(1))) date_to = dateutil.parser.isoparse(expires_at) duration = humanize_delta(relativedelta(date_to, date_from)) -- cgit v1.2.3 From 22f155e70cf0219d93e9dc17f6f352884ea4eb57 Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Fri, 22 Oct 2021 21:57:47 +0100 Subject: Use datetime.fromtimestamp so we pass relativedelta a datetime object, not Arrow Fixes #1907 Fixes BOT-1PA --- bot/exts/moderation/infraction/management.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bot/exts/moderation/infraction/management.py b/bot/exts/moderation/infraction/management.py index eaaa1e00b..1cd259a4b 100644 --- a/bot/exts/moderation/infraction/management.py +++ b/bot/exts/moderation/infraction/management.py @@ -1,9 +1,9 @@ import textwrap import typing as t +from datetime import datetime, timezone import dateutil.parser import discord -from arrow import Arrow from dateutil.relativedelta import relativedelta from discord.ext import commands from discord.ext.commands import Context @@ -314,7 +314,10 @@ class ModManagement(commands.Cog): if expires_at is None: duration = "*Permanent*" else: - date_from = Arrow.fromtimestamp(float(time.DISCORD_TIMESTAMP_REGEX.match(created).group(1))) + date_from = datetime.fromtimestamp( + float(time.DISCORD_TIMESTAMP_REGEX.match(created).group(1)), + timezone.utc + ) date_to = dateutil.parser.isoparse(expires_at) duration = humanize_delta(relativedelta(date_to, date_from)) -- cgit v1.2.3 From c504c16f5b438c7c38d60587c0bf5185b3927062 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 24 Oct 2021 21:41:32 +0400 Subject: Unpin All Messages When Moving Help Channels Occasional hiccups in the Discord API would cause unpinning in help channel to sometimes fails. This gets around that by unpinning all messages when making the channel available. Signed-off-by: Hassan Abouelela --- bot/exts/help_channels/_cog.py | 6 ++++++ bot/exts/help_channels/_message.py | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bot/exts/help_channels/_cog.py b/bot/exts/help_channels/_cog.py index 3c6cf7f26..0905cb23d 100644 --- a/bot/exts/help_channels/_cog.py +++ b/bot/exts/help_channels/_cog.py @@ -376,6 +376,12 @@ class HelpChannels(commands.Cog): log.trace(f"Moving #{channel} ({channel.id}) to the Available category.") + # Unpin any previously stuck pins + log.trace(f"Looking for pins stuck in #{channel} ({channel.id}).") + for message in await channel.pins(): + await _message.pin_wrapper(message.id, channel, pin=False) + log.debug(f"Removed a stuck pin from #{channel} ({channel.id}). ID: {message.id}") + await _channel.move_to_bottom( channel=channel, category_id=constants.Categories.help_available, diff --git a/bot/exts/help_channels/_message.py b/bot/exts/help_channels/_message.py index a52c67570..241dd606c 100644 --- a/bot/exts/help_channels/_message.py +++ b/bot/exts/help_channels/_message.py @@ -174,7 +174,7 @@ async def notify(channel: discord.TextChannel, last_notification: t.Optional[Arr async def pin(message: discord.Message) -> None: """Pin an initial question `message` and store it in a cache.""" - if await _pin_wrapper(message.id, message.channel, pin=True): + if await pin_wrapper(message.id, message.channel, pin=True): await _caches.question_messages.set(message.channel.id, message.id) @@ -205,7 +205,7 @@ async def unpin(channel: discord.TextChannel) -> None: if msg_id is None: log.debug(f"#{channel} ({channel.id}) doesn't have a message pinned.") else: - await _pin_wrapper(msg_id, channel, pin=False) + await pin_wrapper(msg_id, channel, pin=False) def _match_bot_embed(message: t.Optional[discord.Message], description: str) -> bool: @@ -220,7 +220,7 @@ def _match_bot_embed(message: t.Optional[discord.Message], description: str) -> return message.author == bot.instance.user and bot_msg_desc.strip() == description.strip() -async def _pin_wrapper(msg_id: int, channel: discord.TextChannel, *, pin: bool) -> bool: +async def pin_wrapper(msg_id: int, channel: discord.TextChannel, *, pin: bool) -> bool: """ Pin message `msg_id` in `channel` if `pin` is True or unpin if it's False. -- cgit v1.2.3 From 6885c6d79d86c5ce8ee93cf9d43d93521e80c3cd Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Mon, 25 Oct 2021 16:54:27 +0200 Subject: Infrac: prioritize mod over bot feedback msg --- bot/exts/filters/filtering.py | 2 +- bot/exts/moderation/infraction/_scheduler.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index 15c12d27c..fa4b83438 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -354,7 +354,7 @@ class Filtering(Cog): # If the filter reason contains `[autoban]`, we want to indeed ban if "[autoban]" in reason.lower(): # We create a new context from that message and make sure the staffer is the bot - # and the feeback message is sent in #mod-alert + # and the feedback message is sent in #mod-alert context = await self.bot.get_context(msg) context.author = self.bot.user context.channel = self.bot.get_channel(Channels.mod_alerts) diff --git a/bot/exts/moderation/infraction/_scheduler.py b/bot/exts/moderation/infraction/_scheduler.py index d4e96b10b..c07b043be 100644 --- a/bot/exts/moderation/infraction/_scheduler.py +++ b/bot/exts/moderation/infraction/_scheduler.py @@ -175,13 +175,7 @@ class InfractionScheduler: dm_log_text = "\nDM: Sent" end_msg = "" - if infraction["actor"] == self.bot.user.id: - log.trace( - f"Infraction #{id_} actor is bot; including the reason in the confirmation message." - ) - if reason: - end_msg = f" (reason: {textwrap.shorten(reason, width=1500, placeholder='...')})" - elif is_mod_channel(ctx.channel): + if is_mod_channel(ctx.channel): log.trace(f"Fetching total infraction count for {user}.") infractions = await self.bot.api_client.get( @@ -190,6 +184,12 @@ class InfractionScheduler: ) total = len(infractions) end_msg = f" (#{id_} ; {total} infraction{ngettext('', 's', total)} total)" + elif infraction["actor"] == self.bot.user.id: + log.trace( + f"Infraction #{id_} actor is bot; including the reason in the confirmation message." + ) + if reason: + end_msg = f" (reason: {textwrap.shorten(reason, width=1500, placeholder='...')})" purge = infraction.get("purge", "") -- cgit v1.2.3 From 4b44480a86ecde68e9f416c9ea8f2b49b1300273 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Mon, 25 Oct 2021 17:16:03 +0200 Subject: Filtering: update auto ban message --- bot/exts/filters/filtering.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index fa4b83438..8d55e7ee3 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -42,9 +42,21 @@ ZALGO_RE = regex.compile(rf"[\p{{NONSPACING MARK}}\p{{ENCLOSING MARK}}--[{VARIAT # Other constants. DAYS_BETWEEN_ALERTS = 3 OFFENSIVE_MSG_DELETE_TIME = timedelta(days=Filter.offensive_msg_delete_days) + +# Autoban +LINK_PASSWORD = "https://support.discord.com/hc/en-us/articles/218410947-I-forgot-my-Password-Where-can-I-set-a-new-one" +LINK_2FA = "https://support.discord.com/hc/en-us/articles/219576828-Setting-up-Two-Factor-Authentication" AUTO_BAN_REASON = ( - "Your account seems to be compromised (%s). " - "You're welcome to appeal this ban once you have regained control of your account." + "Your account has been used to send links to a phishing website. You have been automatically banned. " + "If you are not aware of sending them, that means your account has been compromised.\n\n" + + f"Here is a guide from Discord on [how to change your password]({LINK_PASSWORD}).\n\n" + + f"We also highly recommend that you [enable 2 factor authentication on your account]({LINK_2FA}), " + "for heightened security.\n\n" + + "Once you have changed your password, feel free to follow the instructions at the bottom of " + "this message to appeal your ban.""" ) AUTO_BAN_DURATION = timedelta(days=4) @@ -363,7 +375,7 @@ class Filtering(Cog): self.bot.get_command("tempban"), msg.author, datetime.now() + AUTO_BAN_DURATION, - reason=AUTO_BAN_REASON % reason.lower().replace("[autoban]", "").strip() + reason=AUTO_BAN_REASON.strip() ) break # We don't want multiple filters to trigger -- cgit v1.2.3 From 798150eca45e52350773e88d7845a1af913b462f Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Mon, 25 Oct 2021 17:25:19 +0200 Subject: Filter list: send warning when autoban trigger is added --- bot/constants.py | 1 + bot/exts/filters/filter_lists.py | 7 +++++++ bot/exts/filters/filtering.py | 4 ++-- config-default.yml | 1 + 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/bot/constants.py b/bot/constants.py index f704c9e6a..a75a26a9b 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -444,6 +444,7 @@ class Channels(metaclass=YAMLGetter): incidents: int incidents_archive: int mod_alerts: int + mod_tools: int nominations: int nomination_voting: int organisation: int diff --git a/bot/exts/filters/filter_lists.py b/bot/exts/filters/filter_lists.py index 4b5200684..b1e07185a 100644 --- a/bot/exts/filters/filter_lists.py +++ b/bot/exts/filters/filter_lists.py @@ -6,6 +6,7 @@ from discord.ext.commands import BadArgument, Cog, Context, IDConverter, group, from bot import constants from bot.api import ResponseCodeError from bot.bot import Bot +from bot.constants import Channels from bot.converters import ValidDiscordServerInvite, ValidFilterListType from bot.log import get_logger from bot.pagination import LinePaginator @@ -100,6 +101,12 @@ class FilterLists(Cog): ) raise + # If it is an autoban trigger we send a warning in #mod-tools + if comment and "[autoban]" in comment: + await self.bot.get_channel(Channels.mod_tools).send( + f":warning: heads-up! The new filter `{content}` (`{comment}`) will automatically ban users." + ) + # Insert the item into the cache self.bot.insert_item_into_filter_list_cache(item) await ctx.message.add_reaction("✅") diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index 8d55e7ee3..b20a9c2c9 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -364,7 +364,7 @@ class Filtering(Cog): await self._send_log(filter_name, _filter, msg, stats, reason) # If the filter reason contains `[autoban]`, we want to indeed ban - if "[autoban]" in reason.lower(): + if reason and "[autoban]" in reason.lower(): # We create a new context from that message and make sure the staffer is the bot # and the feedback message is sent in #mod-alert context = await self.bot.get_context(msg) @@ -400,7 +400,7 @@ class Filtering(Cog): ping_everyone = Filter.ping_everyone and _filter.get("ping_everyone", True) # If we are going to autoban, we don't want to ping - if "[autoban]" in reason: + if reason and "[autoban]" in reason: ping_everyone = False eval_msg = "using !eval " if is_eval else "" diff --git a/config-default.yml b/config-default.yml index b61d9c99c..c0e561cca 100644 --- a/config-default.yml +++ b/config-default.yml @@ -207,6 +207,7 @@ guild: incidents_archive: 720668923636351037 mod_alerts: 473092532147060736 mods: &MODS 305126844661760000 + mod_tools: 775413915391098921 nominations: 822920136150745168 nomination_voting: 822853512709931008 organisation: &ORGANISATION 551789653284356126 -- cgit v1.2.3 From 30612bc2f61361ac62a0615181f24366bab3f957 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Mon, 25 Oct 2021 19:20:54 +0200 Subject: Filter list: move warning to #mod-meta --- bot/constants.py | 2 +- bot/exts/filters/filter_lists.py | 4 ++-- config-default.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/constants.py b/bot/constants.py index a75a26a9b..e3846fb3d 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -444,7 +444,7 @@ class Channels(metaclass=YAMLGetter): incidents: int incidents_archive: int mod_alerts: int - mod_tools: int + mod_meta: int nominations: int nomination_voting: int organisation: int diff --git a/bot/exts/filters/filter_lists.py b/bot/exts/filters/filter_lists.py index b1e07185a..4af76af76 100644 --- a/bot/exts/filters/filter_lists.py +++ b/bot/exts/filters/filter_lists.py @@ -101,9 +101,9 @@ class FilterLists(Cog): ) raise - # If it is an autoban trigger we send a warning in #mod-tools + # If it is an autoban trigger we send a warning in #mod-meta if comment and "[autoban]" in comment: - await self.bot.get_channel(Channels.mod_tools).send( + await self.bot.get_channel(Channels.mod_meta).send( f":warning: heads-up! The new filter `{content}` (`{comment}`) will automatically ban users." ) diff --git a/config-default.yml b/config-default.yml index c0e561cca..4a85ccc56 100644 --- a/config-default.yml +++ b/config-default.yml @@ -207,7 +207,7 @@ guild: incidents_archive: 720668923636351037 mod_alerts: 473092532147060736 mods: &MODS 305126844661760000 - mod_tools: 775413915391098921 + mod_meta: 775412552795947058 nominations: 822920136150745168 nomination_voting: 822853512709931008 organisation: &ORGANISATION 551789653284356126 -- cgit v1.2.3 From aa666737ba0bf3cfcd58a4c9b782382d342632fe Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 25 Oct 2021 21:55:16 +0300 Subject: Adjust docstring to #1876 --- bot/exts/moderation/clean.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index d5bfdb485..c01430a04 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -13,9 +13,7 @@ from discord.ext.commands.converter import TextChannelConverter from discord.ext.commands.errors import BadArgument from bot.bot import Bot -from bot.constants import ( - Channels, CleanMessages, Colours, Emojis, Event, Icons, MODERATION_ROLES -) +from bot.constants import Channels, CleanMessages, Colours, Emojis, Event, Icons, MODERATION_ROLES from bot.converters import Age, ISODateTime from bot.exts.moderation.modlog import ModLog from bot.utils.channel import is_mod_channel @@ -439,20 +437,21 @@ class Clean(Cog): Commands for cleaning messages in channels. If arguments are provided, will act as a master command from which all subcommands can be derived. - • `users`: A series of user mentions, ID's, or names. - • `traverse`: The number of messages to look at in each channel. If using the cache, will look at the first - `traverse` messages in the cache. - • `first_limit` and `second_limit`: A message, a duration delta, or an ISO datetime. + + \u2003• `users`: A series of user mentions, ID's, or names. + \u2003• `traverse`: The number of messages to look at in each channel. If using the cache, will look at the + first `traverse` messages in the cache. + \u2003• `first_limit` and `second_limit`: A message, a duration delta, or an ISO datetime. If a message is provided, cleaning will happen in that channel, and channels cannot be provided. If a limit is provided, multiple channels cannot be provided. If only one of them is provided, acts as `clean until`. If both are provided, acts as `clean between`. - • `use_cache`: Whether to use the message cache. + \u2003• `use_cache`: Whether to use the message cache. If not provided, will default to False unless an asterisk is used for the channels. - • `bots_only`: Whether to delete only bots. If specified, users cannot be specified. - • `regex`: A regex pattern the message must contain to be deleted. + \u2003• `bots_only`: Whether to delete only bots. If specified, users cannot be specified. + \u2003• `regex`: A regex pattern the message must contain to be deleted. The pattern must be provided enclosed in backticks. If the pattern contains spaces, it still needs to be enclosed in double quotes on top of that. - • `channels`: A series of channels to delete in, or an asterisk to delete from all channels. + \u2003• `channels`: A series of channels to delete in, or an asterisk to delete from all channels. """ if not any([traverse, users, first_limit, second_limit, regex, channels]): await ctx.send_help(ctx.command) @@ -521,7 +520,7 @@ class Clean(Cog): Delete all messages that match a certain regex, stop cleaning after traversing `traverse` messages. The pattern must be provided enclosed in backticks. - If the pattern contains spaces, and still needs to be enclosed in double quotes on top of that. + If the pattern contains spaces, it still needs to be enclosed in double quotes on top of that. For example: `[0-9]` """ await self._clean_messages(ctx, traverse, regex=regex, channels=channels, use_cache=use_cache) -- cgit v1.2.3 From b42f148955600d85260c43c50260333fe62b823e Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 25 Oct 2021 22:00:58 +0300 Subject: Apply requested style changes --- bot/exts/moderation/clean.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index c01430a04..65ffec88b 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -1,3 +1,4 @@ +import contextlib import logging import re import time @@ -46,7 +47,8 @@ class Regex(Converter): async def convert(self, ctx: Context, argument: str) -> re.Pattern: """Strips the backticks from the string and compiles it to a regex pattern.""" - if not (match := re.fullmatch(r"`(.+?)`", argument)): + match = re.fullmatch(r"`(.+?)`", argument) + if not match: raise BadArgument("Regex pattern missing wrapping backticks") try: return re.compile(match.group(1), re.IGNORECASE + re.DOTALL) @@ -252,12 +254,8 @@ class Clean(Cog): # Ensure that deletion was not canceled if not self.cleaning: return deleted - try: + with contextlib.suppress(NotFound): # Message doesn't exist or was already deleted await message.delete() - except NotFound: - # Message doesn't exist or was already deleted - continue - else: deleted.append(message) return deleted -- cgit v1.2.3 From cae048338aa31a6c9c12a75e2f7f1674d817ce7f Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 25 Oct 2021 22:07:22 +0300 Subject: Improve documentation of global variables --- bot/exts/moderation/clean.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 65ffec88b..9001b4fe2 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -21,12 +21,14 @@ from bot.utils.channel import is_mod_channel log = logging.getLogger(__name__) +# Default number of messages to look at in each channel. DEFAULT_TRAVERSE = 10 +# Number of seconds before command invocations and responses are deleted in non-moderation channels. MESSAGE_DELETE_DELAY = 5 -# Type alias for checks +# Type alias for checks for whether a message should be deleted. Predicate = Callable[[Message], bool] - +# Type alias for message lookup ranges. CleanLimit = Union[Message, Age, ISODateTime] @@ -56,7 +58,7 @@ class Regex(Converter): raise BadArgument(f"Regex error: {e.msg}") -if TYPE_CHECKING: +if TYPE_CHECKING: # Used to allow method resolution in IDEs like in converters.py. CleanChannels = Union[Literal["*"], list[TextChannel]] # noqa: F811 Regex = re.Pattern # noqa: F811 -- cgit v1.2.3 From 37b7a3b5f6424039f11d4ee8d6f087568ebded16 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 25 Oct 2021 22:16:17 +0300 Subject: Update Age converter to use TZ aware datetime --- bot/converters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/converters.py b/bot/converters.py index 0cd06bf5e..0984fa0a3 100644 --- a/bot/converters.py +++ b/bot/converters.py @@ -405,7 +405,7 @@ class Age(DurationDelta): The converter supports the same symbols for each unit of time as its parent class. """ delta = await super().convert(ctx, duration) - now = datetime.utcnow() + now = datetime.now(timezone.utc) try: return now - delta -- cgit v1.2.3 From d19824d76c0cbe793b387002bcf1c6932579a668 Mon Sep 17 00:00:00 2001 From: mbaruh Date: Mon, 25 Oct 2021 22:35:12 +0300 Subject: Remove channel limitation with time range Discussion in the pull request raised some legitimate use cases for supplying a time range for multiple channels (e.g clean the last couple of minutes instead of specifying number of messages to traverse). --- bot/exts/moderation/clean.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index 9001b4fe2..94494b983 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -99,9 +99,6 @@ class Clean(Cog): if traverse > CleanMessages.message_limit: raise BadArgument(f"Cannot traverse more than {CleanMessages.message_limit} messages.") - if first_limit and channels and (channels == "*" or len(channels) > 1): - raise BadArgument("Message or time range specified across multiple channels.") - if (isinstance(first_limit, Message) or isinstance(second_limit, Message)) and channels: raise BadArgument("Both a message limit and channels specified.") -- cgit v1.2.3 From 874978e575ad6ef5c615f4736e47fd0f6b360af8 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Mon, 25 Oct 2021 21:49:34 +0200 Subject: Fltering: clean up autoban code --- bot/exts/filters/filter_lists.py | 2 +- bot/exts/filters/filtering.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/exts/filters/filter_lists.py b/bot/exts/filters/filter_lists.py index 4af76af76..ee5bd89f3 100644 --- a/bot/exts/filters/filter_lists.py +++ b/bot/exts/filters/filter_lists.py @@ -104,7 +104,7 @@ class FilterLists(Cog): # If it is an autoban trigger we send a warning in #mod-meta if comment and "[autoban]" in comment: await self.bot.get_channel(Channels.mod_meta).send( - f":warning: heads-up! The new filter `{content}` (`{comment}`) will automatically ban users." + f":warning: Heads-up! The new filter `{content}` (`{comment}`) will automatically ban users." ) # Insert the item into the cache diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index bda4e7ac2..5e91e7e3d 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -1,6 +1,6 @@ import asyncio import re -from datetime import datetime, timedelta +from datetime import timedelta from typing import Any, Dict, List, Mapping, NamedTuple, Optional, Tuple, Union import arrow @@ -375,8 +375,8 @@ class Filtering(Cog): await context.invoke( self.bot.get_command("tempban"), msg.author, - datetime.now() + AUTO_BAN_DURATION, - reason=AUTO_BAN_REASON.strip() + arrow.utcnow() + AUTO_BAN_DURATION, + reason=AUTO_BAN_REASON ) break # We don't want multiple filters to trigger -- cgit v1.2.3 From d957e2d9158e2fb0e43de33c8b174b759da4d905 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Mon, 25 Oct 2021 21:58:36 +0200 Subject: Filtering: fix ban flow --- bot/exts/filters/filtering.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index 5e91e7e3d..30b447620 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -369,11 +369,12 @@ class Filtering(Cog): # We create a new context from that message and make sure the staffer is the bot # and the feedback message is sent in #mod-alert context = await self.bot.get_context(msg) - context.author = self.bot.user + context.author = self.bot.get_guild(Guild.id).get_member(self.bot.user.id) context.channel = self.bot.get_channel(Channels.mod_alerts) + context.command = self.bot.get_command("tempban") await context.invoke( - self.bot.get_command("tempban"), + context.command, msg.author, arrow.utcnow() + AUTO_BAN_DURATION, reason=AUTO_BAN_REASON -- cgit v1.2.3 From 4594b2b85d714a5e9a65c3ae6e2a2264b1c9b38d Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Mon, 25 Oct 2021 22:33:35 +0200 Subject: Filtering: update auto-ban comments Co-authored-by: ChrisJL --- bot/exts/filters/filtering.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index 30b447620..804f60547 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -364,10 +364,11 @@ class Filtering(Cog): stats = self._add_stats(filter_name, match, msg.content) await self._send_log(filter_name, _filter, msg, stats, reason) - # If the filter reason contains `[autoban]`, we want to indeed ban + # If the filter reason contains `[autoban]`, we want to auto-ban the user if reason and "[autoban]" in reason.lower(): - # We create a new context from that message and make sure the staffer is the bot - # and the feedback message is sent in #mod-alert + # Create a new context, with the author as is the bot, and the channel as #mod-alerts. + # This sends the ban confirmation directly under watchlist trigger embed, to inform + # mods that the user was auto-banned for the message. context = await self.bot.get_context(msg) context.author = self.bot.get_guild(Guild.id).get_member(self.bot.user.id) context.channel = self.bot.get_channel(Channels.mod_alerts) -- cgit v1.2.3 From d55197b405d8fd71bf09ff32dc339215997368fa Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Mon, 25 Oct 2021 22:34:00 +0200 Subject: Filtering: remove dangling empty quote Co-authored-by: ChrisJL --- bot/exts/filters/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index 804f60547..b7a7e8093 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -57,7 +57,7 @@ AUTO_BAN_REASON = ( "for heightened security.\n\n" "Once you have changed your password, feel free to follow the instructions at the bottom of " - "this message to appeal your ban.""" + "this message to appeal your ban." ) AUTO_BAN_DURATION = timedelta(days=4) -- cgit v1.2.3 From e7959146d7949377d35a433ad83f0841070587d9 Mon Sep 17 00:00:00 2001 From: Boris Muratov <8bee278@gmail.com> Date: Tue, 26 Oct 2021 13:29:37 +0300 Subject: Handle autoban filtering in DMs (#1914) An autoban trigger being sent in DMs caused the ban to fail, but for it to still be registered in the database. That is becuase the ban command uses the `ctx.guild.ban` method, but in DMs `ctx.guild` is None. This commit solves it by overriding the `context.guild` field. --- bot/exts/filters/filtering.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index b7a7e8093..022b4ab02 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -370,7 +370,8 @@ class Filtering(Cog): # This sends the ban confirmation directly under watchlist trigger embed, to inform # mods that the user was auto-banned for the message. context = await self.bot.get_context(msg) - context.author = self.bot.get_guild(Guild.id).get_member(self.bot.user.id) + context.guild = self.bot.get_guild(Guild.id) + context.author = context.guild.get_member(self.bot.user.id) context.channel = self.bot.get_channel(Channels.mod_alerts) context.command = self.bot.get_command("tempban") -- cgit v1.2.3 From c4837978399ce42b7073e17bae7e30b7a43d088d Mon Sep 17 00:00:00 2001 From: Lainika Date: Sun, 31 Oct 2021 16:12:28 +0100 Subject: GH-1873 Fix BigBrother embeds Move text from footer to description. --- bot/exts/moderation/watchchannels/_watchchannel.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bot/exts/moderation/watchchannels/_watchchannel.py b/bot/exts/moderation/watchchannels/_watchchannel.py index 8f97130ca..34d445912 100644 --- a/bot/exts/moderation/watchchannels/_watchchannel.py +++ b/bot/exts/moderation/watchchannels/_watchchannel.py @@ -298,8 +298,7 @@ class WatchChannel(metaclass=CogABCMeta): message_jump = f"in [#{msg.channel.name}]({msg.jump_url})" footer = f"Added {time_delta} by {actor} | Reason: {reason}" - embed = Embed(description=f"{msg.author.mention} {message_jump}") - embed.set_footer(text=textwrap.shorten(footer, width=256, placeholder="...")) + embed = Embed(description=f"{msg.author.mention} {message_jump}\n\n{footer}") await self.webhook_send(embed=embed, username=msg.author.display_name, avatar_url=msg.author.display_avatar.url) -- cgit v1.2.3 From a2ec6f93403ee77aef803c0eefb90fa16f60f181 Mon Sep 17 00:00:00 2001 From: Chris Lovering Date: Mon, 25 Oct 2021 13:11:37 +0100 Subject: consider parent channels when checking mod channels --- bot/utils/channel.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bot/utils/channel.py b/bot/utils/channel.py index b9e234857..954a10e56 100644 --- a/bot/utils/channel.py +++ b/bot/utils/channel.py @@ -1,3 +1,5 @@ +from typing import Union + import discord import bot @@ -16,8 +18,11 @@ def is_help_channel(channel: discord.TextChannel) -> bool: return any(is_in_category(channel, category) for category in categories) -def is_mod_channel(channel: discord.TextChannel) -> bool: - """True if `channel` is considered a mod channel.""" +def is_mod_channel(channel: Union[discord.TextChannel, discord.Thread]) -> bool: + """True if channel, or channel.parent for threads, is considered a mod channel.""" + if isinstance(channel, discord.Thread): + channel = channel.parent + if channel.id in constants.MODERATION_CHANNELS: log.trace(f"Channel #{channel} is a configured mod channel") return True -- cgit v1.2.3 From e08764a59443eebd217f67e77bd4f5403e9a189f Mon Sep 17 00:00:00 2001 From: TizzySaurus <47674925+TizzySaurus@users.noreply.github.com> Date: Mon, 1 Nov 2021 18:53:46 +0000 Subject: Migrate to `og_blurple` (#1901) Migrate to `og_blurple` --- bot/exts/backend/branding/_cog.py | 4 ++-- bot/exts/events/code_jams/_cog.py | 2 +- bot/exts/info/information.py | 6 +++--- bot/exts/info/site.py | 12 ++++++------ bot/exts/moderation/defcon.py | 4 ++-- bot/exts/moderation/infraction/management.py | 2 +- bot/exts/moderation/modlog.py | 20 ++++++++++---------- bot/exts/utils/extensions.py | 2 +- bot/exts/utils/internal.py | 2 +- bot/exts/utils/reminders.py | 6 +++--- bot/exts/utils/utils.py | 2 +- tests/bot/exts/info/test_information.py | 14 +++++++------- 12 files changed, 38 insertions(+), 38 deletions(-) diff --git a/bot/exts/backend/branding/_cog.py b/bot/exts/backend/branding/_cog.py index 9c5bdbb4e..0c5839a7a 100644 --- a/bot/exts/backend/branding/_cog.py +++ b/bot/exts/backend/branding/_cog.py @@ -294,7 +294,7 @@ class Branding(commands.Cog): else: content = "Python Discord is entering a new event!" if is_notification else None - embed = discord.Embed(description=description[:4096], colour=discord.Colour.blurple()) + embed = discord.Embed(description=description[:4096], colour=discord.Colour.og_blurple()) embed.set_footer(text=duration[:4096]) await channel.send(content=content, embed=embed) @@ -573,7 +573,7 @@ class Branding(commands.Cog): await ctx.send(embed=resp) return - embed = discord.Embed(title="Current event calendar", colour=discord.Colour.blurple()) + embed = discord.Embed(title="Current event calendar", colour=discord.Colour.og_blurple()) # Because Discord embeds can only contain up to 25 fields, we only show the first 25. first_25 = list(available_events.items())[:25] diff --git a/bot/exts/events/code_jams/_cog.py b/bot/exts/events/code_jams/_cog.py index b31d628d5..452199f5f 100644 --- a/bot/exts/events/code_jams/_cog.py +++ b/bot/exts/events/code_jams/_cog.py @@ -160,7 +160,7 @@ class CodeJams(commands.Cog): embed = Embed( title=str(member), - colour=Colour.blurple() + colour=Colour.og_blurple() ) embed.add_field(name="Team", value=self.team_name(channel), inline=True) diff --git a/bot/exts/info/information.py b/bot/exts/info/information.py index 0dcb8de11..7f4811a43 100644 --- a/bot/exts/info/information.py +++ b/bot/exts/info/information.py @@ -112,7 +112,7 @@ class Information(Cog): # Build an embed embed = Embed( title=f"Role information (Total {len(roles)} role{'s' * (len(role_list) > 1)})", - colour=Colour.blurple() + colour=Colour.og_blurple() ) await LinePaginator.paginate(role_list, ctx, embed, empty=False) @@ -170,7 +170,7 @@ class Information(Cog): @command(name="server", aliases=["server_info", "guild", "guild_info"]) async def server_info(self, ctx: Context) -> None: """Returns an embed full of server information.""" - embed = Embed(colour=Colour.blurple(), title="Server Information") + embed = Embed(colour=Colour.og_blurple(), title="Server Information") created = discord_timestamp(ctx.guild.created_at, TimestampFormats.RELATIVE) region = ctx.guild.region @@ -316,7 +316,7 @@ class Information(Cog): embed.add_field(name=field_name, value=field_content, inline=False) embed.set_thumbnail(url=user.display_avatar.url) - embed.colour = user.colour if user.colour != Colour.default() else Colour.blurple() + embed.colour = user.colour if user.colour != Colour.default() else Colour.og_blurple() return embed diff --git a/bot/exts/info/site.py b/bot/exts/info/site.py index e1f2f5153..e8e71558b 100644 --- a/bot/exts/info/site.py +++ b/bot/exts/info/site.py @@ -29,7 +29,7 @@ class Site(Cog): embed = Embed(title="Python Discord website") embed.set_footer(text=url) - embed.colour = Colour.blurple() + embed.colour = Colour.og_blurple() embed.description = ( f"[Our official website]({url}) is an open-source community project " "created with Python and Django. It contains information about the server " @@ -46,7 +46,7 @@ class Site(Cog): embed = Embed(title="Resources") embed.set_footer(text=f"{learning_url}") - embed.colour = Colour.blurple() + embed.colour = Colour.og_blurple() embed.description = ( f"The [Resources page]({learning_url}) on our website contains a " "list of hand-selected learning resources that we regularly recommend " @@ -62,7 +62,7 @@ class Site(Cog): embed = Embed(title="Tools") embed.set_footer(text=f"{tools_url}") - embed.colour = Colour.blurple() + embed.colour = Colour.og_blurple() embed.description = ( f"The [Tools page]({tools_url}) on our website contains a " f"couple of the most popular tools for programming in Python." @@ -77,7 +77,7 @@ class Site(Cog): embed = Embed(title="Asking Good Questions") embed.set_footer(text=url) - embed.colour = Colour.blurple() + embed.colour = Colour.og_blurple() embed.description = ( "Asking the right question about something that's new to you can sometimes be tricky. " f"To help with this, we've created a [guide to asking good questions]({url}) on our website. " @@ -93,7 +93,7 @@ class Site(Cog): embed = Embed(title="FAQ") embed.set_footer(text=url) - embed.colour = Colour.blurple() + embed.colour = Colour.og_blurple() embed.description = ( "As the largest Python community on Discord, we get hundreds of questions every day. " "Many of these questions have been asked before. We've compiled a list of the most " @@ -106,7 +106,7 @@ class Site(Cog): @site_group.command(name="rules", aliases=("r", "rule"), root_aliases=("rules", "rule")) async def site_rules(self, ctx: Context, rules: Greedy[int]) -> None: """Provides a link to all rules or, if specified, displays specific rule(s).""" - rules_embed = Embed(title='Rules', color=Colour.blurple(), url=f'{BASE_URL}/pages/rules') + rules_embed = Embed(title='Rules', color=Colour.og_blurple(), url=f'{BASE_URL}/pages/rules') if not rules: # Rules were not submitted. Return the default description. diff --git a/bot/exts/moderation/defcon.py b/bot/exts/moderation/defcon.py index 822a87b61..14db37367 100644 --- a/bot/exts/moderation/defcon.py +++ b/bot/exts/moderation/defcon.py @@ -50,7 +50,7 @@ class Action(Enum): SERVER_OPEN = ActionInfo(Icons.defcon_unshutdown, Emojis.defcon_unshutdown, Colours.soft_green, "") SERVER_SHUTDOWN = ActionInfo(Icons.defcon_shutdown, Emojis.defcon_shutdown, Colours.soft_red, "") DURATION_UPDATE = ActionInfo( - Icons.defcon_update, Emojis.defcon_update, Colour.blurple(), "**Threshold:** {threshold}\n\n" + Icons.defcon_update, Emojis.defcon_update, Colour.og_blurple(), "**Threshold:** {threshold}\n\n" ) @@ -152,7 +152,7 @@ class Defcon(Cog): async def status(self, ctx: Context) -> None: """Check the current status of DEFCON mode.""" embed = Embed( - colour=Colour.blurple(), title="DEFCON Status", + colour=Colour.og_blurple(), title="DEFCON Status", description=f""" **Threshold:** {humanize_delta(self.threshold) if self.threshold else "-"} **Expires:** {discord_timestamp(self.expiry, TimestampFormats.RELATIVE) if self.expiry else "-"} diff --git a/bot/exts/moderation/infraction/management.py b/bot/exts/moderation/infraction/management.py index 1cd259a4b..0a33ac5e2 100644 --- a/bot/exts/moderation/infraction/management.py +++ b/bot/exts/moderation/infraction/management.py @@ -203,7 +203,7 @@ class ModManagement(commands.Cog): await self.mod_log.send_log_message( icon_url=constants.Icons.pencil, - colour=discord.Colour.blurple(), + colour=discord.Colour.og_blurple(), title="Infraction edited", thumbnail=thumbnail, text=textwrap.dedent(f""" diff --git a/bot/exts/moderation/modlog.py b/bot/exts/moderation/modlog.py index 6fcf43d8a..462f8533d 100644 --- a/bot/exts/moderation/modlog.py +++ b/bot/exts/moderation/modlog.py @@ -251,7 +251,7 @@ class ModLog(Cog, name="ModLog"): message = f"**#{after.name}** (`{after.id}`)\n{message}" await self.send_log_message( - Icons.hash_blurple, Colour.blurple(), + Icons.hash_blurple, Colour.og_blurple(), "Channel updated", message ) @@ -326,7 +326,7 @@ class ModLog(Cog, name="ModLog"): message = f"**{after.name}** (`{after.id}`)\n{message}" await self.send_log_message( - Icons.crown_blurple, Colour.blurple(), + Icons.crown_blurple, Colour.og_blurple(), "Role updated", message ) @@ -376,7 +376,7 @@ class ModLog(Cog, name="ModLog"): message = f"**{after.name}** (`{after.id}`)\n{message}" await self.send_log_message( - Icons.guild_update, Colour.blurple(), + Icons.guild_update, Colour.og_blurple(), "Guild updated", message, thumbnail=after.icon.with_static_format("png") ) @@ -447,7 +447,7 @@ class ModLog(Cog, name="ModLog"): return await self.send_log_message( - Icons.user_unban, Colour.blurple(), + Icons.user_unban, Colour.og_blurple(), "User unbanned", format_user(member), thumbnail=member.display_avatar.url, channel_id=Channels.mod_log @@ -512,7 +512,7 @@ class ModLog(Cog, name="ModLog"): await self.send_log_message( icon_url=Icons.user_update, - colour=Colour.blurple(), + colour=Colour.og_blurple(), title="Member updated", text=message, thumbnail=after.display_avatar.url, @@ -718,7 +718,7 @@ class ModLog(Cog, name="ModLog"): footer = None await self.send_log_message( - Icons.message_edit, Colour.blurple(), "Message edited", response, + Icons.message_edit, Colour.og_blurple(), "Message edited", response, channel_id=Channels.message_log, timestamp_override=timestamp, footer=footer ) @@ -761,12 +761,12 @@ class ModLog(Cog, name="ModLog"): ) await self.send_log_message( - Icons.message_edit, Colour.blurple(), "Message edited (Before)", + Icons.message_edit, Colour.og_blurple(), "Message edited (Before)", before_response, channel_id=Channels.message_log ) await self.send_log_message( - Icons.message_edit, Colour.blurple(), "Message edited (After)", + Icons.message_edit, Colour.og_blurple(), "Message edited (After)", after_response, channel_id=Channels.message_log ) @@ -776,7 +776,7 @@ class ModLog(Cog, name="ModLog"): if before.name != after.name: await self.send_log_message( Icons.hash_blurple, - Colour.blurple(), + Colour.og_blurple(), "Thread name edited", ( f"Thread {after.mention} (`{after.id}`) from {after.parent.mention} (`{after.parent.id}`): " @@ -870,7 +870,7 @@ class ModLog(Cog, name="ModLog"): diff_values = {**diff.get("values_changed", {}), **diff.get("type_changes", {})} icon = Icons.voice_state_blue - colour = Colour.blurple() + colour = Colour.og_blurple() changes = [] for attr, values in diff_values.items(): diff --git a/bot/exts/utils/extensions.py b/bot/exts/utils/extensions.py index fa5d38917..fda1e49e2 100644 --- a/bot/exts/utils/extensions.py +++ b/bot/exts/utils/extensions.py @@ -113,7 +113,7 @@ class Extensions(commands.Cog): Grey indicates that the extension is unloaded. Green indicates that the extension is currently loaded. """ - embed = Embed(colour=Colour.blurple()) + embed = Embed(colour=Colour.og_blurple()) embed.set_author( name="Extensions List", url=URLs.github_bot_repo, diff --git a/bot/exts/utils/internal.py b/bot/exts/utils/internal.py index 165b5917d..e7113c09c 100644 --- a/bot/exts/utils/internal.py +++ b/bot/exts/utils/internal.py @@ -243,7 +243,7 @@ async def func(): # (None,) -> Any stats_embed = discord.Embed( title="WebSocket statistics", description=f"Receiving {per_s:0.2f} events per second.", - color=discord.Color.blurple() + color=discord.Color.og_blurple() ) for event_type, count in self.socket_events.most_common(25): diff --git a/bot/exts/utils/reminders.py b/bot/exts/utils/reminders.py index 3dbcc4513..86e4505fa 100644 --- a/bot/exts/utils/reminders.py +++ b/bot/exts/utils/reminders.py @@ -183,7 +183,7 @@ class Reminders(Cog): name="Sorry, your reminder should have arrived earlier!" ) else: - embed.colour = discord.Colour.blurple() + embed.colour = discord.Colour.og_blurple() embed.set_author( icon_url=Icons.remind_blurple, name="It has arrived!" @@ -350,7 +350,7 @@ class Reminders(Cog): lines.append(text) embed = discord.Embed() - embed.colour = discord.Colour.blurple() + embed.colour = discord.Colour.og_blurple() embed.title = f"Reminders for {ctx.author}" # Remind the user that they have no reminders :^) @@ -360,7 +360,7 @@ class Reminders(Cog): return # Construct the embed and paginate it. - embed.colour = discord.Colour.blurple() + embed.colour = discord.Colour.og_blurple() await LinePaginator.paginate( lines, diff --git a/bot/exts/utils/utils.py b/bot/exts/utils/utils.py index f69bab781..821cebd8c 100644 --- a/bot/exts/utils/utils.py +++ b/bot/exts/utils/utils.py @@ -96,7 +96,7 @@ class Utils(Cog): If a string is provided, the line which matches best will be produced. """ embed = Embed( - colour=Colour.blurple(), + colour=Colour.og_blurple(), title="The Zen of Python", description=ZEN_OF_PYTHON ) diff --git a/tests/bot/exts/info/test_information.py b/tests/bot/exts/info/test_information.py index 4b50c3fd9..632287322 100644 --- a/tests/bot/exts/info/test_information.py +++ b/tests/bot/exts/info/test_information.py @@ -42,7 +42,7 @@ class InformationCogTests(unittest.IsolatedAsyncioTestCase): embed = kwargs.pop('embed') self.assertEqual(embed.title, "Role information (Total 1 role)") - self.assertEqual(embed.colour, discord.Colour.blurple()) + self.assertEqual(embed.colour, discord.Colour.og_blurple()) self.assertEqual(embed.description, f"\n`{self.moderator_role.id}` - {self.moderator_role.mention}\n") async def test_role_info_command(self): @@ -50,7 +50,7 @@ class InformationCogTests(unittest.IsolatedAsyncioTestCase): dummy_role = helpers.MockRole( name="Dummy", id=112233445566778899, - colour=discord.Colour.blurple(), + colour=discord.Colour.og_blurple(), position=10, members=[self.ctx.author], permissions=discord.Permissions(0) @@ -80,11 +80,11 @@ class InformationCogTests(unittest.IsolatedAsyncioTestCase): admin_embed = admin_kwargs["embed"] self.assertEqual(dummy_embed.title, "Dummy info") - self.assertEqual(dummy_embed.colour, discord.Colour.blurple()) + self.assertEqual(dummy_embed.colour, discord.Colour.og_blurple()) self.assertEqual(dummy_embed.fields[0].value, str(dummy_role.id)) self.assertEqual(dummy_embed.fields[1].value, f"#{dummy_role.colour.value:0>6x}") - self.assertEqual(dummy_embed.fields[2].value, "0.65 0.64 242") + self.assertEqual(dummy_embed.fields[2].value, "0.63 0.48 218") self.assertEqual(dummy_embed.fields[3].value, "1") self.assertEqual(dummy_embed.fields[4].value, "10") self.assertEqual(dummy_embed.fields[5].value, "0") @@ -417,14 +417,14 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) ) - async def test_create_user_embed_uses_blurple_colour_when_user_has_no_roles(self): - """The embed should be created with a blurple colour if the user has no assigned roles.""" + async def test_create_user_embed_uses_og_blurple_colour_when_user_has_no_roles(self): + """The embed should be created with the og blurple colour if the user has no assigned roles.""" ctx = helpers.MockContext() user = helpers.MockMember(id=217, colour=discord.Colour.default()) embed = await self.cog.create_user_embed(ctx, user) - self.assertEqual(embed.colour, discord.Colour.blurple()) + self.assertEqual(embed.colour, discord.Colour.og_blurple()) @unittest.mock.patch( f"{COG_PATH}.basic_user_infraction_counts", -- cgit v1.2.3 From 69fdd3649e2f9a646b97e445bc4d5440440e5890 Mon Sep 17 00:00:00 2001 From: Janine vN Date: Thu, 4 Nov 2021 12:30:06 -0400 Subject: Add sql-fstring tag * Add sql-fstring tag * Correct link and wording * Correction to grammar and wording Also adds a semicolon * Add missing " Co-authored-by: Bluenix Co-authored-by: TizzySaurus <47674925+TizzySaurus@users.noreply.github.com> --- bot/resources/tags/sql-fstring.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 bot/resources/tags/sql-fstring.md diff --git a/bot/resources/tags/sql-fstring.md b/bot/resources/tags/sql-fstring.md new file mode 100644 index 000000000..94dd870fd --- /dev/null +++ b/bot/resources/tags/sql-fstring.md @@ -0,0 +1,16 @@ +**SQL & f-strings** +Don't use f-strings (`f""`) or other forms of "string interpolation" (`%`, `+`, `.format`) to inject data into a SQL query. It is an endless source of bugs and syntax errors. Additionally, in user-facing applications, it presents a major security risk via SQL injection. + +Your database library should support "query parameters". A query parameter is a placeholder that you put in the SQL query. When the query is executed, you provide data to the database library, and the library inserts the data into the query for you, **safely**. + +For example, the sqlite3 package supports using `?` as a placeholder: +```py +query = "SELECT * FROM stocks WHERE symbol = ?;" +params = ("RHAT",) +db.execute(query, params) +``` +Note: Different database libraries support different placeholder styles, e.g. `%s` and `$1`. Consult your library's documentation for details. + +**See Also** +• [Extended Example with SQLite](https://docs.python.org/3/library/sqlite3.html) (search for "Instead, use the DB-API's parameter substitution") +• [PEP-249](https://www.python.org/dev/peps/pep-0249) - A specification of how database libraries in Python should work -- cgit v1.2.3 From 67390298852513d13e0213870e50fb3cff1424e0 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Fri, 5 Nov 2021 16:31:05 +0400 Subject: Merge pull request from GHSA-j8c3-8x46-8pp6 * Don't Exit Token Filtering Early On URLs The token filtering function would exit early if it detected a URL within the message, but it made no extra checks to ensure there weren't other tokens within that message that would trigger it. This made sense when the filtering logic was written, but it's been modified since to introduce this bug. Regression tests included. Signed-off-by: Hassan Abouelela * Links Advisory In Token Filter Tests Adds a link to the advisory with reasoning for the existence of the test. Signed-off-by: Hassan Abouelela --- bot/exts/filters/filtering.py | 4 ---- tests/bot/exts/filters/test_filtering.py | 40 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 tests/bot/exts/filters/test_filtering.py diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index 022b4ab02..f05b1d00b 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -496,10 +496,6 @@ class Filtering(Cog): text = self.clean_input(text) - # Make sure it's not a URL - if URL_RE.search(text): - return False, None - watchlist_patterns = self._get_filterlist_items('filter_token', allowed=False) for pattern in watchlist_patterns: match = re.search(pattern, text, flags=re.IGNORECASE) diff --git a/tests/bot/exts/filters/test_filtering.py b/tests/bot/exts/filters/test_filtering.py new file mode 100644 index 000000000..8ae59c1f1 --- /dev/null +++ b/tests/bot/exts/filters/test_filtering.py @@ -0,0 +1,40 @@ +import unittest +from unittest.mock import patch + +from bot.exts.filters import filtering +from tests.helpers import MockBot, autospec + + +class FilteringCogTests(unittest.IsolatedAsyncioTestCase): + """Tests the `Filtering` cog.""" + + def setUp(self): + """Instantiate the bot and cog.""" + self.bot = MockBot() + with patch("bot.utils.scheduling.create_task", new=lambda task, **_: task.close()): + self.cog = filtering.Filtering(self.bot) + + @autospec(filtering.Filtering, "_get_filterlist_items", pass_mocks=False, return_value=["TOKEN"]) + async def test_token_filter(self): + """Ensure that a filter token is correctly detected in a message.""" + messages = { + "": False, + "no matches": False, + "TOKEN": True, + + # See advisory https://github.com/python-discord/bot/security/advisories/GHSA-j8c3-8x46-8pp6 + "https://google.com TOKEN": True, + "https://google.com something else": False, + } + + for message, match in messages.items(): + with self.subTest(input=message, match=match): + result, _ = await self.cog._has_watch_regex_match(message) + + self.assertEqual( + match, + bool(result), + msg=f"Hit was {'expected' if match else 'not expected'} for this input." + ) + if result: + self.assertEqual("TOKEN", result.group()) -- cgit v1.2.3 From 1302632c39469a124e85e6b43c5278526874ce5e Mon Sep 17 00:00:00 2001 From: TizzySaurus <47674925+TizzySaurus@users.noreply.github.com> Date: Sat, 6 Nov 2021 10:56:09 +0000 Subject: Only re-run filters in `on_message_update` if contents/attachments changed (#1937) --- bot/exts/filters/filtering.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index f05b1d00b..79b7abe9f 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -189,8 +189,16 @@ class Filtering(Cog): """ Invoke message filter for message edits. - If there have been multiple edits, calculate the time delta from the previous edit. + Also calculates the time delta from the previous edit or when message was sent if there's no prior edits. """ + # We only care about changes to the message contents/attachments and embed additions, not pin status etc. + if all(( + before.content == after.content, # content hasn't changed + before.attachments == after.attachments, # attachments haven't changed + len(before.embeds) >= len(after.embeds) # embeds haven't been added + )): + return + if not before.edited_at: delta = relativedelta(after.edited_at, before.created_at).microseconds else: @@ -341,7 +349,7 @@ class Filtering(Cog): await self.notify_member(msg.author, _filter["notification_msg"], msg.channel) # If the message is classed as offensive, we store it in the site db and - # it will be deleted it after one week. + # it will be deleted after one week. if _filter["schedule_deletion"] and not is_private: delete_date = (msg.created_at + OFFENSIVE_MSG_DELETE_TIME).isoformat() data = { -- cgit v1.2.3 From 19df10cb968081442e9854ffffcdbc7c783dd0af Mon Sep 17 00:00:00 2001 From: Qwerty-133 <74311372+Qwerty-133@users.noreply.github.com> Date: Tue, 9 Nov 2021 15:27:14 +0530 Subject: Filter the same messages in both listeners. --- bot/exts/moderation/modlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/moderation/modlog.py b/bot/exts/moderation/modlog.py index 462f8533d..1d8e571fb 100644 --- a/bot/exts/moderation/modlog.py +++ b/bot/exts/moderation/modlog.py @@ -613,7 +613,7 @@ class ModLog(Cog, name="ModLog"): @Cog.listener() async def on_raw_message_delete(self, event: discord.RawMessageDeleteEvent) -> None: """Log raw message delete event to message change log.""" - if self.is_channel_ignored(event.channel_id): + if self.is_message_blacklisted(event.channel_id): return await asyncio.sleep(1) # Wait here in case the normal event was fired -- cgit v1.2.3 From 2d2e7a31699a4eedd1f14aecf348368711b53536 Mon Sep 17 00:00:00 2001 From: Qwerty-133 <74311372+Qwerty-133@users.noreply.github.com> Date: Tue, 9 Nov 2021 16:14:59 +0530 Subject: Call the appropriate function in the raw listener --- bot/exts/moderation/modlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/moderation/modlog.py b/bot/exts/moderation/modlog.py index 1d8e571fb..462f8533d 100644 --- a/bot/exts/moderation/modlog.py +++ b/bot/exts/moderation/modlog.py @@ -613,7 +613,7 @@ class ModLog(Cog, name="ModLog"): @Cog.listener() async def on_raw_message_delete(self, event: discord.RawMessageDeleteEvent) -> None: """Log raw message delete event to message change log.""" - if self.is_message_blacklisted(event.channel_id): + if self.is_channel_ignored(event.channel_id): return await asyncio.sleep(1) # Wait here in case the normal event was fired -- cgit v1.2.3 From 78957a7ae4fbbe974b59f6a70fee5c0970378629 Mon Sep 17 00:00:00 2001 From: Qwerty-133 <74311372+Qwerty-133@users.noreply.github.com> Date: Tue, 9 Nov 2021 16:27:01 +0530 Subject: Listen to only on_raw_message_delete --- bot/exts/moderation/modlog.py | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/bot/exts/moderation/modlog.py b/bot/exts/moderation/modlog.py index 462f8533d..6416bc3c7 100644 --- a/bot/exts/moderation/modlog.py +++ b/bot/exts/moderation/modlog.py @@ -41,7 +41,6 @@ class ModLog(Cog, name="ModLog"): self.bot = bot self._ignored = {event: [] for event in Event} - self._cached_deletes = [] self._cached_edits = [] async def upload_log( @@ -552,24 +551,22 @@ class ModLog(Cog, name="ModLog"): return channel.id in GuildConstant.modlog_blacklist - @Cog.listener() - async def on_message_delete(self, message: discord.Message) -> None: - """Log message delete event to message change log.""" + async def log_cached_deleted_message(self, message: discord.Message) -> None: + """ + Log the message's details to message change log. + + This is called when a cached message is deleted. + """ channel = message.channel author = message.author if self.is_message_blacklisted(message): return - self._cached_deletes.append(message.id) - if message.id in self._ignored[Event.message_delete]: self._ignored[Event.message_delete].remove(message.id) return - if author.bot: - return - if channel.category: response = ( f"**Author:** {format_user(author)}\n" @@ -610,17 +607,14 @@ class ModLog(Cog, name="ModLog"): channel_id=Channels.message_log ) - @Cog.listener() - async def on_raw_message_delete(self, event: discord.RawMessageDeleteEvent) -> None: - """Log raw message delete event to message change log.""" - if self.is_channel_ignored(event.channel_id): - return - - await asyncio.sleep(1) # Wait here in case the normal event was fired + async def log_uncached_deleted_message(self, event: discord.RawMessageDeleteEvent) -> None: + """ + Log the message's details to message change log. - if event.message_id in self._cached_deletes: - # It was in the cache and the normal event was fired, so we can just ignore it - self._cached_deletes.remove(event.message_id) + This is called when a message absent from the cache is deleted. + Hence, the message contents aren't logged. + """ + if self.is_channel_ignored(event.channel_id): return if event.message_id in self._ignored[Event.message_delete]: @@ -651,6 +645,14 @@ class ModLog(Cog, name="ModLog"): channel_id=Channels.message_log ) + @Cog.listener() + async def on_raw_message_delete(self, event: discord.RawMessageDeleteEvent) -> None: + """Log message deletions to message change log.""" + if event.cached_message is not None: + await self.log_cached_deleted_message(event.cached_message) + else: + await self.log_uncached_deleted_message(event) + @Cog.listener() async def on_message_edit(self, msg_before: discord.Message, msg_after: discord.Message) -> None: """Log message edit event to message change log.""" -- cgit v1.2.3 From 59c05a8494a5425545d38029b4b26890b6a631c1 Mon Sep 17 00:00:00 2001 From: aru Date: Wed, 10 Nov 2021 01:26:40 -0500 Subject: commands: add pip as an alias to pypi (#1942) Co-authored-by: Xithrius <15021300+Xithrius@users.noreply.github.com> --- bot/exts/info/pypi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/info/pypi.py b/bot/exts/info/pypi.py index c3d2e2a3c..dacf7bc12 100644 --- a/bot/exts/info/pypi.py +++ b/bot/exts/info/pypi.py @@ -29,7 +29,7 @@ class PyPi(Cog): def __init__(self, bot: Bot): self.bot = bot - @command(name="pypi", aliases=("package", "pack")) + @command(name="pypi", aliases=("package", "pack", "pip")) async def get_package_info(self, ctx: Context, package: str) -> None: """Provide information about a specific package from PyPI.""" embed = Embed(title=random.choice(NEGATIVE_REPLIES), colour=Colours.soft_red) -- cgit v1.2.3