From 50548104768b500b12ef5773d3654565b3eb84d1 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 21 Jul 2018 02:46:12 +0200 Subject: initial commit for the cleaning cog. Not completely done, need to merge in master so I can work with new changes from modlog merge. --- bot/__main__.py | 1 + bot/cogs/clean.py | 152 +++++++++++++++++++++++++++++++++++++++++++++++++++++ bot/constants.py | 9 ++++ config-default.yml | 2 + 4 files changed, 164 insertions(+) create mode 100644 bot/cogs/clean.py diff --git a/bot/__main__.py b/bot/__main__.py index 9014bce14..70336cb59 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -53,6 +53,7 @@ bot.load_extension("bot.cogs.events") # Commands, etc bot.load_extension("bot.cogs.bigbrother") bot.load_extension("bot.cogs.bot") +bot.load_extension("bot.cogs.clean") bot.load_extension("bot.cogs.cogs") # Local setups usually don't have the clickup key set, diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py new file mode 100644 index 000000000..8848befab --- /dev/null +++ b/bot/cogs/clean.py @@ -0,0 +1,152 @@ +import json +import logging +import random + +from discord import Embed, Message, User, Colour +from discord.ext.commands import Bot, Context, group + +from bot.constants import Roles, Clean, URLs, Keys, NEGATIVE_REPLIES +from bot.decorators import with_role + +log = logging.getLogger(__name__) + + +class Clean: + + def __init__(self, bot: Bot): + self.bot = bot + self.headers = {"X-API-KEY": Keys.site_api} + self.cleaning = False + + async def _upload_log(self, log_data): + """ + Uploads the log data to the database via + an API endpoint for uploading logs. + + Returns a URL that can be used to view the log. + """ + + response = await self.bot.http_session.post( + URLs.site_clean_api, + headers=self.headers, + json={"log_data": log_data} + ) + + data = await response.json() + log_id = data["log_id"] + + return f"{URLs.site_clean_logs}/{log_id}" + + async def _clean_messages(self, amount, channel, bots_only: bool=False, user: User=None): + """ + A helper function that does the actual message cleaning. + + :param bots_only: Set this to True if you only want to delete bot messages. + :param user: Specify a user and it will only delete messages by this user. + :return: Returns an embed + """ + + # Is this an acceptable amount of messages to clean? + if amount > Clean.message_limit: + embed = Embed( + color=Colour.red(), + title=random.choice(NEGATIVE_REPLIES), + description=f"You cannot clean more than {Clean.message_limit} messages." + ) + return embed + + # Are we already performing a clean? + if self.cleaning: + embed = Embed( + color=Colour.red(), + title=random.choice(NEGATIVE_REPLIES), + description="Multiple simultaneous cleaning processes is not allowed." + ) + return embed + + # Skip the first message, as that will be the invocation + history = channel.history(limit=amount) + await history.next() + + message_log = [] + + async for message in history: + + delete_condition = ( + bots_only and message.author.bot # Delete bot messages + or user and message.author == user # Delete user messages + or not bots_only and not user # Delete all messages + ) + + if delete_condition: + await message.delete() + content = message.content or message.embeds[0].description + author = f"{message.author.name}#{message.author.discriminator}" + message_log.append({ + "content": content, + "author": author, + "timestamp": message.created_at.strftime("%D %H:%M") + }) + + if message_log: + # Reverse the list to restore chronological order + message_log = list(reversed(message_log)) + upload_log = await self._upload_log(message_log) + else: + upload_log = "Naw, nothing there!" + + embed = Embed( + description=upload_log + ) + + return embed + + @group(invoke_without_command=True, name="clean", hidden=True) + @with_role(Roles.moderator, Roles.admin, Roles.owner) + async def clean_group(self, ctx: Context): + """ + Commands for cleaning messages in channels + """ + + await ctx.invoke(self.bot.get_command("help"), "clean") + + @clean_group.command(aliases=["user"]) + @with_role(Roles.moderator, Roles.admin, Roles.owner) + async def clean_user(self, ctx: Context, user: User, amount: int = 10): + """ + Delete messages posted by the provided user, + and stop cleaning after traversing `amount` messages. + """ + + embed = await self._clean_messages(amount, ctx.channel, user=user) + + await ctx.send(embed=embed) + + @clean_group.command(aliases=["all"]) + @with_role(Roles.moderator, Roles.admin, Roles.owner) + async def clean_all(self, ctx: Context, amount: int = 10): + """ + Delete all messages, regardless of posted, + and stop cleaning after traversing `amount` messages. + """ + + embed = await self._clean_messages(amount, ctx.channel) + + await ctx.send(embed=embed) + + @clean_group.command(aliases=["bots"]) + @with_role(Roles.moderator, Roles.admin, Roles.owner) + async def clean_bots(self, ctx: Context, amount: int = 10): + """ + Delete all messages posted by a bot, + and stop cleaning after traversing `amount` messages. + """ + + embed = await self._clean_messages(amount, ctx.channel, bots_only=True) + + await ctx.send(embed=embed) + + +def setup(bot): + bot.add_cog(Clean(bot)) + log.info("Cog loaded: Clean") diff --git a/bot/constants.py b/bot/constants.py index 7248cd36a..6a8eb6fe2 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -176,6 +176,13 @@ class Emojis(metaclass=YAMLGetter): white_chevron: str +class Clean(metaclass=YAMLGetter): + section = "bot" + subsection = "clean" + + message_limit: int + + class Channels(metaclass=YAMLGetter): section = "guild" subsection = "channels" @@ -259,6 +266,8 @@ class URLs(metaclass=YAMLGetter): omdb: str site: str site_facts_api: str + site_clean_api: str + site_clean_logs: str site_hiphopify_api: str site_idioms_api: str site_names_api: str diff --git a/config-default.yml b/config-default.yml index bd77e695e..ca15040e8 100644 --- a/config-default.yml +++ b/config-default.yml @@ -80,6 +80,8 @@ urls: omdb: 'http://omdbapi.com' site: 'pythondiscord.com' site_bigbrother_api: 'https://api.pythondiscord.com/bot/bigbrother' + site_clean_api: 'https://api.pythondiscord.com/bot/clean' + site_clean_logs: 'https://pythondiscord.com/bot/clean_logs' site_docs_api: 'https://api.pythondiscord.com/bot/docs' site_facts_api: 'https://api.pythondiscord.com/bot/snake_facts' site_hiphopify_api: 'https://api.pythondiscord.com/bot/hiphopify' -- cgit v1.2.3 From 3fd68ad50801f4c5fc006fa043fc4fac4f50c9fe Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 21 Jul 2018 04:46:08 +0200 Subject: Completed clean cog --- bot/cogs/clean.py | 132 +++++++++++++++++++++++++++++++++++++++++------------ bot/constants.py | 2 +- config-default.yml | 4 ++ 3 files changed, 108 insertions(+), 30 deletions(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index 8848befab..7675206a7 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -1,15 +1,20 @@ -import json import logging import random -from discord import Embed, Message, User, Colour +from discord import Colour, Embed, Message, User from discord.ext.commands import Bot, Context, group -from bot.constants import Roles, Clean, URLs, Keys, NEGATIVE_REPLIES +from bot.cogs.modlog import ModLog +from bot.constants import ( + Channels, CleanMessages, Icons, + Keys, NEGATIVE_REPLIES, Roles, URLs +) from bot.decorators import with_role log = logging.getLogger(__name__) +COLOUR_RED = Colour(0xcd6d6d) + class Clean: @@ -18,7 +23,11 @@ class Clean: self.headers = {"X-API-KEY": Keys.site_api} self.cleaning = False - async def _upload_log(self, log_data): + @property + def mod_log(self) -> ModLog: + return self.bot.get_cog("ModLog") + + async def _upload_log(self, log_data: list) -> str: """ Uploads the log data to the database via an API endpoint for uploading logs. @@ -37,23 +46,33 @@ class Clean: return f"{URLs.site_clean_logs}/{log_id}" - async def _clean_messages(self, amount, channel, bots_only: bool=False, user: User=None): + async def _clean_messages( + self, amount: int, ctx: Context, + bots_only: bool=False, user: User=None + ): """ A helper function that does the actual message cleaning. :param bots_only: Set this to True if you only want to delete bot messages. :param user: Specify a user and it will only delete messages by this user. - :return: Returns an embed """ + # Bulk delete checks + def predicate_bots_only(message: Message): + return message.author.bot + + def predicate_specific_user(message: Message): + return message.author == user + # Is this an acceptable amount of messages to clean? - if amount > Clean.message_limit: + if amount > CleanMessages.message_limit: embed = Embed( color=Colour.red(), title=random.choice(NEGATIVE_REPLIES), - description=f"You cannot clean more than {Clean.message_limit} messages." + description=f"You cannot clean more than {CleanMessages.message_limit} messages." ) - return embed + await ctx.send(embed=embed) + return # Are we already performing a clean? if self.cleaning: @@ -62,44 +81,89 @@ class Clean: title=random.choice(NEGATIVE_REPLIES), description="Multiple simultaneous cleaning processes is not allowed." ) - return embed - - # Skip the first message, as that will be the invocation - history = channel.history(limit=amount) - await history.next() + await ctx.send(embed=embed) + return + # Look through the history and retrieve message data message_log = [] + message_ids = [] - async for message in history: + self.cleaning = True - delete_condition = ( + async for message in ctx.channel.history(limit=amount): + + if not self.cleaning: + return + + delete = ( bots_only and message.author.bot # Delete bot messages or user and message.author == user # Delete user messages or not bots_only and not user # Delete all messages ) - if delete_condition: - await message.delete() + if delete and message.content or message.embeds: content = message.content or message.embeds[0].description author = f"{message.author.name}#{message.author.discriminator}" + + # Store the message data + message_ids.append(message.id) message_log.append({ "content": content, "author": author, "timestamp": message.created_at.strftime("%D %H:%M") }) + self.cleaning = False + + # We should ignore the ID's we stored, so we don't get mod-log spam. + self.mod_log.ignore_message_deletion(*message_ids) + + # Use bulk delete to actually do the cleaning. It's far faster. + if bots_only: + await ctx.channel.purge( + limit=amount, + check=predicate_bots_only, + ) + elif user: + await ctx.channel.purge( + limit=amount, + check=predicate_specific_user, + ) + else: + await ctx.channel.purge( + limit=amount + ) + + # Reverse the list to restore chronological order if message_log: - # Reverse the list to restore chronological order message_log = list(reversed(message_log)) upload_log = await self._upload_log(message_log) else: - upload_log = "Naw, nothing there!" + # Can't build an embed, nothing to clean! + embed = Embed( + color=Colour.red(), + description="No matching messages could be found." + ) + await ctx.send(embed=embed) + return + + # Build the embed and send it + message = ( + f"**{len(message_ids)}** messages deleted in <#{ctx.channel.id}> by **{ctx.author.name}**\n\n" + f"A log of the deleted messages can be found [here]({upload_log})." + ) embed = Embed( - description=upload_log + color=COLOUR_RED, + description=message + ) + + embed.set_author( + name=f"Bulk message delete", + icon_url=Icons.message_bulk_delete ) - return embed + await self.bot.get_channel(Channels.modlog).send(embed=embed) @group(invoke_without_command=True, name="clean", hidden=True) @with_role(Roles.moderator, Roles.admin, Roles.owner) @@ -118,9 +182,7 @@ class Clean: and stop cleaning after traversing `amount` messages. """ - embed = await self._clean_messages(amount, ctx.channel, user=user) - - await ctx.send(embed=embed) + await self._clean_messages(amount, ctx, user=user) @clean_group.command(aliases=["all"]) @with_role(Roles.moderator, Roles.admin, Roles.owner) @@ -130,9 +192,7 @@ class Clean: and stop cleaning after traversing `amount` messages. """ - embed = await self._clean_messages(amount, ctx.channel) - - await ctx.send(embed=embed) + await self._clean_messages(amount, ctx) @clean_group.command(aliases=["bots"]) @with_role(Roles.moderator, Roles.admin, Roles.owner) @@ -142,8 +202,22 @@ class Clean: and stop cleaning after traversing `amount` messages. """ - embed = await self._clean_messages(amount, ctx.channel, bots_only=True) + await self._clean_messages(amount, ctx, bots_only=True) + @clean_group.command(aliases=["stop", "cancel", "abort"]) + @with_role(Roles.moderator, Roles.admin, Roles.owner) + async def clean_cancel(self, ctx: Context): + """ + If there is an ongoing cleaning process, + attempt to immediately cancel it. + """ + + self.cleaning = False + + embed = Embed( + color=Colour.blurple(), + description="Clean interrupted." + ) await ctx.send(embed=embed) diff --git a/bot/constants.py b/bot/constants.py index 182510e99..b49cb48f6 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -236,7 +236,7 @@ class Icons(metaclass=YAMLGetter): user_update: str -class Clean(metaclass=YAMLGetter): +class CleanMessages(metaclass=YAMLGetter): section = "bot" subsection = "clean" diff --git a/config-default.yml b/config-default.yml index 6dc3cce63..118ba165d 100644 --- a/config-default.yml +++ b/config-default.yml @@ -37,6 +37,10 @@ bot: user_unban: "https://cdn.discordapp.com/emojis/469952898692808704.png" user_update: "https://cdn.discordapp.com/emojis/469952898684551168.png" + clean: + # Maximum number of messages to traverse for clean commands + message_limit: 10000 + guild: id: 267624335836053506 -- cgit v1.2.3 From d321962d0d966bd9b77c05a89fcc2862a6e98a01 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 22 Jul 2018 12:15:09 +0200 Subject: Moving constants into the yaml, fixing feedback from gdudes review. --- bot/cogs/clean.py | 35 +++++++++++++++-------------------- bot/cogs/modlog.py | 29 +++++++++++++---------------- bot/constants.py | 13 +++++++++++-- config-default.yml | 15 +++++++++++---- 4 files changed, 50 insertions(+), 42 deletions(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index 7675206a7..fe9b14003 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -6,15 +6,13 @@ from discord.ext.commands import Bot, Context, group from bot.cogs.modlog import ModLog from bot.constants import ( - Channels, CleanMessages, Icons, + Channels, CleanMessages, Colours, Icons, Keys, NEGATIVE_REPLIES, Roles, URLs ) from bot.decorators import with_role log = logging.getLogger(__name__) -COLOUR_RED = Colour(0xcd6d6d) - class Clean: @@ -67,7 +65,7 @@ class Clean: # Is this an acceptable amount of messages to clean? if amount > CleanMessages.message_limit: embed = Embed( - color=Colour.red(), + color=Colour(Colours.soft_red), title=random.choice(NEGATIVE_REPLIES), description=f"You cannot clean more than {CleanMessages.message_limit} messages." ) @@ -77,7 +75,7 @@ class Clean: # Are we already performing a clean? if self.cleaning: embed = Embed( - color=Colour.red(), + color=Colour(Colours.soft_red), title=random.choice(NEGATIVE_REPLIES), description="Multiple simultaneous cleaning processes is not allowed." ) @@ -119,20 +117,17 @@ class Clean: self.mod_log.ignore_message_deletion(*message_ids) # Use bulk delete to actually do the cleaning. It's far faster. + predicate = None + if bots_only: - await ctx.channel.purge( - limit=amount, - check=predicate_bots_only, - ) + predicate = predicate_bots_only elif user: - await ctx.channel.purge( - limit=amount, - check=predicate_specific_user, - ) - else: - await ctx.channel.purge( - limit=amount - ) + predicate = predicate_specific_user + + await ctx.channel.purge( + limit=amount, + check=predicate + ) # Reverse the list to restore chronological order if message_log: @@ -141,7 +136,7 @@ class Clean: else: # Can't build an embed, nothing to clean! embed = Embed( - color=Colour.red(), + color=Colour(Colours.soft_red), description="No matching messages could be found." ) await ctx.send(embed=embed) @@ -154,7 +149,7 @@ class Clean: ) embed = Embed( - color=COLOUR_RED, + color=Colour(Colours.soft_red), description=message ) @@ -188,7 +183,7 @@ class Clean: @with_role(Roles.moderator, Roles.admin, Roles.owner) async def clean_all(self, ctx: Context, amount: int = 10): """ - Delete all messages, regardless of posted, + Delete all messages, regardless of poster, and stop cleaning after traversing `amount` messages. """ diff --git a/bot/cogs/modlog.py b/bot/cogs/modlog.py index 2f13fe32b..dfeca473d 100644 --- a/bot/cogs/modlog.py +++ b/bot/cogs/modlog.py @@ -13,15 +13,12 @@ from discord import ( from discord.abc import GuildChannel from discord.ext.commands import Bot -from bot.constants import Channels, Emojis, Icons +from bot.constants import Channels, Colours, Emojis, Icons from bot.constants import Guild as GuildConstant log = logging.getLogger(__name__) -BULLET_POINT = "\u2022" -COLOUR_RED = Colour(0xcd6d6d) -COLOUR_GREEN = Colour(0x68c290) GUILD_CHANNEL = Union[CategoryChannel, TextChannel, VoiceChannel] CHANNEL_CHANGES_UNSUPPORTED = ("permissions",) @@ -85,7 +82,7 @@ class ModLog: else: message = f"{channel.name} (`{channel.id}`)" - await self.send_log_message(Icons.hash_green, COLOUR_GREEN, title, message) + await self.send_log_message(Icons.hash_green, Colour(Colours.soft_green), title, message) async def on_guild_channel_delete(self, channel: GUILD_CHANNEL): if channel.guild.id != GuildConstant.id: @@ -104,7 +101,7 @@ class ModLog: message = f"{channel.name} (`{channel.id}`)" await self.send_log_message( - Icons.hash_red, COLOUR_RED, + Icons.hash_red, Colour(Colours.soft_red), title, message ) @@ -150,7 +147,7 @@ class ModLog: message = "" for item in sorted(changes): - message += f"{BULLET_POINT} {item}\n" + message += f"{Emojis.bullet} {item}\n" if after.category: message = f"**{after.category}/#{after.name} (`{after.id}`)**\n{message}" @@ -167,7 +164,7 @@ class ModLog: return await self.send_log_message( - Icons.crown_green, COLOUR_GREEN, + Icons.crown_green, Colour(Colours.soft_green), "Role created", f"`{role.id}`" ) @@ -176,7 +173,7 @@ class ModLog: return await self.send_log_message( - Icons.crown_red, COLOUR_RED, + Icons.crown_red, Colour(Colours.soft_red), "Role removed", f"{role.name} (`{role.id}`)" ) @@ -222,7 +219,7 @@ class ModLog: message = "" for item in sorted(changes): - message += f"{BULLET_POINT} {item}\n" + message += f"{Emojis.bullet} {item}\n" message = f"**{after.name} (`{after.id}`)**\n{message}" @@ -270,7 +267,7 @@ class ModLog: message = "" for item in sorted(changes): - message += f"{BULLET_POINT} {item}\n" + message += f"{Emojis.bullet} {item}\n" message = f"**{after.name} (`{after.id}`)**\n{message}" @@ -285,7 +282,7 @@ class ModLog: return await self.send_log_message( - Icons.user_ban, COLOUR_RED, + Icons.user_ban, Colour(Colours.soft_red), "User banned", f"{member.name}#{member.discriminator} (`{member.id}`)", thumbnail=member.avatar_url_as(static_format="png") ) @@ -325,7 +322,7 @@ class ModLog: message = f"{Emojis.new} {message}" await self.send_log_message( - Icons.sign_in, COLOUR_GREEN, + Icons.sign_in, Colour(Colours.soft_green), "User joined", message, thumbnail=member.avatar_url_as(static_format="png") ) @@ -335,7 +332,7 @@ class ModLog: return await self.send_log_message( - Icons.sign_out, COLOUR_RED, + Icons.sign_out, Colour(Colours.soft_red), "User left", f"{member.name}#{member.discriminator} (`{member.id}`)", thumbnail=member.avatar_url_as(static_format="png") ) @@ -404,7 +401,7 @@ class ModLog: message = "" for item in sorted(changes): - message += f"{BULLET_POINT} {item}\n" + message += f"{Emojis.bullet} {item}\n" message = f"**{after.name}#{after.discriminator} (`{after.id}`)**\n{message}" @@ -482,7 +479,7 @@ class ModLog: ) await self.send_log_message( - Icons.message_delete, COLOUR_RED, + Icons.message_delete, Colour(Colours.soft_red), "Message deleted", response, channel_id=Channels.message_log diff --git a/bot/constants.py b/bot/constants.py index b49cb48f6..532a2af1d 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -198,20 +198,29 @@ class Cooldowns(metaclass=YAMLGetter): tags: int +class Colours(metaclass=YAMLGetter): + section = "style" + subsection = "colours" + + soft_red: int + soft_green: int + + class Emojis(metaclass=YAMLGetter): - section = "bot" + section = "style" subsection = "emojis" green_chevron: str red_chevron: str white_chevron: str + bullet: str new: str pencil: str class Icons(metaclass=YAMLGetter): - section = "bot" + section = "style" subsection = "icons" crown_blurple: str diff --git a/config-default.yml b/config-default.yml index 118ba165d..d55a50d57 100644 --- a/config-default.yml +++ b/config-default.yml @@ -6,12 +6,23 @@ bot: # Per channel, per tag. tags: 60 + clean: + # Maximum number of messages to traverse for clean commands + message_limit: 10000 + + +style: + colours: + soft_red: 0xcd6d6d + soft_green: 0x68c290 + emojis: green_chevron: "<:greenchevron:418104310329769993>" red_chevron: "<:redchevron:418112778184818698>" white_chevron: "<:whitechevron:418110396973711363>" lemoneye2: "<:lemoneye2:435193765582340098>" + bullet: "\u2022" pencil: "\u270F" new: "\U0001F195" @@ -37,10 +48,6 @@ bot: user_unban: "https://cdn.discordapp.com/emojis/469952898692808704.png" user_update: "https://cdn.discordapp.com/emojis/469952898684551168.png" - clean: - # Maximum number of messages to traverse for clean commands - message_limit: 10000 - guild: id: 267624335836053506 -- cgit v1.2.3 From fa39f51e8abaead97f86ee3b3624da23d6f01f20 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 22 Jul 2018 21:12:48 +0200 Subject: Sending the users top_role --- bot/cogs/clean.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index fe9b14003..8df6d6e83 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -102,12 +102,14 @@ class Clean: if delete and message.content or message.embeds: content = message.content or message.embeds[0].description author = f"{message.author.name}#{message.author.discriminator}" + role = message.author.top_role.name # Store the message data message_ids.append(message.id) message_log.append({ "content": content, "author": author, + "role": role.lower(), "timestamp": message.created_at.strftime("%D %H:%M") }) -- cgit v1.2.3 From ea33140c3eab95ae8f42cd36afda7c83972b41f7 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 29 Jul 2018 01:17:29 +0200 Subject: Added regex cleanups, and now sending over complete embed details and a number of other message details. --- bot/cogs/clean.py | 126 +++++++++++++++++++++++++++++++++++++++++------------ bot/cogs/modlog.py | 2 +- 2 files changed, 99 insertions(+), 29 deletions(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index 8df6d6e83..efedc2dce 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -1,6 +1,9 @@ import logging import random +import re +from typing import Optional +from aiohttp.client_exceptions import ClientResponseError from discord import Colour, Embed, Message, User from discord.ext.commands import Bot, Context, group @@ -39,29 +42,76 @@ class Clean: json={"log_data": log_data} ) - data = await response.json() - log_id = data["log_id"] + try: + data = await response.json() + log_id = data["log_id"] + except (KeyError, ClientResponseError): + log.debug( + "API returned an unexpected result:\n" + f"{response.text}" + ) + return return f"{URLs.site_clean_logs}/{log_id}" async def _clean_messages( self, amount: int, ctx: Context, - bots_only: bool=False, user: User=None + bots_only: bool = False, user: User = None, + regex: Optional[str] = None ): """ A helper function that does the actual message cleaning. :param bots_only: Set this to True if you only want to delete bot messages. :param user: Specify a user and it will only delete messages by this user. + :param regular_expression: Specify a regular expression and it will only + delete messages that match this. """ - # Bulk delete checks - def predicate_bots_only(message: Message): + def predicate_bots_only(message: Message) -> bool: + """ + Returns true if the message was sent by a bot + """ + return message.author.bot - def predicate_specific_user(message: Message): + def predicate_specific_user(message: Message) -> bool: + """ + Return True if the message was sent by the + user provided in the _clean_messages call. + """ + return message.author == user + def predicate_regex(message: Message): + """ + Returns True if the regex provided in the + _clean_messages matches the message content + or any embed attributes the message may have. + """ + + content = [message.content] + + # Add the content for all embed attributes + for embed in message.embeds: + content.append(embed.title) + content.append(embed.description) + content.append(embed.footer.text) + content.append(embed.author.name) + for field in embed.fields: + content.append(field.name) + content.append(field.value) + + # Get rid of empty attributes and turn it into a string + content = [attr for attr in content if attr] + content = "\n".join(content) + + # Now let's see if there's a regex match + if not content: + return False + else: + return bool(re.search(regex.lower(), content.lower())) + # Is this an acceptable amount of messages to clean? if amount > CleanMessages.message_limit: embed = Embed( @@ -82,35 +132,52 @@ class Clean: await ctx.send(embed=embed) return + # Set up the correct predicate + if bots_only: + predicate = predicate_bots_only # Delete messages from bots + elif user: + predicate = predicate_specific_user # Delete messages from specific user + elif regex: + predicate = predicate_regex # Delete messages that match regex + else: + predicate = None # Delete all messages + # Look through the history and retrieve message data message_log = [] message_ids = [] - self.cleaning = True + invocation_deleted = False async for message in ctx.channel.history(limit=amount): + # If at any point the cancel command is invoked, we should stop. if not self.cleaning: return - delete = ( - bots_only and message.author.bot # Delete bot messages - or user and message.author == user # Delete user messages - or not bots_only and not user # Delete all messages - ) + # Always start by deleting the invocation + if not invocation_deleted: + await message.delete() + invocation_deleted = True + continue - if delete and message.content or message.embeds: - content = message.content or message.embeds[0].description + # If the message passes predicate, let's save it. + if predicate is None or predicate(message): author = f"{message.author.name}#{message.author.discriminator}" role = message.author.top_role.name - # Store the message data + content = message.content + embeds = [embed.to_dict() for embed in message.embeds] + attachments = ["" for _ in message.attachments] + message_ids.append(message.id) message_log.append({ "content": content, "author": author, + "user_id": str(message.author.id), "role": role.lower(), - "timestamp": message.created_at.strftime("%D %H:%M") + "timestamp": message.created_at.strftime("%D %H:%M"), + "attachments": attachments, + "embeds": embeds, }) self.cleaning = False @@ -119,13 +186,6 @@ class Clean: self.mod_log.ignore_message_deletion(*message_ids) # Use bulk delete to actually do the cleaning. It's far faster. - predicate = None - - if bots_only: - predicate = predicate_bots_only - elif user: - predicate = predicate_specific_user - await ctx.channel.purge( limit=amount, check=predicate @@ -141,7 +201,7 @@ class Clean: color=Colour(Colours.soft_red), description="No matching messages could be found." ) - await ctx.send(embed=embed) + await ctx.send(embed=embed, delete_after=10.0) return # Build the embed and send it @@ -171,7 +231,7 @@ class Clean: await ctx.invoke(self.bot.get_command("help"), "clean") - @clean_group.command(aliases=["user"]) + @clean_group.command(name="user", aliases=["users"]) @with_role(Roles.moderator, Roles.admin, Roles.owner) async def clean_user(self, ctx: Context, user: User, amount: int = 10): """ @@ -181,7 +241,7 @@ class Clean: await self._clean_messages(amount, ctx, user=user) - @clean_group.command(aliases=["all"]) + @clean_group.command(name="all", aliases=["everything"]) @with_role(Roles.moderator, Roles.admin, Roles.owner) async def clean_all(self, ctx: Context, amount: int = 10): """ @@ -191,7 +251,7 @@ class Clean: await self._clean_messages(amount, ctx) - @clean_group.command(aliases=["bots"]) + @clean_group.command(name="bots", aliases=["bot"]) @with_role(Roles.moderator, Roles.admin, Roles.owner) async def clean_bots(self, ctx: Context, amount: int = 10): """ @@ -201,7 +261,17 @@ class Clean: await self._clean_messages(amount, ctx, bots_only=True) - @clean_group.command(aliases=["stop", "cancel", "abort"]) + @clean_group.command(name="regex", aliases=["word", "expression"]) + @with_role(Roles.moderator, Roles.admin, Roles.owner) + async def clean_regex(self, ctx: Context, regex, amount: int = 10): + """ + Delete all messages that match a certain regex, + and stop cleaning after traversing `amount` messages. + """ + + await self._clean_messages(amount, ctx, regex=regex) + + @clean_group.command(name="stop", aliases=["cancel", "abort"]) @with_role(Roles.moderator, Roles.admin, Roles.owner) async def clean_cancel(self, ctx: Context): """ diff --git a/bot/cogs/modlog.py b/bot/cogs/modlog.py index e8ff19bb5..b5a73d6e0 100644 --- a/bot/cogs/modlog.py +++ b/bot/cogs/modlog.py @@ -486,7 +486,7 @@ class ModLog: response = f"**Attachments:** {len(message.attachments)}\n" + response await self.send_log_message( - Icons.message_delete, COLOUR_RED, + Icons.message_delete, Colours.soft_red, "Message deleted", response, channel_id=Channels.message_log -- cgit v1.2.3 From d4d6be7104e6eab72c1c0d73ec7e8770bc7cdeb1 Mon Sep 17 00:00:00 2001 From: Christopher Baklid Date: Sun, 29 Jul 2018 09:40:50 +0000 Subject: Pip cache --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 88ab5d927..3edfb2bf8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,13 @@ image: pythondiscord/bot-ci:latest +variables: + PIPENV_CACHE_DIR: "$CI_PROJECT_DIR/pipenv-cache" + +cache: + paths: + - "$CI_PROJECT_DIR/pipenv-cache" + - "$CI_PROJECT_DIR/.venv" + stages: - test - build -- cgit v1.2.3 From 89db231108d309c9ad2c2ccf476e662d807c7c0e Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 29 Jul 2018 11:41:10 +0200 Subject: Addressing all gdude comments --- bot/cogs/clean.py | 36 +++++++++++++++++++++++------------- bot/cogs/defcon.py | 16 ++++++++-------- bot/cogs/token_remover.py | 6 +++--- bot/cogs/verification.py | 4 ++-- 4 files changed, 36 insertions(+), 26 deletions(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index efedc2dce..5644913b5 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -18,6 +18,19 @@ log = logging.getLogger(__name__) class Clean: + """ + A cog that allows messages to be deleted in + bulk, while applying various filters. + + You can delete messages sent by a specific user, + messages sent by bots, all messages, or messages + that match a specific regular expression. + + The deleted messages are saved and uploaded + to the database via an API endpoint, and a URL is + returned which can be used to view the messages + in the Discord dark theme style. + """ def __init__(self, bot: Bot): self.bot = bot @@ -127,7 +140,7 @@ class Clean: embed = Embed( color=Colour(Colours.soft_red), title=random.choice(NEGATIVE_REPLIES), - description="Multiple simultaneous cleaning processes is not allowed." + description="Please wait for the currently ongoing clean operation to complete." ) await ctx.send(embed=embed) return @@ -163,7 +176,7 @@ class Clean: # If the message passes predicate, let's save it. if predicate is None or predicate(message): author = f"{message.author.name}#{message.author.discriminator}" - role = message.author.top_role.name + role_id = message.author.top_role.id content = message.content embeds = [embed.to_dict() for embed in message.embeds] @@ -174,7 +187,7 @@ class Clean: "content": content, "author": author, "user_id": str(message.author.id), - "role": role.lower(), + "role_id": str(role_id), "timestamp": message.created_at.strftime("%D %H:%M"), "attachments": attachments, "embeds": embeds, @@ -205,23 +218,20 @@ class Clean: return # Build the embed and send it + print(upload_log) message = ( f"**{len(message_ids)}** messages deleted in <#{ctx.channel.id}> by **{ctx.author.name}**\n\n" f"A log of the deleted messages can be found [here]({upload_log})." ) - embed = Embed( - color=Colour(Colours.soft_red), - description=message - ) - - embed.set_author( - name=f"Bulk message delete", - icon_url=Icons.message_bulk_delete + await self.mod_log.send_log_message( + icon_url=Icons.message_bulk_delete, + colour=Colour(Colours.soft_red), + title="Bulk message delete", + text=message, + channel_id=Channels.modlog, ) - await self.bot.get_channel(Channels.modlog).send(embed=embed) - @group(invoke_without_command=True, name="clean", hidden=True) @with_role(Roles.moderator, Roles.admin, Roles.owner) async def clean_group(self, ctx: Context): diff --git a/bot/cogs/defcon.py b/bot/cogs/defcon.py index 8ca59b058..beb05ba46 100644 --- a/bot/cogs/defcon.py +++ b/bot/cogs/defcon.py @@ -36,7 +36,7 @@ class Defcon: self.headers = {"X-API-KEY": Keys.site_api} @property - def modlog(self) -> ModLog: + def mod_log(self) -> ModLog: return self.bot.get_cog("ModLog") async def on_ready(self): @@ -92,7 +92,7 @@ class Defcon: if not message_sent: message = f"{message}\n\nUnable to send rejection message via DM; they probably have DMs disabled." - await self.modlog.send_log_message( + await self.mod_log.send_log_message( Icons.defcon_denied, COLOUR_RED, "Entry denied", message, member.avatar_url_as(static_format="png") ) @@ -133,7 +133,7 @@ class Defcon: f"```py\n{e}\n```" ) - await self.modlog.send_log_message( + await self.mod_log.send_log_message( Icons.defcon_enabled, COLOUR_GREEN, "DEFCON enabled", f"**Staffer:** {ctx.author.name}#{ctx.author.discriminator} (`{ctx.author.id}`)\n" f"**Days:** {self.days.days}\n\n" @@ -144,7 +144,7 @@ class Defcon: else: await ctx.send(f"{Emojis.defcon_enabled} DEFCON enabled.") - await self.modlog.send_log_message( + await self.mod_log.send_log_message( Icons.defcon_enabled, COLOUR_GREEN, "DEFCON enabled", f"**Staffer:** {ctx.author.name}#{ctx.author.discriminator} (`{ctx.author.id}`)\n" f"**Days:** {self.days.days}\n\n" @@ -176,7 +176,7 @@ class Defcon: f"```py\n{e}\n```" ) - await self.modlog.send_log_message( + await self.mod_log.send_log_message( Icons.defcon_disabled, COLOUR_RED, "DEFCON disabled", f"**Staffer:** {ctx.author.name}#{ctx.author.discriminator} (`{ctx.author.id}`)\n" "**There was a problem updating the site** - This setting may be reverted when the bot is " @@ -186,7 +186,7 @@ class Defcon: else: await ctx.send(f"{Emojis.defcon_disabled} DEFCON disabled.") - await self.modlog.send_log_message( + await self.mod_log.send_log_message( Icons.defcon_disabled, COLOUR_RED, "DEFCON disabled", f"**Staffer:** {ctx.author.name}#{ctx.author.discriminator} (`{ctx.author.id}`)" ) @@ -233,7 +233,7 @@ class Defcon: f"```py\n{e}\n```" ) - await self.modlog.send_log_message( + await self.mod_log.send_log_message( Icons.defcon_updated, Colour.blurple(), "DEFCON updated", f"**Staffer:** {ctx.author.name}#{ctx.author.discriminator} (`{ctx.author.id}`)\n" f"**Days:** {self.days.days}\n\n" @@ -246,7 +246,7 @@ class Defcon: f"{Emojis.defcon_updated} DEFCON days updated; accounts must be {days} days old to join to the server" ) - await self.modlog.send_log_message( + await self.mod_log.send_log_message( Icons.defcon_updated, Colour.blurple(), "DEFCON updated", f"**Staffer:** {ctx.author.name}#{ctx.author.discriminator} (`{ctx.author.id}`)\n" f"**Days:** {self.days.days}" diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index c8621118b..74bc0d9b2 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -40,10 +40,10 @@ class TokenRemover: def __init__(self, bot: Bot): self.bot = bot - self.modlog = None + self.mod_log = None async def on_ready(self): - self.modlog = self.bot.get_channel(Channels.modlog) + self.mod_log = self.bot.get_channel(Channels.modlog) async def on_message(self, msg: Message): if msg.author.bot: @@ -61,7 +61,7 @@ class TokenRemover: if self.is_valid_user_id(user_id) and self.is_valid_timestamp(creation_timestamp): await msg.delete() await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention)) - await self.modlog.send( + await self.mod_log.send( ":key2::mute: censored a seemingly valid token sent by " f"{msg.author} (`{msg.author.id}`) in {msg.channel.mention}, token was " f"`{user_id}.{creation_timestamp}.{'x' * len(hmac)}`" diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index b0667fdd0..7f1c9e68a 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -37,7 +37,7 @@ class Verification: self.bot = bot @property - def modlog(self) -> ModLog: + def mod_log(self) -> ModLog: return self.bot.get_cog("ModLog") async def on_message(self, message: Message): @@ -90,7 +90,7 @@ class Verification: log.trace(f"Deleting the message posted by {ctx.author}.") try: - self.modlog.ignore_message_deletion(ctx.message.id) + self.mod_log.ignore_message_deletion(ctx.message.id) await ctx.message.delete() except NotFound: log.trace("No message found, it must have been deleted by another bot.") -- cgit v1.2.3