From 05616ae19ed685c261f13d99d6f6150e863795b7 Mon Sep 17 00:00:00 2001 From: mathsman5133 Date: Sat, 12 Oct 2019 12:36:04 +1100 Subject: Refactor the `Help` command. - `redirect_output` has been adjusted to run the `delete_invocation` inside a task as the help command will wait for that to run before sending the help or doing anything else. - `pagination` has been adjusted to support deleting the paginated message if `cleanup` is True, and an optional `description` that is present through all pages of pagination. - The help command has been refactored to subclass `commands.HelpCommand`. This means that it now supports methods such as `ctx.send_help(ctx.command)`. - `help_cleanup` provides the opportunity to use the :x: reaction to cleanup help even with no pagination. - Pagination purely happens through the `LinePaginator`, forcing a pagination session with 1 line per page where we format the page style before sending it through. - Categories are properly dealt with by finding a match and sending a seperate help where a named tuple of the Category name, description and relevant cogs is the only parameter. - Choices for when a command was not found has been updated to include category names, cog names, aliases of group and command names, and include all subcommands and aliases. This should provide a more helpful output when an error message is sent - Sending command, group, cog, category and bot help has been split into different functions that are called from `command_callback`. This provides an easier way to alter future changes, and cleans up code considerably. - Important note: no outward facing formatting should have changed. Any desired changes can be discussed in review. --- bot/cogs/help.py | 708 +++++++++++++++++++----------------------------------- bot/decorators.py | 29 ++- bot/pagination.py | 25 +- 3 files changed, 287 insertions(+), 475 deletions(-) diff --git a/bot/cogs/help.py b/bot/cogs/help.py index 9607dbd8d..16fd62154 100644 --- a/bot/cogs/help.py +++ b/bot/cogs/help.py @@ -1,32 +1,44 @@ -import asyncio import itertools +import logging +from asyncio import TimeoutError from collections import namedtuple from contextlib import suppress -from typing import Union -from discord import Colour, Embed, HTTPException, Message, Reaction, User -from discord.ext import commands -from discord.ext.commands import Bot, CheckFailure, Cog as DiscordCog, Command, Context +from discord import Colour, Embed, HTTPException, Member, Message, Reaction, User +from discord.ext.commands import Bot, Cog, Command, Context, Group, HelpCommand from fuzzywuzzy import fuzz, process from bot import constants from bot.constants import Channels, STAFF_ROLES from bot.decorators import redirect_output -from bot.pagination import ( - DELETE_EMOJI, FIRST_EMOJI, LAST_EMOJI, - LEFT_EMOJI, LinePaginator, RIGHT_EMOJI, -) +from bot.pagination import DELETE_EMOJI, LinePaginator +log = logging.getLogger(__name__) -REACTIONS = { - FIRST_EMOJI: 'first', - LEFT_EMOJI: 'back', - RIGHT_EMOJI: 'next', - LAST_EMOJI: 'end', - DELETE_EMOJI: 'stop' -} +COMMANDS_PER_PAGE = 5 +PREFIX = constants.Bot.prefix -Cog = namedtuple('Cog', ['name', 'description', 'commands']) +Category = namedtuple("Category", ["name", "description", "cogs"]) + + +async def help_cleanup(bot: Bot, author: Member, message: Message) -> None: + """ + Runs the cleanup for the help command. + + Adds a :x: reaction that, when clicked, will delete the help message. + After a 300 second timeout, the reaction will be removed. + """ + def check(r: Reaction, u: User) -> bool: + """Checks the reaction is :x:, the author is original author and messages are the same.""" + return str(r) == DELETE_EMOJI and u.id == author.id and r.message.id == message.id + + await message.add_reaction(DELETE_EMOJI) + with suppress(HTTPException, TimeoutError): + _, _ = await bot.wait_for("reaction_add", check=check, timeout=300) + await message.delete() + return + + await message.remove_reaction(DELETE_EMOJI, bot.user) class HelpQueryNotFound(ValueError): @@ -44,22 +56,9 @@ class HelpQueryNotFound(ValueError): self.possible_matches = possible_matches -class HelpSession: +class CustomHelpCommand(HelpCommand): """ - An interactive session for bot and command help output. - - Expected attributes include: - * title: str - The title of the help message. - * query: Union[discord.ext.commands.Bot, discord.ext.commands.Command] - * description: str - The description of the query. - * pages: list[str] - A list of the help content split into manageable pages. - * message: `discord.Message` - The message object that's showing the help contents. - * destination: `discord.abc.Messageable` - Where the help message is to be sent to. + An interactive instance for the bot help command. Cogs can be grouped into custom categories. All cogs with the same category will be displayed under a single category name in the help output. Custom categories are defined inside the cogs @@ -68,499 +67,296 @@ class HelpSession: the regular description (class docstring) of the first cog found in the category. """ - def __init__( - self, - ctx: Context, - *command, - cleanup: bool = False, - only_can_run: bool = True, - show_hidden: bool = False, - max_lines: int = 15 - ): - """Creates an instance of the HelpSession class.""" - self._ctx = ctx - self._bot = ctx.bot - self.title = "Command Help" - - # set the query details for the session - if command: - query_str = ' '.join(command) - self.query = self._get_query(query_str) - self.description = self.query.description or self.query.help - else: - self.query = ctx.bot - self.description = self.query.description - self.author = ctx.author - self.destination = ctx.channel - - # set the config for the session - self._cleanup = cleanup - self._only_can_run = only_can_run - self._show_hidden = show_hidden - self._max_lines = max_lines - - # init session states - self._pages = None - self._current_page = 0 - self.message = None - self._timeout_task = None - self.reset_timeout() - - def _get_query(self, query: str) -> Union[Command, Cog]: + def __init__(self): + super().__init__(command_attrs={"help": "Shows help for bot commands"}) + + @redirect_output(destination_channel=Channels.bot, bypass_roles=STAFF_ROLES) + async def prepare_help_command(self, ctx: Context, command: str = None) -> None: + """Adjust context to redirect to a new channel if required.""" + self.context = ctx + + async def command_callback(self, ctx: Context, *, command: str = None) -> None: """Attempts to match the provided query with a valid command or cog.""" - command = self._bot.get_command(query) - if command: - return command + # the only reason we need to tamper with this is because d.py does not support "categories", + # so we need to deal with them ourselves. + + # handle any command redirection and adjust context channel accordingly. + await self.prepare_help_command(ctx, command=command) + bot = ctx.bot + + if command is None: + # quick and easy, send bot help if command is none + mapping = self.get_bot_mapping() + await self.send_bot_help(mapping) + return - # Find all cog categories that match. cog_matches = [] description = None - for cog in self._bot.cogs.values(): - if hasattr(cog, "category") and cog.category == query: + for cog in bot.cogs.values(): + if hasattr(cog, "category") and cog.category == command: cog_matches.append(cog) if hasattr(cog, "category_description"): description = cog.category_description - # Try to search by cog name if no categories match. - if not cog_matches: - cog = self._bot.cogs.get(query) + if cog_matches: + category = Category(name=command, description=description, cogs=cog_matches) + await self.send_category_help(category) + return - # Don't consider it a match if the cog has a category. - if cog and not hasattr(cog, "category"): - cog_matches = [cog] + # it's either a cog, group, command or subcommand, let super deal with it + await super().command_callback(ctx, command=command) - if cog_matches: - cog = cog_matches[0] - cmds = (cog.get_commands() for cog in cog_matches) # Commands of all cogs + def get_all_help_choices(self) -> set: + """ + Get all the possible options for getting help in the bot. - return Cog( - name=cog.category if hasattr(cog, "category") else cog.qualified_name, - description=description or cog.description, - commands=tuple(itertools.chain.from_iterable(cmds)) # Flatten the list - ) + These include: + - Category names + - Cog names + - Group command names (and aliases) + - Command names (and aliases) + - Subcommand names (with parent group and aliases for subcommand, but not including aliases for group) + + Options and choices are case sensitive. + """ + # first get all commands including subcommands and full command name aliases + choices = set() + for c in self.context.bot.walk_commands(): + # the the command or group name + choices.add(str(c)) + + # all aliases if it's just a command + choices.update(n for n in c.aliases if isinstance(c, Command)) + + # else aliases with parent if group. we need to strip() in case it's a Command and `full_parent` is None, + # otherwise we get 2 commands: ` help` and normal `help`. + # We could do case-by-case with f-string but this is the cleanest solution + choices.update(f"{c.full_parent_name} {a}".strip() for a in c.aliases) - self._handle_not_found(query) + # all cog names + choices.update(self.context.bot.cogs) - def _handle_not_found(self, query: str) -> None: + # all category names + choices.update(getattr(n, "category", None) for n in self.context.bot.cogs if hasattr(n, "category")) + return choices + + def command_not_found(self, string: str) -> "HelpQueryNotFound": """ - Handles when a query does not match a valid command or cog. + Handles when a query does not match a valid command, group, cog or category. - Will pass on possible close matches along with the `HelpQueryNotFound` exception. + Will return an instance of the `HelpQueryNotFound` exception with the error message and possible matches. """ - # Combine command and cog names - choices = list(self._bot.all_commands) + list(self._bot.cogs) + choices = self.get_all_help_choices() + result = process.extractBests(string, choices, scorer=fuzz.ratio, score_cutoff=90) - result = process.extractBests(query, choices, scorer=fuzz.ratio, score_cutoff=90) + return HelpQueryNotFound(f'Query "{string}" not found.', dict(result)) - raise HelpQueryNotFound(f'Query "{query}" not found.', dict(result)) + def subcommand_not_found(self, command: Command, string: str) -> "HelpQueryNotFound": + """ + Redirects the error to `command_not_found`. - async def timeout(self, seconds: int = 30) -> None: - """Waits for a set number of seconds, then stops the help session.""" - await asyncio.sleep(seconds) - await self.stop() + `command_not_found` deals with searching and getting best choices for both commands and subcommands. + """ + return self.command_not_found(f"{command.qualified_name} {string}") - def reset_timeout(self) -> None: - """Cancels the original timeout task and sets it again from the start.""" - # cancel original if it exists - if self._timeout_task: - if not self._timeout_task.cancelled(): - self._timeout_task.cancel() + async def send_error_message(self, error: HelpQueryNotFound) -> None: + """Send the error message to the channel.""" + embed = Embed(colour=Colour.red(), title=str(error)) - # recreate the timeout task - self._timeout_task = self._bot.loop.create_task(self.timeout()) + if getattr(error, "possible_matches", None): + matches = "\n".join(f"`{n}`" for n in error.possible_matches.keys()) + embed.add_field(name="Did you mean:", value=matches) - async def on_reaction_add(self, reaction: Reaction, user: User) -> None: - """Event handler for when reactions are added on the help message.""" - # ensure it was the relevant session message - if reaction.message.id != self.message.id: - return + await self.context.send(embed=embed) - # ensure it was the session author who reacted - if user.id != self.author.id: - return + async def command_formatting(self, command: Command) -> Embed: + """ + Takes a command and turns it into an embed. - emoji = str(reaction.emoji) + It will add an author, command signature + help, aliases and a note if the user can't run the command. + """ + embed = Embed() + embed.set_author(name="Command Help", icon_url=constants.Icons.questionmark) - # check if valid action - if emoji not in REACTIONS: - return + parent = command.full_parent_name - self.reset_timeout() + name = str(command) if not parent else f"{parent} {command.name}" + fmt = f"**```{PREFIX}{name} {command.signature}```**\n" - # Run relevant action method - action = getattr(self, f'do_{REACTIONS[emoji]}', None) - if action: - await action() + # show command aliases + aliases = ", ".join(f"`{a}`" if not parent else f"`{parent} {a}`" for a in command.aliases) + if aliases: + fmt += f"**Can also use:** {aliases}\n\n" - # remove the added reaction to prep for re-use - with suppress(HTTPException): - await self.message.remove_reaction(reaction, user) + # check if the user is allowed to run this command + if not await command.can_run(self.context): + fmt += "**You cannot run this command.**" - async def on_message_delete(self, message: Message) -> None: - """Closes the help session when the help message is deleted.""" - if message.id == self.message.id: - await self.stop() + fmt += f"*{command.help or 'No details provided.'}*\n" + embed.description = fmt - async def prepare(self) -> None: - """Sets up the help session pages, events, message and reactions.""" - # create paginated content - await self.build_pages() + return embed - # setup listeners - self._bot.add_listener(self.on_reaction_add) - self._bot.add_listener(self.on_message_delete) + async def send_command_help(self, command: Command) -> None: + """Send help for a single command.""" + embed = await self.command_formatting(command) + message = await self.context.send(embed=embed) + await help_cleanup(self.context.bot, self.context.author, message) - # Send the help message - await self.update_page() - self.add_reactions() + async def send_group_help(self, group: Group) -> None: + """Sends help for a group command.""" + subcommands = group.commands - def add_reactions(self) -> None: - """Adds the relevant reactions to the help message based on if pagination is required.""" - # if paginating - if len(self._pages) > 1: - for reaction in REACTIONS: - self._bot.loop.create_task(self.message.add_reaction(reaction)) + if len(subcommands) == 0: + # no subcommands, just treat it like a regular command + await self.send_command_help(group) + return - # if single-page - else: - self._bot.loop.create_task(self.message.add_reaction(DELETE_EMOJI)) + # remove commands that the user can't run and are hidden, and sort by name + _commands = await self.filter_commands(subcommands, sort=True) - def _category_key(self, cmd: Command) -> str: + embed = await self.command_formatting(group) + + # add in subcommands with brief help + # note: the extra f-string around the signature is necessary because otherwise an extra space before the + # last back tick is present. + fmt = "\n".join( + f"**`{PREFIX}{c.qualified_name}{f' {c.signature}' if c.signature else ''}`**" + f"\n*{c.short_doc or 'No details provided.'}*" for c in _commands + ) + embed.description += f"\n**Subcommands:**\n{fmt}" + message = await self.context.send(embed=embed) + await help_cleanup(self.context.bot, self.context.author, message) + + async def send_cog_help(self, cog: Cog) -> None: + """Send help for a cog.""" + # sort commands by name, and remove any the user cant run or are hidden. + _commands = await self.filter_commands(cog.get_commands(), sort=True) + + embed = Embed() + embed.set_author(name="Command Help", icon_url=constants.Icons.questionmark) + embed.description = f"**{cog.qualified_name}**\n*{cog.description}*\n\n**Commands:**\n" + + lines = [ + f"`{PREFIX}{c.qualified_name}{f' {c.signature}' if c.signature else ''}`" + f"\n*{c.short_doc or 'No details provided.'}*\n" for c in _commands + ] + embed.description += "\n".join(n for n in lines) + + message = await self.context.send(embed=embed) + await help_cleanup(self.context.bot, self.context.author, message) + + @staticmethod + def _category_key(cmd: Command) -> str: """ Returns a cog name of a given command for use as a key for `sorted` and `groupby`. A zero width space is used as a prefix for results with no cogs to force them last in ordering. """ if cmd.cog: - try: + with suppress(AttributeError): if cmd.cog.category: - return f'**{cmd.cog.category}**' - except AttributeError: - pass - - return f'**{cmd.cog_name}**' + return f"**{cmd.cog.category}**" + return f"**{cmd.cog_name}**" else: return "**\u200bNo Category:**" - def _get_command_params(self, cmd: Command) -> str: + async def send_category_help(self, category: Category) -> None: """ - Returns the command usage signature. + Sends help for a bot category. - This is a custom implementation of `command.signature` in order to format the command - signature without aliases. + This sends a brief help for all commands in all cogs registered to the category. """ - results = [] - for name, param in cmd.clean_params.items(): - - # if argument has a default value - if param.default is not param.empty: - - if isinstance(param.default, str): - show_default = param.default - else: - show_default = param.default is not None - - # if default is not an empty string or None - if show_default: - results.append(f'[{name}={param.default}]') - else: - results.append(f'[{name}]') - - # if variable length argument - elif param.kind == param.VAR_POSITIONAL: - results.append(f'[{name}...]') - - # if required - else: - results.append(f'<{name}>') - - return f"{cmd.name} {' '.join(results)}" - - async def build_pages(self) -> None: - """Builds the list of content pages to be paginated through in the help message, as a list of str.""" - # Use LinePaginator to restrict embed line height - paginator = LinePaginator(prefix='', suffix='', max_lines=self._max_lines) - - prefix = constants.Bot.prefix + embed = Embed() + embed.set_author(name="Command Help", icon_url=constants.Icons.questionmark) - # show signature if query is a command - if isinstance(self.query, commands.Command): - signature = self._get_command_params(self.query) - parent = self.query.full_parent_name + ' ' if self.query.parent else '' - paginator.add_line(f'**```{prefix}{parent}{signature}```**') - - # show command aliases - aliases = ', '.join(f'`{a}`' for a in self.query.aliases) - if aliases: - paginator.add_line(f'**Can also use:** {aliases}\n') - - if not await self.query.can_run(self._ctx): - paginator.add_line('***You cannot run this command.***\n') - - # show name if query is a cog - if isinstance(self.query, Cog): - paginator.add_line(f'**{self.query.name}**') - - if self.description: - paginator.add_line(f'*{self.description}*') + all_commands = [] + for c in category.cogs: + all_commands.extend(c.get_commands()) - # list all children commands of the queried object - if isinstance(self.query, (commands.GroupMixin, Cog)): - - # remove hidden commands if session is not wanting hiddens - if not self._show_hidden: - filtered = [c for c in self.query.commands if not c.hidden] - else: - filtered = self.query.commands - - # if after filter there are no commands, finish up - if not filtered: - self._pages = paginator.pages - return - - # set category to Commands if cog - if isinstance(self.query, Cog): - grouped = (('**Commands:**', self.query.commands),) - - # set category to Subcommands if command - elif isinstance(self.query, commands.Command): - grouped = (('**Subcommands:**', self.query.commands),) - - # don't show prefix for subcommands - prefix = '' - - # otherwise sort and organise all commands into categories - else: - cat_sort = sorted(filtered, key=self._category_key) - grouped = itertools.groupby(cat_sort, key=self._category_key) - - # process each category - for category, cmds in grouped: - cmds = sorted(cmds, key=lambda c: c.name) - - # if there are no commands, skip category - if len(cmds) == 0: - continue - - cat_cmds = [] - - # format details for each child command - for command in cmds: - - # skip if hidden and hide if session is set to - if command.hidden and not self._show_hidden: - continue - - # see if the user can run the command - strikeout = '' - - # Patch to make the !help command work outside of #bot-commands again - # This probably needs a proper rewrite, but this will make it work in - # the mean time. - try: - can_run = await command.can_run(self._ctx) - except CheckFailure: - can_run = False - - if not can_run: - # skip if we don't show commands they can't run - if self._only_can_run: - continue - strikeout = '~~' + filtered_commands = await self.filter_commands(all_commands, sort=True, key=self._category_key) - signature = self._get_command_params(command) - info = f"{strikeout}**`{prefix}{signature}`**{strikeout}" - - # handle if the command has no docstring - if command.short_doc: - cat_cmds.append(f'{info}\n*{command.short_doc}*') - else: - cat_cmds.append(f'{info}\n*No details provided.*') - - # state var for if the category should be added next - print_cat = 1 - new_page = True + lines = [ + f"`{PREFIX}{c.qualified_name}{f' {c.signature}' if c.signature else ''}`" + f"\n*{c.short_doc or 'No details provided.'}*" for c in filtered_commands + ] - for details in cat_cmds: - - # keep details together, paginating early if it won't fit - lines_adding = len(details.split('\n')) + print_cat - if paginator._linecount + lines_adding > self._max_lines: - paginator._linecount = 0 - new_page = True - paginator.close_page() - - # new page so print category title again - print_cat = 1 + description = f"**{category.name}**\n*{category.description}*\n\n**Commands:**" - if print_cat: - if new_page: - paginator.add_line('') - paginator.add_line(category) - print_cat = 0 - - paginator.add_line(details) + await LinePaginator.paginate( + lines, self.context, embed, max_lines=COMMANDS_PER_PAGE, + max_size=2040, description=description, cleanup=True + ) - # save organised pages to session - self._pages = paginator.pages + async def send_bot_help(self, mapping: dict) -> None: + """Sends help for all bot commands and cogs.""" + bot = self.context.bot - def embed_page(self, page_number: int = 0) -> Embed: - """Returns an Embed with the requested page formatted within.""" embed = Embed() + embed.set_author(name="Command Help", icon_url=constants.Icons.questionmark) - # if command or cog, add query to title for pages other than first - if isinstance(self.query, (commands.Command, Cog)) and page_number > 0: - title = f'Command Help | "{self.query.name}"' - else: - title = self.title + filter_commands = await self.filter_commands(bot.commands, sort=True, key=self._category_key) - embed.set_author(name=title, icon_url=constants.Icons.questionmark) - embed.description = self._pages[page_number] + lines = [] - # add page counter to footer if paginating - page_count = len(self._pages) - if page_count > 1: - embed.set_footer(text=f'Page {self._current_page+1} / {page_count}') + for cog_or_category, _commands in itertools.groupby(filter_commands, key=self._category_key): + sorted_commands = sorted(_commands, key=lambda c: c.name) - return embed + if len(sorted_commands) == 0: + continue - async def update_page(self, page_number: int = 0) -> None: - """Sends the intial message, or changes the existing one to the given page number.""" - self._current_page = page_number - embed_page = self.embed_page(page_number) + fmt = [ + f"`{PREFIX}{c.qualified_name}{f' {c.signature}' if c.signature else ''}`" + f"\n*{c.short_doc or 'No details provided.'}*" for c in sorted_commands + ] - if not self.message: - self.message = await self.destination.send(embed=embed_page) - else: - await self.message.edit(embed=embed_page) + # we can't embed a '\n'.join() inside an f-string so this is a bit of a compromise + def get_fmt(i: int) -> str: + """Get a formatted version of commands for an index.""" + return "\n".join(fmt[i:i+COMMANDS_PER_PAGE]) - @classmethod - async def start(cls, ctx: Context, *command, **options) -> "HelpSession": - """ - Create and begin a help session based on the given command context. - - Available options kwargs: - * cleanup: Optional[bool] - Set to `True` to have the message deleted on session end. Defaults to `False`. - * only_can_run: Optional[bool] - Set to `True` to hide commands the user can't run. Defaults to `False`. - * show_hidden: Optional[bool] - Set to `True` to include hidden commands. Defaults to `False`. - * max_lines: Optional[int] - Sets the max number of lines the paginator will add to a single page. Defaults to 20. - """ - session = cls(ctx, *command, **options) - await session.prepare() - - return session - - async def stop(self) -> None: - """Stops the help session, removes event listeners and attempts to delete the help message.""" - self._bot.remove_listener(self.on_reaction_add) - self._bot.remove_listener(self.on_message_delete) - - # ignore if permission issue, or the message doesn't exist - with suppress(HTTPException, AttributeError): - if self._cleanup: - await self.message.delete() - else: - await self.message.clear_reactions() - - @property - def is_first_page(self) -> bool: - """Check if session is currently showing the first page.""" - return self._current_page == 0 - - @property - def is_last_page(self) -> bool: - """Check if the session is currently showing the last page.""" - return self._current_page == (len(self._pages)-1) - - async def do_first(self) -> None: - """Event that is called when the user requests the first page.""" - if not self.is_first_page: - await self.update_page(0) - - async def do_back(self) -> None: - """Event that is called when the user requests the previous page.""" - if not self.is_first_page: - await self.update_page(self._current_page-1) - - async def do_next(self) -> None: - """Event that is called when the user requests the next page.""" - if not self.is_last_page: - await self.update_page(self._current_page+1) - - async def do_end(self) -> None: - """Event that is called when the user requests the last page.""" - if not self.is_last_page: - await self.update_page(len(self._pages)-1) - - async def do_stop(self) -> None: - """Event that is called when the user requests to stop the help session.""" - await self.message.delete() - - -class Help(DiscordCog): - """Custom Embed Pagination Help feature.""" + # this is a bit yuck because moderation category has 8 commands which needs to be split over 2 pages. + # pretty much it only splits that category, but also gives the number of commands it's adding to + # the pages every iteration so we can easily use this below rather than trying to split the string. + lines.extend( + ( + (f"**{cog_or_category}**\n{get_fmt(i)}", len(fmt[i:i+COMMANDS_PER_PAGE])) + for i in range(0, len(sorted_commands), COMMANDS_PER_PAGE) + ) + ) - @commands.command('help') - @redirect_output(destination_channel=Channels.bot, bypass_roles=STAFF_ROLES) - async def new_help(self, ctx: Context, *commands) -> None: - """Shows Command Help.""" - try: - await HelpSession.start(ctx, *commands) - except HelpQueryNotFound as error: - embed = Embed() - embed.colour = Colour.red() - embed.title = str(error) + pages = [] + counter = 0 + formatted = "" + for (fmt, length) in lines: + counter += length + if counter > COMMANDS_PER_PAGE: + # force a new page on paginator even if it falls short of the max pages + # since we still want to group categories/cogs. + counter = length + pages.append(formatted) + formatted = f"{fmt}\n\n" + continue + formatted += f"{fmt}\n\n" - if error.possible_matches: - matches = '\n'.join(error.possible_matches.keys()) - embed.description = f'**Did you mean:**\n`{matches}`' + await LinePaginator.paginate(pages, self.context, embed=embed, max_lines=1, max_size=2040, cleanup=True) - await ctx.send(embed=embed) +class Help(Cog): + """Custom Embed Pagination Help feature.""" -def unload(bot: Bot) -> None: - """ - Reinstates the original help command. + def __init__(self, bot: Bot) -> None: + self.bot = bot + self.old_help_command = bot.help_command + bot.help_command = CustomHelpCommand() + bot.help_command.cog = self - This is run if the cog raises an exception on load, or if the extension is unloaded. - """ - bot.remove_command('help') - bot.add_command(bot._old_help) + def cog_unload(self) -> None: + """Reset the help command when the cog is unloaded.""" + self.bot.help_command = self.old_help_command def setup(bot: Bot) -> None: - """ - The setup for the help extension. - - This is called automatically on `bot.load_extension` being run. - - Stores the original help command instance on the `bot._old_help` attribute for later - reinstatement, before removing it from the command registry so the new help command can be - loaded successfully. - - If an exception is raised during the loading of the cog, `unload` will be called in order to - reinstate the original help command. - """ - bot._old_help = bot.get_command('help') - bot.remove_command('help') - - try: - bot.add_cog(Help()) - except Exception: - unload(bot) - raise - - -def teardown(bot: Bot) -> None: - """ - The teardown for the help extension. - - This is called automatically on `bot.unload_extension` being run. - - Calls `unload` in order to reinstate the original help command. - """ - unload(bot) + """Load the Help cog.""" + bot.add_cog(Help(bot)) + log.info("Cog loaded: Help") diff --git a/bot/decorators.py b/bot/decorators.py index 935df4af0..b31324f36 100644 --- a/bot/decorators.py +++ b/bot/decorators.py @@ -6,7 +6,7 @@ from functools import wraps from typing import Callable, Container, Union from weakref import WeakValueDictionary -from discord import Colour, Embed, Member +from discord import Colour, Embed, Member, Message from discord.errors import NotFound from discord.ext import commands from discord.ext.commands import CheckFailure, Cog, Context @@ -98,6 +98,20 @@ def locked() -> Callable: return wrap +async def delete_invocation(ctx: Context, message: Message) -> None: + """Task to delete the invocation and user redirection messages.""" + if RedirectOutput.delete_invocation: + await sleep(RedirectOutput.delete_delay) + + with suppress(NotFound): + await message.delete() + log.trace("Redirect output: Deleted user redirection message") + + with suppress(NotFound): + await ctx.message.delete() + log.trace("Redirect output: Deleted invocation message") + + def redirect_output(destination_channel: int, bypass_roles: Container[int] = None) -> Callable: """ Changes the channel in the context of the command to redirect the output to a certain channel. @@ -131,17 +145,8 @@ def redirect_output(destination_channel: int, bypass_roles: Container[int] = Non f"Hey, {ctx.author.mention}, you can find the output of your command here: " f"{redirect_channel.mention}" ) - - if RedirectOutput.delete_invocation: - await sleep(RedirectOutput.delete_delay) - - with suppress(NotFound): - await message.delete() - log.trace("Redirect output: Deleted user redirection message") - - with suppress(NotFound): - await ctx.message.delete() - log.trace("Redirect output: Deleted invocation message") + # we need to run it in a task for the help command - which gets held up if waiting for invocation deletion. + ctx.bot.loop.create_task(delete_invocation(ctx, message)) return inner return wrap diff --git a/bot/pagination.py b/bot/pagination.py index 76082f459..f2cf192c4 100644 --- a/bot/pagination.py +++ b/bot/pagination.py @@ -1,8 +1,9 @@ import asyncio import logging +from contextlib import suppress from typing import Iterable, List, Optional, Tuple -from discord import Embed, Member, Message, Reaction +from discord import Embed, HTTPException, Member, Message, Reaction from discord.abc import User from discord.ext.commands import Context, Paginator @@ -99,7 +100,9 @@ class LinePaginator(Paginator): timeout: int = 300, footer_text: str = None, url: str = None, - exception_on_empty_embed: bool = False + exception_on_empty_embed: bool = False, + description: str = '', + cleanup: bool = False ) -> Optional[Message]: """ Use a paginator and set of reactions to provide pagination over a set of lines. @@ -111,6 +114,9 @@ class LinePaginator(Paginator): Pagination will also be removed automatically if no reaction is added for five minutes (300 seconds). + The description is a string that should appear at the top of every page. + If cleanup is True, the paginated message will be deleted when :x: reaction is added. + Example: >>> embed = Embed() >>> embed.set_author(name="Some Operation", url=url, icon_url=icon) @@ -161,7 +167,7 @@ class LinePaginator(Paginator): log.debug(f"Paginator created with {len(paginator.pages)} pages") - embed.description = paginator.pages[current_page] + embed.description = description + paginator.pages[current_page] if len(paginator.pages) <= 1: if footer_text: @@ -205,6 +211,11 @@ class LinePaginator(Paginator): if reaction.emoji == DELETE_EMOJI: log.debug("Got delete reaction") + if cleanup: + with suppress(HTTPException, AttributeError): + log.debug("Deleting help message") + await message.delete() + return break if reaction.emoji == FIRST_EMOJI: @@ -215,7 +226,7 @@ class LinePaginator(Paginator): embed.description = "" await message.edit(embed=embed) - embed.description = paginator.pages[current_page] + embed.description = description + paginator.pages[current_page] if footer_text: embed.set_footer(text=f"{footer_text} (Page {current_page + 1}/{len(paginator.pages)})") else: @@ -230,7 +241,7 @@ class LinePaginator(Paginator): embed.description = "" await message.edit(embed=embed) - embed.description = paginator.pages[current_page] + embed.description = description + paginator.pages[current_page] if footer_text: embed.set_footer(text=f"{footer_text} (Page {current_page + 1}/{len(paginator.pages)})") else: @@ -249,7 +260,7 @@ class LinePaginator(Paginator): embed.description = "" await message.edit(embed=embed) - embed.description = paginator.pages[current_page] + embed.description = description + paginator.pages[current_page] if footer_text: embed.set_footer(text=f"{footer_text} (Page {current_page + 1}/{len(paginator.pages)})") @@ -270,7 +281,7 @@ class LinePaginator(Paginator): embed.description = "" await message.edit(embed=embed) - embed.description = paginator.pages[current_page] + embed.description = description + paginator.pages[current_page] if footer_text: embed.set_footer(text=f"{footer_text} (Page {current_page + 1}/{len(paginator.pages)})") -- cgit v1.2.3 From 87f9fdda65e7a9c425e727b766d57dda2ce5a364 Mon Sep 17 00:00:00 2001 From: mathsman5133 Date: Sat, 12 Oct 2019 16:48:42 +1100 Subject: Minor formatting changes to align with current help. --- bot/cogs/help.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/help.py b/bot/cogs/help.py index 16fd62154..dd19dffd3 100644 --- a/bot/cogs/help.py +++ b/bot/cogs/help.py @@ -190,7 +190,7 @@ class CustomHelpCommand(HelpCommand): # check if the user is allowed to run this command if not await command.can_run(self.context): - fmt += "**You cannot run this command.**" + fmt += "***You cannot run this command.***\n\n" fmt += f"*{command.help or 'No details provided.'}*\n" embed.description = fmt @@ -239,7 +239,7 @@ class CustomHelpCommand(HelpCommand): lines = [ f"`{PREFIX}{c.qualified_name}{f' {c.signature}' if c.signature else ''}`" - f"\n*{c.short_doc or 'No details provided.'}*\n" for c in _commands + f"\n*{c.short_doc or 'No details provided.'}*" for c in _commands ] embed.description += "\n".join(n for n in lines) -- cgit v1.2.3 From 5ebb87d4dbdf2c4c415f4a37b01746909f2b67b9 Mon Sep 17 00:00:00 2001 From: mathsman5133 Date: Sun, 13 Oct 2019 17:03:06 +1100 Subject: Add a special case for when the help command invokes wolfram checks. - Before, running `!help` would invoke the cooldown check, and increase the cooldown counter unnecessarily as no wolfram API calls were being made. - Once `!help` was called enough, the bot would send an error embed to let you know your wolfram cooldown has expired. --- bot/cogs/wolfram.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bot/cogs/wolfram.py b/bot/cogs/wolfram.py index ab0ed2472..3da349bd1 100644 --- a/bot/cogs/wolfram.py +++ b/bot/cogs/wolfram.py @@ -59,6 +59,14 @@ def custom_cooldown(*ignore: List[int]) -> Callable: A list of roles may be provided to ignore the per-user cooldown """ async def predicate(ctx: Context) -> bool: + if ctx.invoked_with == 'help': + # if the invoked command is help we don't want to increase the ratelimits since it's not actually + # invoking the command/making a request, so instead just check if the user/guild are on cooldown. + guild_cooldown = not guildcd.get_bucket(ctx.message).get_tokens() == 0 # if guild is on cooldown + if not any(r.id in ignore for r in ctx.author.roles): # check user bucket if user is not ignored + return guild_cooldown and not usercd.get_bucket(ctx.message).get_tokens() == 0 + return guild_cooldown + user_bucket = usercd.get_bucket(ctx.message) if all(role.id not in ignore for role in ctx.author.roles): -- cgit v1.2.3 From 79d5b2df86d2676caed7665f9e083ffbb984ebc2 Mon Sep 17 00:00:00 2001 From: mathsman5133 Date: Sun, 13 Oct 2019 17:08:27 +1100 Subject: Few changes to keep formatting same as current - Change `add_field` back to `description` for error message possible matches - Only add `Commands` and `Subcommands` if subcommands exist to cog/group/command help --- bot/cogs/help.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/bot/cogs/help.py b/bot/cogs/help.py index dd19dffd3..3513c3373 100644 --- a/bot/cogs/help.py +++ b/bot/cogs/help.py @@ -165,7 +165,7 @@ class CustomHelpCommand(HelpCommand): if getattr(error, "possible_matches", None): matches = "\n".join(f"`{n}`" for n in error.possible_matches.keys()) - embed.add_field(name="Did you mean:", value=matches) + embed.description = f"**Did you mean:**\n{matches}" await self.context.send(embed=embed) @@ -224,7 +224,9 @@ class CustomHelpCommand(HelpCommand): f"**`{PREFIX}{c.qualified_name}{f' {c.signature}' if c.signature else ''}`**" f"\n*{c.short_doc or 'No details provided.'}*" for c in _commands ) - embed.description += f"\n**Subcommands:**\n{fmt}" + if fmt: + embed.description += f"\n**Subcommands:**\n{fmt}" + message = await self.context.send(embed=embed) await help_cleanup(self.context.bot, self.context.author, message) @@ -235,13 +237,15 @@ class CustomHelpCommand(HelpCommand): embed = Embed() embed.set_author(name="Command Help", icon_url=constants.Icons.questionmark) - embed.description = f"**{cog.qualified_name}**\n*{cog.description}*\n\n**Commands:**\n" + embed.description = f"**{cog.qualified_name}**\n*{cog.description}*" lines = [ f"`{PREFIX}{c.qualified_name}{f' {c.signature}' if c.signature else ''}`" f"\n*{c.short_doc or 'No details provided.'}*" for c in _commands ] - embed.description += "\n".join(n for n in lines) + if lines: + embed.description += "\n\n**Commands:**\n" + embed.description += "\n".join(n for n in lines) message = await self.context.send(embed=embed) await help_cleanup(self.context.bot, self.context.author, message) @@ -281,7 +285,10 @@ class CustomHelpCommand(HelpCommand): f"\n*{c.short_doc or 'No details provided.'}*" for c in filtered_commands ] - description = f"**{category.name}**\n*{category.description}*\n\n**Commands:**" + description = f"**{category.name}**\n*{category.description}*" + + if lines: + description += "\n\n**Commands:**" await LinePaginator.paginate( lines, self.context, embed, max_lines=COMMANDS_PER_PAGE, @@ -339,6 +346,10 @@ class CustomHelpCommand(HelpCommand): continue formatted += f"{fmt}\n\n" + if formatted: + # add any remaining command help that didn't get added in the last iteration above. + pages.append(formatted) + await LinePaginator.paginate(pages, self.context, embed=embed, max_lines=1, max_size=2040, cleanup=True) -- cgit v1.2.3 From d8384b214bb0085bc53af9ef39386662957274f9 Mon Sep 17 00:00:00 2001 From: mathsman5133 Date: Sun, 3 Nov 2019 14:33:10 +1100 Subject: Show a maximum of 8 commands per page rather than 5. --- bot/cogs/help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/help.py b/bot/cogs/help.py index 3513c3373..cd8c797b4 100644 --- a/bot/cogs/help.py +++ b/bot/cogs/help.py @@ -15,7 +15,7 @@ from bot.pagination import DELETE_EMOJI, LinePaginator log = logging.getLogger(__name__) -COMMANDS_PER_PAGE = 5 +COMMANDS_PER_PAGE = 8 PREFIX = constants.Bot.prefix Category = namedtuple("Category", ["name", "description", "cogs"]) -- cgit v1.2.3 From af44f629dc5666ea17dd435dd3329784651c4412 Mon Sep 17 00:00:00 2001 From: mathsman5133 Date: Sat, 16 Nov 2019 21:07:24 +1100 Subject: Apply suggestions from review - Description was the same as prefix parameter of paginator - Cleanup is redundant pending closure of #514 - Clean/fix couple if statements in help.py --- bot/cogs/help.py | 14 +++++++------- bot/pagination.py | 23 ++++++----------------- 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/bot/cogs/help.py b/bot/cogs/help.py index cd8c797b4..76d584d64 100644 --- a/bot/cogs/help.py +++ b/bot/cogs/help.py @@ -126,7 +126,8 @@ class CustomHelpCommand(HelpCommand): choices.add(str(c)) # all aliases if it's just a command - choices.update(n for n in c.aliases if isinstance(c, Command)) + if isinstance(c, Command): + choices.update(c.aliases) # else aliases with parent if group. we need to strip() in case it's a Command and `full_parent` is None, # otherwise we get 2 commands: ` help` and normal `help`. @@ -137,7 +138,7 @@ class CustomHelpCommand(HelpCommand): choices.update(self.context.bot.cogs) # all category names - choices.update(getattr(n, "category", None) for n in self.context.bot.cogs if hasattr(n, "category")) + choices.update(n.category for n in self.context.bot.cogs if hasattr(n, "category")) return choices def command_not_found(self, string: str) -> "HelpQueryNotFound": @@ -164,7 +165,7 @@ class CustomHelpCommand(HelpCommand): embed = Embed(colour=Colour.red(), title=str(error)) if getattr(error, "possible_matches", None): - matches = "\n".join(f"`{n}`" for n in error.possible_matches.keys()) + matches = "\n".join(f"`{n}`" for n in error.possible_matches) embed.description = f"**Did you mean:**\n{matches}" await self.context.send(embed=embed) @@ -285,14 +286,13 @@ class CustomHelpCommand(HelpCommand): f"\n*{c.short_doc or 'No details provided.'}*" for c in filtered_commands ] - description = f"**{category.name}**\n*{category.description}*" + description = f"```**{category.name}**\n*{category.description}*" if lines: description += "\n\n**Commands:**" await LinePaginator.paginate( - lines, self.context, embed, max_lines=COMMANDS_PER_PAGE, - max_size=2040, description=description, cleanup=True + lines, self.context, embed, prefix=description, max_lines=COMMANDS_PER_PAGE, max_size=2040 ) async def send_bot_help(self, mapping: dict) -> None: @@ -350,7 +350,7 @@ class CustomHelpCommand(HelpCommand): # add any remaining command help that didn't get added in the last iteration above. pages.append(formatted) - await LinePaginator.paginate(pages, self.context, embed=embed, max_lines=1, max_size=2040, cleanup=True) + await LinePaginator.paginate(pages, self.context, embed=embed, max_lines=1, max_size=2040) class Help(Cog): diff --git a/bot/pagination.py b/bot/pagination.py index f2cf192c4..bb49ead5e 100644 --- a/bot/pagination.py +++ b/bot/pagination.py @@ -1,9 +1,8 @@ import asyncio import logging -from contextlib import suppress from typing import Iterable, List, Optional, Tuple -from discord import Embed, HTTPException, Member, Message, Reaction +from discord import Embed, Member, Message, Reaction from discord.abc import User from discord.ext.commands import Context, Paginator @@ -101,8 +100,6 @@ class LinePaginator(Paginator): footer_text: str = None, url: str = None, exception_on_empty_embed: bool = False, - description: str = '', - cleanup: bool = False ) -> Optional[Message]: """ Use a paginator and set of reactions to provide pagination over a set of lines. @@ -114,9 +111,6 @@ class LinePaginator(Paginator): Pagination will also be removed automatically if no reaction is added for five minutes (300 seconds). - The description is a string that should appear at the top of every page. - If cleanup is True, the paginated message will be deleted when :x: reaction is added. - Example: >>> embed = Embed() >>> embed.set_author(name="Some Operation", url=url, icon_url=icon) @@ -167,7 +161,7 @@ class LinePaginator(Paginator): log.debug(f"Paginator created with {len(paginator.pages)} pages") - embed.description = description + paginator.pages[current_page] + embed.description = paginator.pages[current_page] if len(paginator.pages) <= 1: if footer_text: @@ -211,11 +205,6 @@ class LinePaginator(Paginator): if reaction.emoji == DELETE_EMOJI: log.debug("Got delete reaction") - if cleanup: - with suppress(HTTPException, AttributeError): - log.debug("Deleting help message") - await message.delete() - return break if reaction.emoji == FIRST_EMOJI: @@ -226,7 +215,7 @@ class LinePaginator(Paginator): embed.description = "" await message.edit(embed=embed) - embed.description = description + paginator.pages[current_page] + embed.description = paginator.pages[current_page] if footer_text: embed.set_footer(text=f"{footer_text} (Page {current_page + 1}/{len(paginator.pages)})") else: @@ -241,7 +230,7 @@ class LinePaginator(Paginator): embed.description = "" await message.edit(embed=embed) - embed.description = description + paginator.pages[current_page] + embed.description = paginator.pages[current_page] if footer_text: embed.set_footer(text=f"{footer_text} (Page {current_page + 1}/{len(paginator.pages)})") else: @@ -260,7 +249,7 @@ class LinePaginator(Paginator): embed.description = "" await message.edit(embed=embed) - embed.description = description + paginator.pages[current_page] + embed.description = paginator.pages[current_page] if footer_text: embed.set_footer(text=f"{footer_text} (Page {current_page + 1}/{len(paginator.pages)})") @@ -281,7 +270,7 @@ class LinePaginator(Paginator): embed.description = "" await message.edit(embed=embed) - embed.description = description + paginator.pages[current_page] + embed.description = paginator.pages[current_page] if footer_text: embed.set_footer(text=f"{footer_text} (Page {current_page + 1}/{len(paginator.pages)})") -- cgit v1.2.3 From 5f896ce4bdb78039624af4a10e16e68bbfff277d Mon Sep 17 00:00:00 2001 From: mathsman5133 Date: Sat, 16 Nov 2019 21:13:52 +1100 Subject: Remove trailing comma --- bot/pagination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/pagination.py b/bot/pagination.py index bb49ead5e..76082f459 100644 --- a/bot/pagination.py +++ b/bot/pagination.py @@ -99,7 +99,7 @@ class LinePaginator(Paginator): timeout: int = 300, footer_text: str = None, url: str = None, - exception_on_empty_embed: bool = False, + exception_on_empty_embed: bool = False ) -> Optional[Message]: """ Use a paginator and set of reactions to provide pagination over a set of lines. -- cgit v1.2.3 From a590fdb5ab946daa9633a8a1513749f6ae399570 Mon Sep 17 00:00:00 2001 From: mathsman5133 Date: Mon, 10 Feb 2020 22:01:57 +1100 Subject: Use the new :trashcan: emoji to delete the help message, as per #625 --- bot/cogs/help.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bot/cogs/help.py b/bot/cogs/help.py index 76d584d64..e002192ff 100644 --- a/bot/cogs/help.py +++ b/bot/cogs/help.py @@ -9,13 +9,14 @@ from discord.ext.commands import Bot, Cog, Command, Context, Group, HelpCommand from fuzzywuzzy import fuzz, process from bot import constants -from bot.constants import Channels, STAFF_ROLES +from bot.constants import Channels, Emojis, STAFF_ROLES from bot.decorators import redirect_output -from bot.pagination import DELETE_EMOJI, LinePaginator +from bot.pagination import LinePaginator log = logging.getLogger(__name__) COMMANDS_PER_PAGE = 8 +DELETE_EMOJI = Emojis.trashcan PREFIX = constants.Bot.prefix Category = namedtuple("Category", ["name", "description", "cogs"]) @@ -25,11 +26,11 @@ async def help_cleanup(bot: Bot, author: Member, message: Message) -> None: """ Runs the cleanup for the help command. - Adds a :x: reaction that, when clicked, will delete the help message. + Adds the :trashcan: reaction that, when clicked, will delete the help message. After a 300 second timeout, the reaction will be removed. """ def check(r: Reaction, u: User) -> bool: - """Checks the reaction is :x:, the author is original author and messages are the same.""" + """Checks the reaction is :trashcan:, the author is original author and messages are the same.""" return str(r) == DELETE_EMOJI and u.id == author.id and r.message.id == message.id await message.add_reaction(DELETE_EMOJI) -- cgit v1.2.3 From 86ee960948d59fc0e4aa1b085633ba95ed5f788e Mon Sep 17 00:00:00 2001 From: mathsman5133 Date: Tue, 10 Mar 2020 19:27:12 +1100 Subject: Apply suggestions from Mark's code review. --- bot/cogs/help.py | 123 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 63 insertions(+), 60 deletions(-) diff --git a/bot/cogs/help.py b/bot/cogs/help.py index e002192ff..334e7fc53 100644 --- a/bot/cogs/help.py +++ b/bot/cogs/help.py @@ -3,6 +3,7 @@ import logging from asyncio import TimeoutError from collections import namedtuple from contextlib import suppress +from typing import List from discord import Colour, Embed, HTTPException, Member, Message, Reaction, User from discord.ext.commands import Bot, Cog, Command, Context, Group, HelpCommand @@ -34,10 +35,13 @@ async def help_cleanup(bot: Bot, author: Member, message: Message) -> None: return str(r) == DELETE_EMOJI and u.id == author.id and r.message.id == message.id await message.add_reaction(DELETE_EMOJI) - with suppress(HTTPException, TimeoutError): - _, _ = await bot.wait_for("reaction_add", check=check, timeout=300) + + try: + await bot.wait_for("reaction_add", check=check, timeout=300) await message.delete() return + except (HTTPException, TimeoutError): + pass await message.remove_reaction(DELETE_EMOJI, bot.user) @@ -107,10 +111,12 @@ class CustomHelpCommand(HelpCommand): # it's either a cog, group, command or subcommand, let super deal with it await super().command_callback(ctx, command=command) - def get_all_help_choices(self) -> set: + async def get_all_help_choices(self) -> set: """ Get all the possible options for getting help in the bot. + This will only display commands the author has permission to run. + These include: - Category names - Cog names @@ -122,44 +128,42 @@ class CustomHelpCommand(HelpCommand): """ # first get all commands including subcommands and full command name aliases choices = set() - for c in self.context.bot.walk_commands(): + for c in await self.filter_commands(self.context.bot.walk_commands()): # the the command or group name choices.add(str(c)) - # all aliases if it's just a command if isinstance(c, Command): + # all aliases if it's just a command choices.update(c.aliases) + else: + # otherwise we need to add the parent name in + choices.update(f"{c.full_parent_name} {a}" for a in c.aliases) - # else aliases with parent if group. we need to strip() in case it's a Command and `full_parent` is None, - # otherwise we get 2 commands: ` help` and normal `help`. - # We could do case-by-case with f-string but this is the cleanest solution - choices.update(f"{c.full_parent_name} {a}".strip() for a in c.aliases) - - # all cog names + # all cog names choices.update(self.context.bot.cogs) # all category names choices.update(n.category for n in self.context.bot.cogs if hasattr(n, "category")) return choices - def command_not_found(self, string: str) -> "HelpQueryNotFound": + async def command_not_found(self, string: str) -> "HelpQueryNotFound": """ Handles when a query does not match a valid command, group, cog or category. Will return an instance of the `HelpQueryNotFound` exception with the error message and possible matches. """ - choices = self.get_all_help_choices() + choices = await self.get_all_help_choices() result = process.extractBests(string, choices, scorer=fuzz.ratio, score_cutoff=90) return HelpQueryNotFound(f'Query "{string}" not found.', dict(result)) - def subcommand_not_found(self, command: Command, string: str) -> "HelpQueryNotFound": + async def subcommand_not_found(self, command: Command, string: str) -> "HelpQueryNotFound": """ Redirects the error to `command_not_found`. `command_not_found` deals with searching and getting best choices for both commands and subcommands. """ - return self.command_not_found(f"{command.qualified_name} {string}") + return await self.command_not_found(f"{command.qualified_name} {string}") async def send_error_message(self, error: HelpQueryNotFound) -> None: """Send the error message to the channel.""" @@ -205,6 +209,18 @@ class CustomHelpCommand(HelpCommand): message = await self.context.send(embed=embed) await help_cleanup(self.context.bot, self.context.author, message) + @staticmethod + def get_commands_brief_details(commands_: List[Command]) -> str: + """Formats the prefix, command name and signature, and short doc for an iterable of commands.""" + details = "" + for c in commands_: + if c.signature: + details += f"\n**`{PREFIX}{c.qualified_name} {c.signature}`**\n*{c.short_doc or 'No details provided'}*" + else: + details += f"\n**`{PREFIX}{c.qualified_name}`**\n*{c.short_doc or 'No details provided.'}*" + + return details + async def send_group_help(self, group: Group) -> None: """Sends help for a group command.""" subcommands = group.commands @@ -215,19 +231,13 @@ class CustomHelpCommand(HelpCommand): return # remove commands that the user can't run and are hidden, and sort by name - _commands = await self.filter_commands(subcommands, sort=True) + commands_ = await self.filter_commands(subcommands, sort=True) embed = await self.command_formatting(group) - # add in subcommands with brief help - # note: the extra f-string around the signature is necessary because otherwise an extra space before the - # last back tick is present. - fmt = "\n".join( - f"**`{PREFIX}{c.qualified_name}{f' {c.signature}' if c.signature else ''}`**" - f"\n*{c.short_doc or 'No details provided.'}*" for c in _commands - ) - if fmt: - embed.description += f"\n**Subcommands:**\n{fmt}" + command_details = self.get_commands_brief_details(commands_) + if command_details: + embed.description += f"\n**Subcommands:**\n{command_details}" message = await self.context.send(embed=embed) await help_cleanup(self.context.bot, self.context.author, message) @@ -235,19 +245,15 @@ class CustomHelpCommand(HelpCommand): async def send_cog_help(self, cog: Cog) -> None: """Send help for a cog.""" # sort commands by name, and remove any the user cant run or are hidden. - _commands = await self.filter_commands(cog.get_commands(), sort=True) + commands_ = await self.filter_commands(cog.get_commands(), sort=True) embed = Embed() embed.set_author(name="Command Help", icon_url=constants.Icons.questionmark) embed.description = f"**{cog.qualified_name}**\n*{cog.description}*" - lines = [ - f"`{PREFIX}{c.qualified_name}{f' {c.signature}' if c.signature else ''}`" - f"\n*{c.short_doc or 'No details provided.'}*" for c in _commands - ] - if lines: - embed.description += "\n\n**Commands:**\n" - embed.description += "\n".join(n for n in lines) + command_details = self.get_commands_brief_details(commands_) + if command_details: + embed.description += f"\n\n**Commands:**\n{command_details}" message = await self.context.send(embed=embed) await help_cleanup(self.context.bot, self.context.author, message) @@ -280,20 +286,25 @@ class CustomHelpCommand(HelpCommand): for c in category.cogs: all_commands.extend(c.get_commands()) - filtered_commands = await self.filter_commands(all_commands, sort=True, key=self._category_key) + filtered_commands = await self.filter_commands(all_commands, sort=True) lines = [ f"`{PREFIX}{c.qualified_name}{f' {c.signature}' if c.signature else ''}`" f"\n*{c.short_doc or 'No details provided.'}*" for c in filtered_commands ] - description = f"```**{category.name}**\n*{category.description}*" + description = f"**{category.name}**\n*{category.description}*" if lines: description += "\n\n**Commands:**" await LinePaginator.paginate( - lines, self.context, embed, prefix=description, max_lines=COMMANDS_PER_PAGE, max_size=2040 + lines, + self.context, + embed, + prefix=description, + max_lines=COMMANDS_PER_PAGE, + max_size=2040 ) async def send_bot_help(self, mapping: dict) -> None: @@ -305,7 +316,7 @@ class CustomHelpCommand(HelpCommand): filter_commands = await self.filter_commands(bot.commands, sort=True, key=self._category_key) - lines = [] + cog_or_category_pages = [] for cog_or_category, _commands in itertools.groupby(filter_commands, key=self._category_key): sorted_commands = sorted(_commands, key=lambda c: c.name) @@ -313,43 +324,35 @@ class CustomHelpCommand(HelpCommand): if len(sorted_commands) == 0: continue - fmt = [ + command_details = [ f"`{PREFIX}{c.qualified_name}{f' {c.signature}' if c.signature else ''}`" f"\n*{c.short_doc or 'No details provided.'}*" for c in sorted_commands ] - # we can't embed a '\n'.join() inside an f-string so this is a bit of a compromise - def get_fmt(i: int) -> str: - """Get a formatted version of commands for an index.""" - return "\n".join(fmt[i:i+COMMANDS_PER_PAGE]) - - # this is a bit yuck because moderation category has 8 commands which needs to be split over 2 pages. - # pretty much it only splits that category, but also gives the number of commands it's adding to - # the pages every iteration so we can easily use this below rather than trying to split the string. - lines.extend( - ( - (f"**{cog_or_category}**\n{get_fmt(i)}", len(fmt[i:i+COMMANDS_PER_PAGE])) - for i in range(0, len(sorted_commands), COMMANDS_PER_PAGE) - ) - ) + # Split cogs or categories which have too many commands to fit in one page. + # The length of commands is included for later use when aggregating into pages for the paginator. + for i in range(0, len(sorted_commands), COMMANDS_PER_PAGE): + truncated_fmt = command_details[i:i + COMMANDS_PER_PAGE] + joined_fmt = "\n".join(truncated_fmt) + cog_or_category_pages.append((f"**{cog_or_category}**\n{joined_fmt}", len(truncated_fmt))) pages = [] counter = 0 - formatted = "" - for (fmt, length) in lines: + page = "" + for fmt, length in cog_or_category_pages: counter += length if counter > COMMANDS_PER_PAGE: # force a new page on paginator even if it falls short of the max pages # since we still want to group categories/cogs. counter = length - pages.append(formatted) - formatted = f"{fmt}\n\n" - continue - formatted += f"{fmt}\n\n" + pages.append(page) + page = f"{fmt}\n\n" + else: + page += f"{fmt}\n\n" - if formatted: + if page: # add any remaining command help that didn't get added in the last iteration above. - pages.append(formatted) + pages.append(page) await LinePaginator.paginate(pages, self.context, embed=embed, max_lines=1, max_size=2040) -- cgit v1.2.3 From 1f68c35a0343c99c1364eba58cafd43aefb0fabc Mon Sep 17 00:00:00 2001 From: mathsman5133 Date: Tue, 31 Mar 2020 20:05:28 +1100 Subject: Apply suggestions from review. - Make exception handling for bin reaction more specific - Channel constants were updated recently - Suggest category names - Tidy up signature formatting - Move score cutoff to 80 to allow a few more matches --- bot/cogs/help.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/bot/cogs/help.py b/bot/cogs/help.py index 334e7fc53..f020a1725 100644 --- a/bot/cogs/help.py +++ b/bot/cogs/help.py @@ -5,7 +5,7 @@ from collections import namedtuple from contextlib import suppress from typing import List -from discord import Colour, Embed, HTTPException, Member, Message, Reaction, User +from discord import Colour, Embed, NotFound, Member, Message, Reaction, User from discord.ext.commands import Bot, Cog, Command, Context, Group, HelpCommand from fuzzywuzzy import fuzz, process @@ -39,12 +39,11 @@ async def help_cleanup(bot: Bot, author: Member, message: Message) -> None: try: await bot.wait_for("reaction_add", check=check, timeout=300) await message.delete() - return - except (HTTPException, TimeoutError): + except TimeoutError: + await message.remove_reaction(DELETE_EMOJI, bot.user) + except NotFound: pass - await message.remove_reaction(DELETE_EMOJI, bot.user) - class HelpQueryNotFound(ValueError): """ @@ -75,7 +74,7 @@ class CustomHelpCommand(HelpCommand): def __init__(self): super().__init__(command_attrs={"help": "Shows help for bot commands"}) - @redirect_output(destination_channel=Channels.bot, bypass_roles=STAFF_ROLES) + @redirect_output(destination_channel=Channels.bot_commands, bypass_roles=STAFF_ROLES) async def prepare_help_command(self, ctx: Context, command: str = None) -> None: """Adjust context to redirect to a new channel if required.""" self.context = ctx @@ -143,7 +142,7 @@ class CustomHelpCommand(HelpCommand): choices.update(self.context.bot.cogs) # all category names - choices.update(n.category for n in self.context.bot.cogs if hasattr(n, "category")) + choices.update(n.category for n in self.context.bot.cogs.values() if hasattr(n, "category")) return choices async def command_not_found(self, string: str) -> "HelpQueryNotFound": @@ -153,7 +152,7 @@ class CustomHelpCommand(HelpCommand): Will return an instance of the `HelpQueryNotFound` exception with the error message and possible matches. """ choices = await self.get_all_help_choices() - result = process.extractBests(string, choices, scorer=fuzz.ratio, score_cutoff=90) + result = process.extractBests(string, choices, scorer=fuzz.ratio, score_cutoff=80) return HelpQueryNotFound(f'Query "{string}" not found.', dict(result)) @@ -214,10 +213,8 @@ class CustomHelpCommand(HelpCommand): """Formats the prefix, command name and signature, and short doc for an iterable of commands.""" details = "" for c in commands_: - if c.signature: - details += f"\n**`{PREFIX}{c.qualified_name} {c.signature}`**\n*{c.short_doc or 'No details provided'}*" - else: - details += f"\n**`{PREFIX}{c.qualified_name}`**\n*{c.short_doc or 'No details provided.'}*" + signature = f" {c.signature}" if c.signature else "" + details += f"\n**`{PREFIX}{c.qualified_name}{signature}`**\n*{c.short_doc or 'No details provided'}*" return details -- cgit v1.2.3 From 72972704559fb9c5b86b6af32cb0d1799be4e563 Mon Sep 17 00:00:00 2001 From: mathsman5133 Date: Tue, 31 Mar 2020 20:21:22 +1100 Subject: Fix linting? Not sure why my precommit didn't pick that up... --- bot/cogs/help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/help.py b/bot/cogs/help.py index f020a1725..74c4cc664 100644 --- a/bot/cogs/help.py +++ b/bot/cogs/help.py @@ -5,7 +5,7 @@ from collections import namedtuple from contextlib import suppress from typing import List -from discord import Colour, Embed, NotFound, Member, Message, Reaction, User +from discord import Colour, Embed, Member, Message, NotFound, Reaction, User from discord.ext.commands import Bot, Cog, Command, Context, Group, HelpCommand from fuzzywuzzy import fuzz, process -- cgit v1.2.3 From 2559bc90517bfbd76f1a57a7a6c0dc75660c1985 Mon Sep 17 00:00:00 2001 From: decorator-factory <42166884+decorator-factory@users.noreply.github.com> Date: Wed, 1 Apr 2020 00:28:43 +0300 Subject: Add mutability.md tag --- bot/resources/tags/mutability.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 bot/resources/tags/mutability.md diff --git a/bot/resources/tags/mutability.md b/bot/resources/tags/mutability.md new file mode 100644 index 000000000..08e28b370 --- /dev/null +++ b/bot/resources/tags/mutability.md @@ -0,0 +1,34 @@ +# Mutable vs immutable objects + +Imagine that you want to make all letters in a string upper case. +Conveniently, strings have an `.upper()` method. + +You might think that this would work: +```python +string = "abcd" +string.upper() +print(string) # abcd +``` + +`string` didn't change. Why is that so? + +That's because strings in Python are _immutable_. You can't change them, you can only pass +around existing strings or create new ones. + +```python +string = "abcd" +string = string.upper() +``` +`string.upper()` creates a new string which is like the old one, but with all +the letters turned to upper case. + +`int`, `float`, `complex`, `tuple`, `frozenset` are other examples of immutable data types in Python. + +Mutable data types like `list`, on the other hand, can be changed in-place: +```python +my_list = [1, 2, 3] +my_list.append(4) +print(my_list) # [1, 2, 3, 4] +``` + +`dict` and `set` are other examples of mutable data types in Python. -- cgit v1.2.3 From 028781aa7cb4bce036c4cc49c636e8adf3bac0df Mon Sep 17 00:00:00 2001 From: decorator-factory <42166884+decorator-factory@users.noreply.github.com> Date: Wed, 1 Apr 2020 00:33:06 +0300 Subject: header->bold in mutability.md --- bot/resources/tags/mutability.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/resources/tags/mutability.md b/bot/resources/tags/mutability.md index 08e28b370..fc9e5374d 100644 --- a/bot/resources/tags/mutability.md +++ b/bot/resources/tags/mutability.md @@ -1,4 +1,4 @@ -# Mutable vs immutable objects +**Mutable vs immutable objects** Imagine that you want to make all letters in a string upper case. Conveniently, strings have an `.upper()` method. -- cgit v1.2.3 From a9468420065a523dc3d9d44fd4bbebefac82544f Mon Sep 17 00:00:00 2001 From: decorator-factory <42166884+decorator-factory@users.noreply.github.com> Date: Wed, 1 Apr 2020 12:54:13 +0300 Subject: Fix hard-wrapping in mutability.md --- bot/resources/tags/mutability.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/bot/resources/tags/mutability.md b/bot/resources/tags/mutability.md index fc9e5374d..48e5bac74 100644 --- a/bot/resources/tags/mutability.md +++ b/bot/resources/tags/mutability.md @@ -1,7 +1,6 @@ **Mutable vs immutable objects** -Imagine that you want to make all letters in a string upper case. -Conveniently, strings have an `.upper()` method. +Imagine that you want to make all letters in a string upper case. Conveniently, strings have an `.upper()` method. You might think that this would work: ```python @@ -12,17 +11,16 @@ print(string) # abcd `string` didn't change. Why is that so? -That's because strings in Python are _immutable_. You can't change them, you can only pass -around existing strings or create new ones. +That's because strings in Python are _immutable_. You can't change them, you can only pass around existing strings or create new oness. ```python string = "abcd" string = string.upper() ``` -`string.upper()` creates a new string which is like the old one, but with all -the letters turned to upper case. -`int`, `float`, `complex`, `tuple`, `frozenset` are other examples of immutable data types in Python. +`string.upper()` creates a new string which is like the old one, but with allthe letters turned to upper case. + +`int`, `float`, `complex`, `tuple`, `frozenset` are other examples of immutable data types in Python. Mutable data types like `list`, on the other hand, can be changed in-place: ```python -- cgit v1.2.3 From 7a02ea8d35628e92ede20f6f1b93418a105040c2 Mon Sep 17 00:00:00 2001 From: Mark Date: Wed, 1 Apr 2020 10:46:33 -0700 Subject: Help: lower score cutoff for fuzzy match --- bot/cogs/help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/help.py b/bot/cogs/help.py index 74c4cc664..6fc4d83f8 100644 --- a/bot/cogs/help.py +++ b/bot/cogs/help.py @@ -152,7 +152,7 @@ class CustomHelpCommand(HelpCommand): Will return an instance of the `HelpQueryNotFound` exception with the error message and possible matches. """ choices = await self.get_all_help_choices() - result = process.extractBests(string, choices, scorer=fuzz.ratio, score_cutoff=80) + result = process.extractBests(string, choices, scorer=fuzz.ratio, score_cutoff=60) return HelpQueryNotFound(f'Query "{string}" not found.', dict(result)) -- cgit v1.2.3 From ce5fcdab852600342fe69211b038426ce2821107 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 4 Apr 2020 14:51:13 +0300 Subject: (Banning): Added logging and truncating to correct length for Discord Audit Log when ban reason length is more than 512 characters. --- bot/cogs/moderation/infractions.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index efa19f59e..f41484711 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -244,7 +244,14 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - action = ctx.guild.ban(user, reason=reason, delete_message_days=0) + if len(reason) > 512: + log.info("Ban reason is longer than 512 characters. Reason will be truncated for Audit Log.") + + action = ctx.guild.ban( + user, + reason=f"{reason[:509]}..." if len(reason) > 512 else reason, + delete_message_days=0 + ) await self.apply_infraction(ctx, infraction, user, action) if infraction.get('expires_at') is not None: -- cgit v1.2.3 From 6ad86ace9f34245180aefeed9553c407602602e8 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 4 Apr 2020 14:53:11 +0300 Subject: (Kick Command): Added logging and truncating to correct length for Discord Audit Log when kick reason length is more than 512 characters. --- bot/cogs/moderation/infractions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index f41484711..f8c3e8da3 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -225,7 +225,10 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - action = user.kick(reason=reason) + if len(reason) > 512: + log.info("Kick reason is longer than 512 characters. Reason will be truncated for Audit Log.") + + action = user.kick(reason=f"{reason[:509]}..." if len(reason) > 512 else reason) await self.apply_infraction(ctx, infraction, user, action) @respect_role_hierarchy() -- cgit v1.2.3 From 80e483a7ebaf57e6544429343c94cf0eed8821ef Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 4 Apr 2020 16:03:58 +0300 Subject: (Ban and Kick): Replaced force reason truncating with `textwrap.shorten`. --- bot/cogs/moderation/infractions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index f8c3e8da3..a0bdf0d97 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -1,4 +1,5 @@ import logging +import textwrap import typing as t import discord @@ -228,7 +229,7 @@ class Infractions(InfractionScheduler, commands.Cog): if len(reason) > 512: log.info("Kick reason is longer than 512 characters. Reason will be truncated for Audit Log.") - action = user.kick(reason=f"{reason[:509]}..." if len(reason) > 512 else reason) + action = user.kick(textwrap.shorten(reason, width=509, placeholder="...") if len(reason) > 512 else reason) await self.apply_infraction(ctx, infraction, user, action) @respect_role_hierarchy() @@ -252,7 +253,7 @@ class Infractions(InfractionScheduler, commands.Cog): action = ctx.guild.ban( user, - reason=f"{reason[:509]}..." if len(reason) > 512 else reason, + reason=textwrap.shorten(reason, width=509, placeholder="...") if len(reason) > 512 else reason, delete_message_days=0 ) await self.apply_infraction(ctx, infraction, user, action) -- cgit v1.2.3 From ec8cc8b02b0823deaa4ea2c97801d16d6aef5244 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sun, 5 Apr 2020 18:13:38 +0300 Subject: (Ban and Kick): Applied simplification to reason truncating. --- bot/cogs/moderation/infractions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index a0bdf0d97..5bdea5755 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -229,7 +229,7 @@ class Infractions(InfractionScheduler, commands.Cog): if len(reason) > 512: log.info("Kick reason is longer than 512 characters. Reason will be truncated for Audit Log.") - action = user.kick(textwrap.shorten(reason, width=509, placeholder="...") if len(reason) > 512 else reason) + action = user.kick(reason=textwrap.shorten(reason, width=509, placeholder="...")) await self.apply_infraction(ctx, infraction, user, action) @respect_role_hierarchy() @@ -253,7 +253,7 @@ class Infractions(InfractionScheduler, commands.Cog): action = ctx.guild.ban( user, - reason=textwrap.shorten(reason, width=509, placeholder="...") if len(reason) > 512 else reason, + reason=textwrap.shorten(reason, width=509, placeholder="..."), delete_message_days=0 ) await self.apply_infraction(ctx, infraction, user, action) -- cgit v1.2.3 From aad67373d3ac6d3d3a236d6734a6c22019dce120 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sun, 5 Apr 2020 18:17:16 +0300 Subject: (Mod Scheduler): Added reason truncations to Scheduler's `apply_infraction` --- bot/cogs/moderation/scheduler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 917697be9..45e9d58ad 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -84,7 +84,8 @@ class InfractionScheduler(Scheduler): """Apply an infraction to the user, log the infraction, and optionally notify the user.""" infr_type = infraction["type"] icon = utils.INFRACTION_ICONS[infr_type][0] - reason = infraction["reason"] + # Truncate reason when it's too long to avoid raising error on sending ModLog entry + reason = textwrap.shorten(infraction["reason"], width=1900, placeholder="...") expiry = time.format_infraction_with_duration(infraction["expires_at"]) id_ = infraction['id'] -- cgit v1.2.3 From 72768b432b07acd3b1bfd5533c55241126329886 Mon Sep 17 00:00:00 2001 From: rohanjnr Date: Sun, 5 Apr 2020 21:22:28 +0530 Subject: Add feature to restrict tags to specific role(s) --- bot/cogs/tags.py | 52 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index 539105017..9c897ad36 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -4,7 +4,7 @@ import time from pathlib import Path from typing import Callable, Dict, Iterable, List, Optional -from discord import Colour, Embed +from discord import Colour, Embed, Member from discord.ext.commands import Cog, Context, group from bot import constants @@ -36,18 +36,32 @@ class Tags(Cog): """Get all tags.""" # Save all tags in memory. cache = {} - tag_files = Path("bot", "resources", "tags").iterdir() + tag_files = Path("bot", "resources", "tags").glob("**/*") for file in tag_files: - tag_title = file.stem - tag = { - "title": tag_title, - "embed": { - "description": file.read_text() + file_path = str(file).split("/") + if file.is_file(): + tag_title = file.stem + tag = { + "title": tag_title, + "embed": { + "description": file.read_text() + }, + "restricted_to": "developers" } - } - cache[tag_title] = tag + if len(file_path) == 5: + restricted_to = file_path[3] + tag["restricted_to"] = restricted_to + + cache[tag_title] = tag return cache + @staticmethod + def check_accessibility(user: Member, tag: dict) -> bool: + """Check if user can access a tag.""" + if tag["restricted_to"].lower() in [role.name.lower() for role in user.roles]: + return True + return False + @staticmethod def _fuzzy_search(search: str, target: str) -> float: """A simple scoring algorithm based on how many letters are found / total, with order in mind.""" @@ -92,7 +106,7 @@ class Tags(Cog): return self._get_suggestions(tag_name) return found - def _get_tags_via_content(self, check: Callable[[Iterable], bool], keywords: str) -> list: + def _get_tags_via_content(self, check: Callable[[Iterable], bool], keywords: str, user: Member) -> list: """ Search for tags via contents. @@ -113,8 +127,9 @@ class Tags(Cog): matching_tags = [] for tag in self._cache.values(): - if check(query in tag['embed']['description'].casefold() for query in keywords_processed): - matching_tags.append(tag) + if self.check_accessibility(user, tag): + if check(query in tag['embed']['description'].casefold() for query in keywords_processed): + matching_tags.append(tag) return matching_tags @@ -151,7 +166,7 @@ class Tags(Cog): Only search for tags that has ALL the keywords. """ - matching_tags = self._get_tags_via_content(all, keywords) + matching_tags = self._get_tags_via_content(all, keywords, ctx.author) await self._send_matching_tags(ctx, keywords, matching_tags) @search_tag_content.command(name='any') @@ -161,7 +176,7 @@ class Tags(Cog): Search for tags that has ANY of the keywords. """ - matching_tags = self._get_tags_via_content(any, keywords or 'any') + matching_tags = self._get_tags_via_content(any, keywords or 'any', ctx.author) await self._send_matching_tags(ctx, keywords, matching_tags) @tags_group.command(name='get', aliases=('show', 'g')) @@ -198,6 +213,10 @@ class Tags(Cog): if tag_name is not None: founds = self._get_tag(tag_name) + for found_tag in founds: + if not self.check_accessibility(ctx.author, found_tag): + founds.remove(found_tag) + if len(founds) == 1: tag = founds[0] if ctx.channel.id not in TEST_CHANNELS: @@ -222,7 +241,10 @@ class Tags(Cog): else: embed: Embed = Embed(title="**Current tags**") await LinePaginator.paginate( - sorted(f"**»** {tag['title']}" for tag in tags), + sorted( + f"**»** {tag['title']}" for tag in tags + if self.check_accessibility(ctx.author, tag) + ), ctx, embed, footer_text=FOOTER_TEXT, -- cgit v1.2.3 From 00d22a316041a8670903eb5fd4b4a7d143993330 Mon Sep 17 00:00:00 2001 From: rohanjnr Date: Sun, 5 Apr 2020 22:04:51 +0530 Subject: Remove unnecessary variable creation and join two if statements --- bot/cogs/tags.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index 9c897ad36..bb74ab1ca 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -49,8 +49,7 @@ class Tags(Cog): "restricted_to": "developers" } if len(file_path) == 5: - restricted_to = file_path[3] - tag["restricted_to"] = restricted_to + tag["restricted_to"] = file_path[3] cache[tag_title] = tag return cache @@ -127,9 +126,11 @@ class Tags(Cog): matching_tags = [] for tag in self._cache.values(): - if self.check_accessibility(user, tag): - if check(query in tag['embed']['description'].casefold() for query in keywords_processed): - matching_tags.append(tag) + if ( + self.check_accessibility(user, tag) + and check(query in tag['embed']['description'].casefold() for query in keywords_processed) + ): + matching_tags.append(tag) return matching_tags -- cgit v1.2.3 From 025857541b7a0cbb77adf2a0282873c9e116169a Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 6 Apr 2020 08:48:32 +0300 Subject: (Ban and Kick): Changed length in `textwrap.shorten` from 309 to 312 because shorten already include `placeholder` to length. --- bot/cogs/moderation/infractions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 5bdea5755..7a044fc1c 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -229,7 +229,7 @@ class Infractions(InfractionScheduler, commands.Cog): if len(reason) > 512: log.info("Kick reason is longer than 512 characters. Reason will be truncated for Audit Log.") - action = user.kick(reason=textwrap.shorten(reason, width=509, placeholder="...")) + action = user.kick(reason=textwrap.shorten(reason, width=512, placeholder="...")) await self.apply_infraction(ctx, infraction, user, action) @respect_role_hierarchy() @@ -253,7 +253,7 @@ class Infractions(InfractionScheduler, commands.Cog): action = ctx.guild.ban( user, - reason=textwrap.shorten(reason, width=509, placeholder="..."), + reason=textwrap.shorten(reason, width=512, placeholder="..."), delete_message_days=0 ) await self.apply_infraction(ctx, infraction, user, action) -- cgit v1.2.3 From c4f7359d2e301e6ab19666a6867c9cb69892da0b Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 6 Apr 2020 08:50:46 +0300 Subject: (Ban and Kick): Added space to `textwrap.shorten` `placeholder`. --- bot/cogs/moderation/infractions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 7a044fc1c..2c809535b 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -253,7 +253,7 @@ class Infractions(InfractionScheduler, commands.Cog): action = ctx.guild.ban( user, - reason=textwrap.shorten(reason, width=512, placeholder="..."), + reason=textwrap.shorten(reason, width=512, placeholder=" ..."), delete_message_days=0 ) await self.apply_infraction(ctx, infraction, user, action) -- cgit v1.2.3 From 482c3f4b475cdbe16b377dd5bb85910be0387166 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 09:04:15 +0300 Subject: (Mod Utils): Added shortening reason on embed creation in `notify_infraction`. --- bot/cogs/moderation/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py index 3598f3b1f..9811d059f 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/cogs/moderation/utils.py @@ -135,7 +135,7 @@ async def notify_infraction( description=textwrap.dedent(f""" **Type:** {infr_type.capitalize()} **Expires:** {expires_at or "N/A"} - **Reason:** {reason or "No reason provided."} + **Reason:** {textwrap.shorten(reason, width=1500, placeholder="...") or "No reason provided."} """), colour=Colours.soft_red ) -- cgit v1.2.3 From a59092659271832a46c8ab0166031bffdc68c0a6 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 09:06:17 +0300 Subject: (Infractions): Removed unnecessary logging that notify when reason will be truncated for Audit Log. --- bot/cogs/moderation/infractions.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 2c809535b..d1e77311c 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -226,9 +226,6 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - if len(reason) > 512: - log.info("Kick reason is longer than 512 characters. Reason will be truncated for Audit Log.") - action = user.kick(reason=textwrap.shorten(reason, width=512, placeholder="...")) await self.apply_infraction(ctx, infraction, user, action) @@ -248,9 +245,6 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - if len(reason) > 512: - log.info("Ban reason is longer than 512 characters. Reason will be truncated for Audit Log.") - action = ctx.guild.ban( user, reason=textwrap.shorten(reason, width=512, placeholder=" ..."), -- cgit v1.2.3 From 42e18061a21e0ec1b8a4a692bc8d96f8ef1fd45b Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 09:09:10 +0300 Subject: (Infractions): Moved truncated reason to variable instead on ban coroutine creating. --- bot/cogs/moderation/infractions.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index d1e77311c..3340744b0 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -245,11 +245,9 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - action = ctx.guild.ban( - user, - reason=textwrap.shorten(reason, width=512, placeholder=" ..."), - delete_message_days=0 - ) + truncated_reason = textwrap.shorten(reason, width=512, placeholder=" ...") + + action = ctx.guild.ban(user, reason=truncated_reason, delete_message_days=0) await self.apply_infraction(ctx, infraction, user, action) if infraction.get('expires_at') is not None: -- cgit v1.2.3 From 10ea74bc5ed390c36d64a0f7413b8422f158708a Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 09:16:36 +0300 Subject: (Superstarify, Scheduler): Added reason shortening for ModLog. --- bot/cogs/moderation/scheduler.py | 2 +- bot/cogs/moderation/superstarify.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 45e9d58ad..7404ec8ac 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -326,7 +326,7 @@ class InfractionScheduler(Scheduler): log_text = { "Member": f"<@{user_id}>", "Actor": str(self.bot.get_user(actor) or actor), - "Reason": infraction["reason"], + "Reason": textwrap.shorten(infraction["reason"], width=1500, placeholder="..."), "Created": created, } diff --git a/bot/cogs/moderation/superstarify.py b/bot/cogs/moderation/superstarify.py index ca3dc4202..d77e61e6b 100644 --- a/bot/cogs/moderation/superstarify.py +++ b/bot/cogs/moderation/superstarify.py @@ -183,7 +183,7 @@ class Superstarify(InfractionScheduler, Cog): text=textwrap.dedent(f""" Member: {member.mention} (`{member.id}`) Actor: {ctx.message.author} - Reason: {reason} + Reason: {textwrap.shorten(reason, width=1500, placeholder="...")} Expires: {expiry_str} Old nickname: `{old_nick}` New nickname: `{forced_nick}` -- cgit v1.2.3 From 27e15c42e71f3d2df828d13a5e92c53c664b4431 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 09:20:04 +0300 Subject: (Scheduler): Changed reason truncating in `apply_infraction` from 1900 chars to 1500, added shortening to end message too. --- bot/cogs/moderation/scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 7404ec8ac..345f08f19 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -85,7 +85,7 @@ class InfractionScheduler(Scheduler): infr_type = infraction["type"] icon = utils.INFRACTION_ICONS[infr_type][0] # Truncate reason when it's too long to avoid raising error on sending ModLog entry - reason = textwrap.shorten(infraction["reason"], width=1900, placeholder="...") + reason = textwrap.shorten(infraction["reason"], width=1500, placeholder="...") expiry = time.format_infraction_with_duration(infraction["expires_at"]) id_ = infraction['id'] @@ -128,7 +128,7 @@ class InfractionScheduler(Scheduler): f"Infraction #{id_} actor is bot; including the reason in the confirmation message." ) - end_msg = f" (reason: {infraction['reason']})" + end_msg = f" (reason: {textwrap.shorten(infraction['reason'], width=1500, placeholder='...')})" elif ctx.channel.id not in STAFF_CHANNELS: log.trace( f"Infraction #{id_} context is not in a staff channel; omitting infraction count." -- cgit v1.2.3 From 1100dba71b789bdc35c6c42d1b4003c7d28dcbb0 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 15:10:40 +0300 Subject: (ModLog): Added mod log item embed description truncating when it's too long. --- bot/cogs/moderation/modlog.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index c63b4bab9..e15a80c6d 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -2,6 +2,7 @@ import asyncio import difflib import itertools import logging +import textwrap import typing as t from datetime import datetime from itertools import zip_longest @@ -98,7 +99,7 @@ class ModLog(Cog, name="ModLog"): footer: t.Optional[str] = None, ) -> Context: """Generate log embed and send to logging channel.""" - embed = discord.Embed(description=text) + embed = discord.Embed(description=textwrap.shorten(text, width=2048, placeholder="...")) if title and icon_url: embed.set_author(name=title, icon_url=icon_url) -- cgit v1.2.3 From b765ecb280a3ba0ca017350a4c69dc9c07f97a67 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 15:12:56 +0300 Subject: (Scheduler): Removed reason truncation from `apply_infraction`, changed order of ModLog embed description item in same function. --- bot/cogs/moderation/scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 345f08f19..fbb2d457b 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -85,7 +85,7 @@ class InfractionScheduler(Scheduler): infr_type = infraction["type"] icon = utils.INFRACTION_ICONS[infr_type][0] # Truncate reason when it's too long to avoid raising error on sending ModLog entry - reason = textwrap.shorten(infraction["reason"], width=1500, placeholder="...") + reason = infraction["reason"] expiry = time.format_infraction_with_duration(infraction["expires_at"]) id_ = infraction['id'] @@ -182,8 +182,8 @@ class InfractionScheduler(Scheduler): text=textwrap.dedent(f""" Member: {user.mention} (`{user.id}`) Actor: {ctx.message.author}{dm_log_text} - Reason: {reason} {expiry_log_text} + Reason: {reason} """), content=log_content, footer=f"ID {infraction['id']}" -- cgit v1.2.3 From 03028eab84a09c047c0ef879fc06fccacbe30420 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 15:18:38 +0300 Subject: (Mod Utils): Removed truncation of reason itself and added truncation to whole embed in `notify_infraction`. --- bot/cogs/moderation/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py index 9811d059f..0423e5373 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/cogs/moderation/utils.py @@ -132,11 +132,11 @@ async def notify_infraction( log.trace(f"Sending {user} a DM about their {infr_type} infraction.") embed = discord.Embed( - description=textwrap.dedent(f""" + description=textwrap.shorten(textwrap.dedent(f""" **Type:** {infr_type.capitalize()} **Expires:** {expires_at or "N/A"} - **Reason:** {textwrap.shorten(reason, width=1500, placeholder="...") or "No reason provided."} - """), + **Reason:** {reason or "No reason provided."} + """), width=2048, placeholder="..."), colour=Colours.soft_red ) -- cgit v1.2.3 From 36f947c1cb52faedc1d2e00869e109f2cde12c11 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 15:19:44 +0300 Subject: (Superstarify): Removed unnecessary truncation on `superstarify` command, reordered ModLog text. --- bot/cogs/moderation/superstarify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/superstarify.py b/bot/cogs/moderation/superstarify.py index d77e61e6b..e221ad909 100644 --- a/bot/cogs/moderation/superstarify.py +++ b/bot/cogs/moderation/superstarify.py @@ -183,10 +183,10 @@ class Superstarify(InfractionScheduler, Cog): text=textwrap.dedent(f""" Member: {member.mention} (`{member.id}`) Actor: {ctx.message.author} - Reason: {textwrap.shorten(reason, width=1500, placeholder="...")} Expires: {expiry_str} Old nickname: `{old_nick}` New nickname: `{forced_nick}` + Reason: {reason} """), footer=f"ID {id_}" ) -- cgit v1.2.3 From 8d283fb8eefd7be288702f88653aaebcdcda37c3 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 08:12:09 +0300 Subject: (Mod Utils): Moved embed description to variable. --- bot/cogs/moderation/utils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py index 0423e5373..fc8c26031 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/cogs/moderation/utils.py @@ -131,12 +131,14 @@ async def notify_infraction( """DM a user about their new infraction and return True if the DM is successful.""" log.trace(f"Sending {user} a DM about their {infr_type} infraction.") + text = textwrap.dedent(f""" + **Type:** {infr_type.capitalize()} + **Expires:** {expires_at or "N/A"} + **Reason:** {reason or "No reason provided."} + """) + embed = discord.Embed( - description=textwrap.shorten(textwrap.dedent(f""" - **Type:** {infr_type.capitalize()} - **Expires:** {expires_at or "N/A"} - **Reason:** {reason or "No reason provided."} - """), width=2048, placeholder="..."), + description=textwrap.shorten(text, width=2048, placeholder="..."), colour=Colours.soft_red ) -- cgit v1.2.3 From b148beeec8c897d91fa100d0bbd1cb4965f58e6e Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 08:27:10 +0300 Subject: (Scheduler): Move reason to end of log text to avoid truncating keys. --- bot/cogs/moderation/scheduler.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index fbb2d457b..3352806e7 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -283,6 +283,9 @@ class InfractionScheduler(Scheduler): f"{log_text.get('Failure', '')}" ) + # Move reason to end of entry to avoid cutting out some keys + log_text["Reason"] = log_text.pop("Reason") + # Send a log message to the mod log. await self.mod_log.send_log_message( icon_url=utils.INFRACTION_ICONS[infr_type][1], @@ -326,7 +329,7 @@ class InfractionScheduler(Scheduler): log_text = { "Member": f"<@{user_id}>", "Actor": str(self.bot.get_user(actor) or actor), - "Reason": textwrap.shorten(infraction["reason"], width=1500, placeholder="..."), + "Reason": infraction["reason"], "Created": created, } @@ -396,6 +399,9 @@ class InfractionScheduler(Scheduler): user = self.bot.get_user(user_id) avatar = user.avatar_url_as(static_format="png") if user else None + # Move reason to end so when reason is too long, this is not gonna cut out required items. + log_text["Reason"] = log_text.pop("Reason") + log.trace(f"Sending deactivation mod log for infraction #{id_}.") await self.mod_log.send_log_message( icon_url=utils.INFRACTION_ICONS[type_][1], @@ -405,7 +411,6 @@ class InfractionScheduler(Scheduler): text="\n".join(f"{k}: {v}" for k, v in log_text.items()), footer=f"ID: {id_}", content=log_content, - ) return log_text -- cgit v1.2.3 From 816e76fe8c5e5231cdc85ab974294c1d2fb4a87c Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 08:28:44 +0300 Subject: (Scheduler): Replaced `infraction['reason']` with `reason` variable using in `end_msg`. --- bot/cogs/moderation/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 3352806e7..b238cf4e2 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -128,7 +128,7 @@ class InfractionScheduler(Scheduler): f"Infraction #{id_} actor is bot; including the reason in the confirmation message." ) - end_msg = f" (reason: {textwrap.shorten(infraction['reason'], width=1500, placeholder='...')})" + end_msg = f" (reason: {textwrap.shorten(reason, width=1500, placeholder='...')})" elif ctx.channel.id not in STAFF_CHANNELS: log.trace( f"Infraction #{id_} context is not in a staff channel; omitting infraction count." -- cgit v1.2.3 From 3bbd10ac91e9677e24588734d256a0558c0b46a2 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 09:52:19 +0300 Subject: (Talent Pool): Applied reason shortening. --- bot/cogs/watchchannels/talentpool.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/cogs/watchchannels/talentpool.py b/bot/cogs/watchchannels/talentpool.py index ad0c51fa6..15af7e34d 100644 --- a/bot/cogs/watchchannels/talentpool.py +++ b/bot/cogs/watchchannels/talentpool.py @@ -106,8 +106,8 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): if history: total = f"({len(history)} previous nominations in total)" - start_reason = f"Watched: {history[0]['reason']}" - end_reason = f"Unwatched: {history[0]['end_reason']}" + start_reason = f"Watched: {textwrap.shorten(history[0]['reason'], width=500, placeholder='...')}" + end_reason = f"Unwatched: {textwrap.shorten(history[0]['end_reason'], width=500, placeholder='...')}" msg += f"\n\nUser's previous watch reasons {total}:```{start_reason}\n\n{end_reason}```" await ctx.send(msg) @@ -224,7 +224,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): Status: **Active** Date: {start_date} Actor: {actor.mention if actor else actor_id} - Reason: {nomination_object["reason"]} + Reason: {textwrap.shorten(nomination_object["reason"], width=200, placeholder="...")} Nomination ID: `{nomination_object["id"]}` =============== """ @@ -237,10 +237,10 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): Status: Inactive Date: {start_date} Actor: {actor.mention if actor else actor_id} - Reason: {nomination_object["reason"]} + Reason: {textwrap.shorten(nomination_object["reason"], width=200, placeholder="...")} End date: {end_date} - Unwatch reason: {nomination_object["end_reason"]} + Unwatch reason: {textwrap.shorten(nomination_object["end_reason"], width=200, placeholder="...")} Nomination ID: `{nomination_object["id"]}` =============== """ -- cgit v1.2.3 From 2c9bc9f6fe5174096a6177560acd91f869c296ef Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 11:30:51 +0300 Subject: (Watchchannel): Added footer shortening. --- bot/cogs/watchchannels/watchchannel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/watchchannels/watchchannel.py b/bot/cogs/watchchannels/watchchannel.py index 479820444..ac1aa38ee 100644 --- a/bot/cogs/watchchannels/watchchannel.py +++ b/bot/cogs/watchchannels/watchchannel.py @@ -280,8 +280,9 @@ class WatchChannel(metaclass=CogABCMeta): else: message_jump = f"in [#{msg.channel.name}]({msg.jump_url})" + footer = f"Added {time_delta} by {actor} | Reason: {reason}" embed = Embed(description=f"{msg.author.mention} {message_jump}") - embed.set_footer(text=f"Added {time_delta} by {actor} | Reason: {reason}") + embed.set_footer(text=textwrap.shorten(footer, width=128, placeholder="...")) await self.webhook_send(embed=embed, username=msg.author.display_name, avatar_url=msg.author.avatar_url) -- cgit v1.2.3 From 472b58b3fb1b3d8695d9de1a13ce92f129e6bcc4 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 11:34:54 +0300 Subject: (Big Brother): Added truncating reason. --- bot/cogs/watchchannels/bigbrother.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bot/cogs/watchchannels/bigbrother.py b/bot/cogs/watchchannels/bigbrother.py index 903c87f85..69df849f0 100644 --- a/bot/cogs/watchchannels/bigbrother.py +++ b/bot/cogs/watchchannels/bigbrother.py @@ -1,4 +1,5 @@ import logging +import textwrap from collections import ChainMap from discord.ext.commands import Cog, Context, group @@ -97,8 +98,8 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): if len(history) > 1: total = f"({len(history) // 2} previous infractions in total)" - end_reason = history[0]["reason"] - start_reason = f"Watched: {history[1]['reason']}" + end_reason = textwrap.shorten(history[0]["reason"], width=500, placeholder="...") + start_reason = f"Watched: {textwrap.shorten(history[1]['reason'], width=500, placeholder='...')}" msg += f"\n\nUser's previous watch reasons {total}:```{start_reason}\n\n{end_reason}```" else: msg = ":x: Failed to post the infraction: response was empty." -- cgit v1.2.3 From 7195a928952c1550a82d27706a57165606aa5f4a Mon Sep 17 00:00:00 2001 From: rohanjnr Date: Tue, 14 Apr 2020 18:57:39 +0530 Subject: Feature now is cross-platform Instead of using string methods to split the file path at `/` which is not cross-platform, I am now entirely using pathlib methods to get the parent folder and restrict the tags. --- bot/cogs/tags.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index bb74ab1ca..a59d28600 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -38,7 +38,7 @@ class Tags(Cog): cache = {} tag_files = Path("bot", "resources", "tags").glob("**/*") for file in tag_files: - file_path = str(file).split("/") + if file.is_file(): tag_title = file.stem tag = { @@ -48,8 +48,9 @@ class Tags(Cog): }, "restricted_to": "developers" } - if len(file_path) == 5: - tag["restricted_to"] = file_path[3] + parent_folder = file.parent.stem + if parent_folder != "tags": + tag["restricted_to"] = parent_folder cache[tag_title] = tag return cache @@ -212,11 +213,13 @@ class Tags(Cog): return if tag_name is not None: - founds = self._get_tag(tag_name) + temp_founds = self._get_tag(tag_name) + + founds = [] - for found_tag in founds: - if not self.check_accessibility(ctx.author, found_tag): - founds.remove(found_tag) + for found_tag in temp_founds: + if self.check_accessibility(ctx.author, found_tag): + founds.append(found_tag) if len(founds) == 1: tag = founds[0] -- cgit v1.2.3 From 39bac8873120801eb51a2c1a996d2760d9af64a4 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 14 Apr 2020 16:32:43 +0300 Subject: (ModLog): Applied force embed description truncating in `send_log_message` to avoid removing newlines. --- bot/cogs/moderation/modlog.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index e15a80c6d..fcc9d4e0a 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -99,7 +99,10 @@ class ModLog(Cog, name="ModLog"): footer: t.Optional[str] = None, ) -> Context: """Generate log embed and send to logging channel.""" - embed = discord.Embed(description=textwrap.shorten(text, width=2048, placeholder="...")) + # Truncate string directly here to avoid removing newlines + embed = discord.Embed( + description=text[:2046] + "..." if len(text) > 2048 else text + ) if title and icon_url: embed.set_author(name=title, icon_url=icon_url) -- cgit v1.2.3 From 0b719040e8eb52fd030322b720de5f5ffbcc9919 Mon Sep 17 00:00:00 2001 From: Rohan Reddy Alleti Date: Tue, 14 Apr 2020 19:02:58 +0530 Subject: simplify if statement Co-Authored-By: Mark --- bot/cogs/tags.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index a59d28600..79bea3b63 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -127,10 +127,8 @@ class Tags(Cog): matching_tags = [] for tag in self._cache.values(): - if ( - self.check_accessibility(user, tag) - and check(query in tag['embed']['description'].casefold() for query in keywords_processed) - ): + matches = (query in tag['embed']['description'].casefold() for query in keywords_processed) + if self.check_accessibility(user, tag) and check(matches): matching_tags.append(tag) return matching_tags -- cgit v1.2.3 From 8bf1df9438ecf456b02725e3f9689c5bd885b2a7 Mon Sep 17 00:00:00 2001 From: Rohan Reddy Alleti Date: Tue, 14 Apr 2020 19:03:38 +0530 Subject: simpl Co-Authored-By: Mark --- bot/cogs/tags.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index 79bea3b63..e5492971d 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -58,9 +58,7 @@ class Tags(Cog): @staticmethod def check_accessibility(user: Member, tag: dict) -> bool: """Check if user can access a tag.""" - if tag["restricted_to"].lower() in [role.name.lower() for role in user.roles]: - return True - return False + return tag["restricted_to"].lower() in [role.name.lower() for role in user.roles] @staticmethod def _fuzzy_search(search: str, target: str) -> float: -- cgit v1.2.3 From 6928c492c1d989567c47dfd49a396730c6c8bb27 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 14 Apr 2020 18:29:55 +0300 Subject: (Scheduler): Removed empty line when expiration not specified in `apply_infraction`. --- bot/cogs/moderation/scheduler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index b238cf4e2..5b59b4d4b 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -102,7 +102,7 @@ class InfractionScheduler(Scheduler): dm_result = "" dm_log_text = "" - expiry_log_text = f"Expires: {expiry}" if expiry else "" + expiry_log_text = f"\nExpires: {expiry}" if expiry else "" log_title = "applied" log_content = None @@ -181,8 +181,7 @@ class InfractionScheduler(Scheduler): thumbnail=user.avatar_url_as(static_format="png"), text=textwrap.dedent(f""" Member: {user.mention} (`{user.id}`) - Actor: {ctx.message.author}{dm_log_text} - {expiry_log_text} + Actor: {ctx.message.author}{dm_log_text} {expiry_log_text} Reason: {reason} """), content=log_content, -- cgit v1.2.3 From 32a5bf97e31addb32d17bd0479bc4cf2f4dd9eb7 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 14 Apr 2020 18:50:24 +0300 Subject: (Scheduler): Added removal of infraction in DB, when applying infraction fail. Also don't send DM in this case. --- bot/cogs/moderation/scheduler.py | 46 ++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 5b59b4d4b..58e363da6 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -105,23 +105,7 @@ class InfractionScheduler(Scheduler): expiry_log_text = f"\nExpires: {expiry}" if expiry else "" log_title = "applied" log_content = None - - # DM the user about the infraction if it's not a shadow/hidden infraction. - if not infraction["hidden"]: - dm_result = f"{constants.Emojis.failmail} " - dm_log_text = "\nDM: **Failed**" - - # Sometimes user is a discord.Object; make it a proper user. - try: - if not isinstance(user, (discord.Member, discord.User)): - user = await self.bot.fetch_user(user.id) - except discord.HTTPException as e: - log.error(f"Failed to DM {user.id}: could not fetch user (status {e.status})") - else: - # Accordingly display whether the user was successfully notified via DM. - if await utils.notify_infraction(user, infr_type, expiry, reason, icon): - dm_result = ":incoming_envelope: " - dm_log_text = "\nDM: Sent" + failed = False if infraction["actor"] == self.bot.user.id: log.trace( @@ -165,11 +149,37 @@ class InfractionScheduler(Scheduler): log.warning(f"{log_msg}: bot lacks permissions.") else: log.exception(log_msg) + failed = True + + # DM the user about the infraction if it's not a shadow/hidden infraction. + # Don't send DM when applying failed. + if not infraction["hidden"] and not failed: + dm_result = f"{constants.Emojis.failmail} " + dm_log_text = "\nDM: **Failed**" + + # Sometimes user is a discord.Object; make it a proper user. + try: + if not isinstance(user, (discord.Member, discord.User)): + user = await self.bot.fetch_user(user.id) + except discord.HTTPException as e: + log.error(f"Failed to DM {user.id}: could not fetch user (status {e.status})") + else: + # Accordingly display whether the user was successfully notified via DM. + if await utils.notify_infraction(user, infr_type, expiry, reason, icon): + dm_result = ":incoming_envelope: " + dm_log_text = "\nDM: Sent" + + if failed: + dm_log_text = "\nDM: **Canceled**" + dm_result = f"{constants.Emojis.failmail} " + log.trace(f"Deleted infraction {infraction['id']} from database because applying infraction failed.") + await self.bot.api_client.delete(f"bot/infractions/{infraction['id']}") # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} confirmation message.") await ctx.send( - f"{dm_result}{confirm_msg} **{infr_type}** to {user.mention}{expiry_msg}{end_msg}." + f"{dm_result}{confirm_msg} " + f"{f'**{infr_type}** to {user.mention}{expiry_msg}{end_msg}' if not failed else ''}." ) # Send a log message to the mod log. -- cgit v1.2.3 From 085decd12867f89a0803806928741fe6dd3c76bb Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 15 Apr 2020 08:18:19 +0300 Subject: (Test Helpers): Added `__ge__` function to `MockRole` for comparing. --- tests/helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/helpers.py b/tests/helpers.py index 8e13f0f28..227bac95f 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -205,6 +205,10 @@ class MockRole(CustomMockMixin, unittest.mock.Mock, ColourMixin, HashableMixin): """Simplified position-based comparisons similar to those of `discord.Role`.""" return self.position < other.position + def __ge__(self, other): + """Simplified position-based comparisons similar to those of `discord.Role`.""" + return self.position >= other.position + # Create a Member instance to get a realistic Mock of `discord.Member` member_data = {'user': 'lemon', 'roles': [1]} -- cgit v1.2.3 From 81f6efc2f4e9e157e2f7fb9f191ea410af066632 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 16 Apr 2020 11:15:16 +0300 Subject: (Infraction Tests): Created reason shortening tests for ban and kick. --- tests/bot/cogs/moderation/test_infractions.py | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/bot/cogs/moderation/test_infractions.py diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py new file mode 100644 index 000000000..39ea93952 --- /dev/null +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -0,0 +1,54 @@ +import textwrap +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from bot.cogs.moderation.infractions import Infractions +from tests.helpers import MockBot, MockContext, MockGuild, MockMember, MockRole + + +class ShorteningTests(unittest.IsolatedAsyncioTestCase): + """Tests for ban and kick command reason shortening.""" + + def setUp(self): + self.bot = MockBot() + self.cog = Infractions(self.bot) + self.user = MockMember(id=1234, top_role=MockRole(id=3577, position=10)) + self.target = MockMember(id=1265, top_role=MockRole(id=9876, position=0)) + self.guild = MockGuild(id=4567) + self.ctx = MockContext(bot=self.bot, author=self.user, guild=self.guild) + + @patch("bot.cogs.moderation.utils.has_active_infraction") + @patch("bot.cogs.moderation.utils.post_infraction") + async def test_apply_ban_reason_shortening(self, post_infraction_mock, has_active_mock): + """Should truncate reason for `ctx.guild.ban`.""" + has_active_mock.return_value = False + post_infraction_mock.return_value = {"foo": "bar"} + + self.cog.apply_infraction = AsyncMock() + self.bot.get_cog.return_value = AsyncMock() + self.cog.mod_log.ignore = Mock() + + await self.cog.apply_ban(self.ctx, self.target, "foo bar" * 3000) + ban = self.cog.apply_infraction.call_args[0][3] + self.assertEqual( + ban.cr_frame.f_locals["kwargs"]["reason"], + textwrap.shorten("foo bar" * 3000, 512, placeholder=" ...") + ) + # Await ban to avoid warning + await ban + + @patch("bot.cogs.moderation.utils.post_infraction") + async def test_apply_kick_reason_shortening(self, post_infraction_mock) -> None: + """Should truncate reason for `Member.kick`.""" + post_infraction_mock.return_value = {"foo": "bar"} + + self.cog.apply_infraction = AsyncMock() + self.cog.mod_log.ignore = Mock() + + await self.cog.apply_kick(self.ctx, self.target, "foo bar" * 3000) + kick = self.cog.apply_infraction.call_args[0][3] + self.assertEqual( + kick.cr_frame.f_locals["kwargs"]["reason"], + textwrap.shorten("foo bar" * 3000, 512, placeholder="...") + ) + await kick -- cgit v1.2.3 From 216953044a870f2440fe44fcd2f9ca3ee7cf37e9 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 16 Apr 2020 11:30:09 +0300 Subject: (ModLog Tests): Created reason shortening tests for `send_log_message`. --- tests/bot/cogs/moderation/test_modlog.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tests/bot/cogs/moderation/test_modlog.py diff --git a/tests/bot/cogs/moderation/test_modlog.py b/tests/bot/cogs/moderation/test_modlog.py new file mode 100644 index 000000000..46e01d2ea --- /dev/null +++ b/tests/bot/cogs/moderation/test_modlog.py @@ -0,0 +1,29 @@ +import unittest + +import discord + +from bot.cogs.moderation.modlog import ModLog +from tests.helpers import MockBot, MockTextChannel + + +class ModLogTests(unittest.IsolatedAsyncioTestCase): + """Tests for moderation logs.""" + + def setUp(self): + self.bot = MockBot() + self.cog = ModLog(self.bot) + self.channel = MockTextChannel() + + async def test_log_entry_description_shortening(self): + """Should truncate embed description for ModLog entry.""" + self.bot.get_channel.return_value = self.channel + await self.cog.send_log_message( + icon_url="foo", + colour=discord.Colour.blue(), + title="bar", + text="foo bar" * 3000 + ) + embed = self.channel.send.call_args[1]["embed"] + self.assertEqual( + embed.description, ("foo bar" * 3000)[:2046] + "..." + ) -- cgit v1.2.3 From b463cb2d21683b7184698f788419d325e2f5f5cc Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 16 Apr 2020 11:33:21 +0300 Subject: (ModLog): Removed unused `textwrap` import. --- bot/cogs/moderation/modlog.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index fcc9d4e0a..eb8bd65cf 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -2,7 +2,6 @@ import asyncio import difflib import itertools import logging -import textwrap import typing as t from datetime import datetime from itertools import zip_longest -- cgit v1.2.3 From 1a3fa6a395141c4fcdd1d388d6ce3e7bd89bcbf0 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 16 Apr 2020 13:40:47 +0300 Subject: (Infractions and ModLog Tests): Replaced `shortening` with `truncation`, removed unnecessary type hint and added comment to kick truncation test about awaiting `kick`. --- tests/bot/cogs/moderation/test_infractions.py | 9 +++++---- tests/bot/cogs/moderation/test_modlog.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 39ea93952..51a8cc645 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -6,8 +6,8 @@ from bot.cogs.moderation.infractions import Infractions from tests.helpers import MockBot, MockContext, MockGuild, MockMember, MockRole -class ShorteningTests(unittest.IsolatedAsyncioTestCase): - """Tests for ban and kick command reason shortening.""" +class TruncationTests(unittest.IsolatedAsyncioTestCase): + """Tests for ban and kick command reason truncation.""" def setUp(self): self.bot = MockBot() @@ -19,7 +19,7 @@ class ShorteningTests(unittest.IsolatedAsyncioTestCase): @patch("bot.cogs.moderation.utils.has_active_infraction") @patch("bot.cogs.moderation.utils.post_infraction") - async def test_apply_ban_reason_shortening(self, post_infraction_mock, has_active_mock): + async def test_apply_ban_reason_truncation(self, post_infraction_mock, has_active_mock): """Should truncate reason for `ctx.guild.ban`.""" has_active_mock.return_value = False post_infraction_mock.return_value = {"foo": "bar"} @@ -38,7 +38,7 @@ class ShorteningTests(unittest.IsolatedAsyncioTestCase): await ban @patch("bot.cogs.moderation.utils.post_infraction") - async def test_apply_kick_reason_shortening(self, post_infraction_mock) -> None: + async def test_apply_kick_reason_truncation(self, post_infraction_mock): """Should truncate reason for `Member.kick`.""" post_infraction_mock.return_value = {"foo": "bar"} @@ -51,4 +51,5 @@ class ShorteningTests(unittest.IsolatedAsyncioTestCase): kick.cr_frame.f_locals["kwargs"]["reason"], textwrap.shorten("foo bar" * 3000, 512, placeholder="...") ) + # Await kick to avoid warning await kick diff --git a/tests/bot/cogs/moderation/test_modlog.py b/tests/bot/cogs/moderation/test_modlog.py index 46e01d2ea..d60836474 100644 --- a/tests/bot/cogs/moderation/test_modlog.py +++ b/tests/bot/cogs/moderation/test_modlog.py @@ -14,7 +14,7 @@ class ModLogTests(unittest.IsolatedAsyncioTestCase): self.cog = ModLog(self.bot) self.channel = MockTextChannel() - async def test_log_entry_description_shortening(self): + async def test_log_entry_description_truncation(self): """Should truncate embed description for ModLog entry.""" self.bot.get_channel.return_value = self.channel await self.cog.send_log_message( -- cgit v1.2.3 From ce0ec7db55a9f12f55ac5ba019a95bc0b7d5cbd7 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 18 Apr 2020 21:00:38 -0700 Subject: Tags: always use top-most folder for role restrictions Ensures that nested directories aren't used as the value for the role name. --- bot/cogs/tags.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index e5492971d..1c124b25a 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -34,25 +34,28 @@ class Tags(Cog): @staticmethod def get_tags() -> dict: """Get all tags.""" - # Save all tags in memory. cache = {} - tag_files = Path("bot", "resources", "tags").glob("**/*") - for file in tag_files: + base_path = Path("bot", "resources", "tags") + for file in base_path.glob("**/*"): if file.is_file(): tag_title = file.stem tag = { "title": tag_title, "embed": { - "description": file.read_text() + "description": file.read_text(), }, - "restricted_to": "developers" + "restricted_to": "developers", } - parent_folder = file.parent.stem - if parent_folder != "tags": - tag["restricted_to"] = parent_folder + + # Convert to a list to allow negative indexing. + parents = list(file.relative_to(base_path).parents) + if len(parents) > 1: + # -1 would be '.' hence -2 is used as the index. + tag["restricted_to"] = parents[-2].name cache[tag_title] = tag + return cache @staticmethod -- cgit v1.2.3 From ecb777b167dce5a8246e9c5ad8a202b370d97b7d Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sun, 19 Apr 2020 20:09:12 +0300 Subject: Created `News` cog Added general content of cog: class and setup. --- bot/cogs/news.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 bot/cogs/news.py diff --git a/bot/cogs/news.py b/bot/cogs/news.py new file mode 100644 index 000000000..8eb8689c2 --- /dev/null +++ b/bot/cogs/news.py @@ -0,0 +1,15 @@ +from discord.ext.commands import Cog + +from bot.bot import Bot + + +class News(Cog): + """Post new PEPs and Python News to `#python-news`.""" + + def __init__(self, bot: Bot): + self.bot = bot + + +def setup(bot: Bot) -> None: + """Add `News` cog.""" + bot.add_cog(News(bot)) -- cgit v1.2.3 From 9e586ef21170953a4879ca038bbc15e354937ddb Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 20 Apr 2020 08:29:07 +0300 Subject: Added #python-news channel ID to constants `Channels` --- bot/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/constants.py b/bot/constants.py index 2add028e7..8135f47a9 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -394,6 +394,7 @@ class Channels(metaclass=YAMLGetter): off_topic_2: int organisation: int python_discussion: int + python_news: int reddit: int talent_pool: int user_event_announcements: int -- cgit v1.2.3 From b99a767b8bde01c730fec0ceb1ddf6fdb31bb983 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 20 Apr 2020 11:37:04 +0300 Subject: Added `News` cog loading --- bot/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/__main__.py b/bot/__main__.py index 3aa36bfc0..42c1a4f3a 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -51,6 +51,7 @@ bot.load_extension("bot.cogs.eval") bot.load_extension("bot.cogs.information") bot.load_extension("bot.cogs.jams") bot.load_extension("bot.cogs.moderation") +bot.load_extension("bot.cogs.news") bot.load_extension("bot.cogs.off_topic_names") bot.load_extension("bot.cogs.reddit") bot.load_extension("bot.cogs.reminders") -- cgit v1.2.3 From bb48c5e6fea14bc8ec42b1188ceb5008fa259463 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 20 Apr 2020 11:46:17 +0300 Subject: Added helper function `News.sync_maillists` Function sync maillists listing with API, that hold IDs of message that have news. PEPs handling is over RSS, so this will added manually in this function. --- bot/cogs/news.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index 8eb8689c2..c850b4192 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -2,12 +2,35 @@ from discord.ext.commands import Cog from bot.bot import Bot +MAIL_LISTS = [ + "python-ideas", + "python-announce-list", + "pypi-announce" +] + class News(Cog): """Post new PEPs and Python News to `#python-news`.""" def __init__(self, bot: Bot): self.bot = bot + self.bot.loop.create_task(self.sync_maillists()) + + async def sync_maillists(self) -> None: + """Sync currently in-use maillists with API.""" + # Wait until guild is available to avoid running before API is ready + await self.bot.wait_until_guild_available() + + response = await self.bot.api_client.get("bot/bot-settings/news") + for mail in MAIL_LISTS: + if mail not in response["data"]: + response["data"][mail] = [] + + # Because we are handling PEPs differently, we don't include it to mail lists + if "pep" not in response["data"]: + response["data"]["pep"] = [] + + await self.bot.api_client.put("bot/bot-settings/news", json=response) def setup(bot: Bot) -> None: -- cgit v1.2.3 From b6450b57207341d5cf9b581b0e56a579a154cae4 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 20 Apr 2020 12:01:27 +0300 Subject: Added new dependency `feedparser` --- Pipfile | 1 + Pipfile.lock | 105 ++++++++++++++++++++++++++++++++--------------------------- 2 files changed, 58 insertions(+), 48 deletions(-) diff --git a/Pipfile b/Pipfile index e7fb61957..9994f58e9 100644 --- a/Pipfile +++ b/Pipfile @@ -21,6 +21,7 @@ sentry-sdk = "~=0.14" coloredlogs = "~=14.0" colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} statsd = "~=3.3" +feedparser = "~=5.2" [dev-packages] coverage = "~=5.0" diff --git a/Pipfile.lock b/Pipfile.lock index 19e03bda4..5aae9e1b6 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "10636aef5a07f17bd00608df2cc5214fcbfe3de4745cdeea7a076b871754620a" + "sha256": "6a53e10f1f1bf5348da7675113ca2be2667960b7ba65630650e54e7d920d9269" }, "pipfile-spec": 6, "requires": { @@ -179,6 +179,15 @@ ], "version": "==0.16" }, + "feedparser": { + "hashes": [ + "sha256:bd030652c2d08532c034c27fcd7c85868e7fa3cb2b17f230a44a6bbc92519bf9", + "sha256:cd2485472e41471632ed3029d44033ee420ad0b57111db95c240c9160a85831c", + "sha256:ce875495c90ebd74b179855449040003a1beb40cd13d5f037a0654251e260b02" + ], + "index": "pypi", + "version": "==5.2.1" + }, "fuzzywuzzy": { "hashes": [ "sha256:45016e92264780e58972dca1b3d939ac864b78437422beecebb3095f8efd00e8", @@ -189,10 +198,10 @@ }, "humanfriendly": { "hashes": [ - "sha256:25c2108a45cfd1e8fbe9cdb30b825d34ef5d5675c8e11e4775c9aedbfb0bdee2", - "sha256:3a831920e40e55ad49adb64c9179ed50c604cabca72cd300e7bd5b51310e4ebb" + "sha256:bf52ec91244819c780341a3438d5d7b09f431d3f113a475147ac9b7b167a3d12", + "sha256:e78960b31198511f45fd455534ae7645a6207d33e512d2e842c766d15d9c8080" ], - "version": "==8.1" + "version": "==8.2" }, "idna": { "hashes": [ @@ -210,10 +219,10 @@ }, "jinja2": { "hashes": [ - "sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250", - "sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49" + "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0", + "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035" ], - "version": "==2.11.1" + "version": "==2.11.2" }, "lxml": { "hashes": [ @@ -527,10 +536,10 @@ }, "urllib3": { "hashes": [ - "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", - "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" + "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", + "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" ], - "version": "==1.25.8" + "version": "==1.25.9" }, "websockets": { "hashes": [ @@ -606,40 +615,40 @@ }, "coverage": { "hashes": [ - "sha256:03f630aba2b9b0d69871c2e8d23a69b7fe94a1e2f5f10df5049c0df99db639a0", - "sha256:046a1a742e66d065d16fb564a26c2a15867f17695e7f3d358d7b1ad8a61bca30", - "sha256:0a907199566269e1cfa304325cc3b45c72ae341fbb3253ddde19fa820ded7a8b", - "sha256:165a48268bfb5a77e2d9dbb80de7ea917332a79c7adb747bd005b3a07ff8caf0", - "sha256:1b60a95fc995649464e0cd48cecc8288bac5f4198f21d04b8229dc4097d76823", - "sha256:1f66cf263ec77af5b8fe14ef14c5e46e2eb4a795ac495ad7c03adc72ae43fafe", - "sha256:2e08c32cbede4a29e2a701822291ae2bc9b5220a971bba9d1e7615312efd3037", - "sha256:3844c3dab800ca8536f75ae89f3cf566848a3eb2af4d9f7b1103b4f4f7a5dad6", - "sha256:408ce64078398b2ee2ec08199ea3fcf382828d2f8a19c5a5ba2946fe5ddc6c31", - "sha256:443be7602c790960b9514567917af538cac7807a7c0c0727c4d2bbd4014920fd", - "sha256:4482f69e0701139d0f2c44f3c395d1d1d37abd81bfafbf9b6efbe2542679d892", - "sha256:4a8a259bf990044351baf69d3b23e575699dd60b18460c71e81dc565f5819ac1", - "sha256:513e6526e0082c59a984448f4104c9bf346c2da9961779ede1fc458e8e8a1f78", - "sha256:5f587dfd83cb669933186661a351ad6fc7166273bc3e3a1531ec5c783d997aac", - "sha256:62061e87071497951155cbccee487980524d7abea647a1b2a6eb6b9647df9006", - "sha256:641e329e7f2c01531c45c687efcec8aeca2a78a4ff26d49184dce3d53fc35014", - "sha256:65a7e00c00472cd0f59ae09d2fb8a8aaae7f4a0cf54b2b74f3138d9f9ceb9cb2", - "sha256:6ad6ca45e9e92c05295f638e78cd42bfaaf8ee07878c9ed73e93190b26c125f7", - "sha256:73aa6e86034dad9f00f4bbf5a666a889d17d79db73bc5af04abd6c20a014d9c8", - "sha256:7c9762f80a25d8d0e4ab3cb1af5d9dffbddb3ee5d21c43e3474c84bf5ff941f7", - "sha256:85596aa5d9aac1bf39fe39d9fa1051b0f00823982a1de5766e35d495b4a36ca9", - "sha256:86a0ea78fd851b313b2e712266f663e13b6bc78c2fb260b079e8b67d970474b1", - "sha256:8a620767b8209f3446197c0e29ba895d75a1e272a36af0786ec70fe7834e4307", - "sha256:922fb9ef2c67c3ab20e22948dcfd783397e4c043a5c5fa5ff5e9df5529074b0a", - "sha256:9fad78c13e71546a76c2f8789623eec8e499f8d2d799f4b4547162ce0a4df435", - "sha256:a37c6233b28e5bc340054cf6170e7090a4e85069513320275a4dc929144dccf0", - "sha256:c3fc325ce4cbf902d05a80daa47b645d07e796a80682c1c5800d6ac5045193e5", - "sha256:cda33311cb9fb9323958a69499a667bd728a39a7aa4718d7622597a44c4f1441", - "sha256:db1d4e38c9b15be1521722e946ee24f6db95b189d1447fa9ff18dd16ba89f732", - "sha256:eda55e6e9ea258f5e4add23bcf33dc53b2c319e70806e180aecbff8d90ea24de", - "sha256:f372cdbb240e09ee855735b9d85e7f50730dcfb6296b74b95a3e5dea0615c4c1" + "sha256:00f1d23f4336efc3b311ed0d807feb45098fc86dee1ca13b3d6768cdab187c8a", + "sha256:01333e1bd22c59713ba8a79f088b3955946e293114479bbfc2e37d522be03355", + "sha256:0cb4be7e784dcdc050fc58ef05b71aa8e89b7e6636b99967fadbdba694cf2b65", + "sha256:0e61d9803d5851849c24f78227939c701ced6704f337cad0a91e0972c51c1ee7", + "sha256:1601e480b9b99697a570cea7ef749e88123c04b92d84cedaa01e117436b4a0a9", + "sha256:2742c7515b9eb368718cd091bad1a1b44135cc72468c731302b3d641895b83d1", + "sha256:2d27a3f742c98e5c6b461ee6ef7287400a1956c11421eb574d843d9ec1f772f0", + "sha256:402e1744733df483b93abbf209283898e9f0d67470707e3c7516d84f48524f55", + "sha256:5c542d1e62eece33c306d66fe0a5c4f7f7b3c08fecc46ead86d7916684b36d6c", + "sha256:5f2294dbf7875b991c381e3d5af2bcc3494d836affa52b809c91697449d0eda6", + "sha256:6402bd2fdedabbdb63a316308142597534ea8e1895f4e7d8bf7476c5e8751fef", + "sha256:66460ab1599d3cf894bb6baee8c684788819b71a5dc1e8fa2ecc152e5d752019", + "sha256:782caea581a6e9ff75eccda79287daefd1d2631cc09d642b6ee2d6da21fc0a4e", + "sha256:79a3cfd6346ce6c13145731d39db47b7a7b859c0272f02cdb89a3bdcbae233a0", + "sha256:7a5bdad4edec57b5fb8dae7d3ee58622d626fd3a0be0dfceda162a7035885ecf", + "sha256:8fa0cbc7ecad630e5b0f4f35b0f6ad419246b02bc750de7ac66db92667996d24", + "sha256:a027ef0492ede1e03a8054e3c37b8def89a1e3c471482e9f046906ba4f2aafd2", + "sha256:a3f3654d5734a3ece152636aad89f58afc9213c6520062db3978239db122f03c", + "sha256:a82b92b04a23d3c8a581fc049228bafde988abacba397d57ce95fe95e0338ab4", + "sha256:acf3763ed01af8410fc36afea23707d4ea58ba7e86a8ee915dfb9ceff9ef69d0", + "sha256:adeb4c5b608574a3d647011af36f7586811a2c1197c861aedb548dd2453b41cd", + "sha256:b83835506dfc185a319031cf853fa4bb1b3974b1f913f5bb1a0f3d98bdcded04", + "sha256:bb28a7245de68bf29f6fb199545d072d1036a1917dca17a1e75bbb919e14ee8e", + "sha256:bf9cb9a9fd8891e7efd2d44deb24b86d647394b9705b744ff6f8261e6f29a730", + "sha256:c317eaf5ff46a34305b202e73404f55f7389ef834b8dbf4da09b9b9b37f76dd2", + "sha256:dbe8c6ae7534b5b024296464f387d57c13caa942f6d8e6e0346f27e509f0f768", + "sha256:de807ae933cfb7f0c7d9d981a053772452217df2bf38e7e6267c9cbf9545a796", + "sha256:dead2ddede4c7ba6cb3a721870f5141c97dc7d85a079edb4bd8d88c3ad5b20c7", + "sha256:dec5202bfe6f672d4511086e125db035a52b00f1648d6407cc8e526912c0353a", + "sha256:e1ea316102ea1e1770724db01998d1603ed921c54a86a2efcb03428d5417e489", + "sha256:f90bfc4ad18450c80b024036eaf91e4a246ae287701aaa88eaebebf150868052" ], "index": "pypi", - "version": "==5.0.4" + "version": "==5.1" }, "distlib": { "hashes": [ @@ -671,11 +680,11 @@ }, "flake8-annotations": { "hashes": [ - "sha256:a38b44d01abd480586a92a02a2b0a36231ec42dcc5e114de78fa5db016d8d3f9", - "sha256:d5b0e8704e4e7728b352fa1464e23539ff2341ba11cc153b536fa2cf921ee659" + "sha256:9091d920406a7ff10e401e0dd1baa396d1d7d2e3d101a9beecf815f5894ad554", + "sha256:f59fdceb8c8f380a20aed20e1ba8a57bde05935958166c52be2249f113f7ab75" ], "index": "pypi", - "version": "==2.0.1" + "version": "==2.1.0" }, "flake8-bugbear": { "hashes": [ @@ -836,10 +845,10 @@ }, "virtualenv": { "hashes": [ - "sha256:00cfe8605fb97f5a59d52baab78e6070e72c12ca64f51151695407cc0eb8a431", - "sha256:c8364ec469084046c779c9a11ae6340094e8a0bf1d844330fc55c1cefe67c172" + "sha256:5021396e8f03d0d002a770da90e31e61159684db2859d0ba4850fbea752aa675", + "sha256:ac53ade75ca189bc97b6c1d9ec0f1a50efe33cbf178ae09452dcd9fd309013c1" ], - "version": "==20.0.17" + "version": "==20.0.18" } } } -- cgit v1.2.3 From ce7efd3c27ea706cb46055c7eae06b52ffce7491 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 20 Apr 2020 12:51:03 +0300 Subject: Added #python-news channel webhook to `Webhooks` in constants --- bot/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/constants.py b/bot/constants.py index 8135f47a9..4c2f22741 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -412,6 +412,7 @@ class Webhooks(metaclass=YAMLGetter): reddit: int duck_pond: int dev_log: int + python_news: int class Roles(metaclass=YAMLGetter): -- cgit v1.2.3 From ab496d4b059673d9a8f3816119ad5fe37e2787cc Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 20 Apr 2020 12:59:35 +0300 Subject: Created helper function `get_webhook` and added property in `News` `News.get_webhook` fetch discord.Webhook by ID provided in config. `self.webhook` use webhook that it got from this function. --- bot/cogs/news.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index c850b4192..69305c93d 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -1,5 +1,7 @@ +import discord from discord.ext.commands import Cog +from bot import constants from bot.bot import Bot MAIL_LISTS = [ @@ -15,10 +17,11 @@ class News(Cog): def __init__(self, bot: Bot): self.bot = bot self.bot.loop.create_task(self.sync_maillists()) + self.webhook = self.bot.loop.create_task(self.get_webhook()) async def sync_maillists(self) -> None: """Sync currently in-use maillists with API.""" - # Wait until guild is available to avoid running before API is ready + # Wait until guild is available to avoid running before everything is ready await self.bot.wait_until_guild_available() response = await self.bot.api_client.get("bot/bot-settings/news") @@ -32,6 +35,10 @@ class News(Cog): await self.bot.api_client.put("bot/bot-settings/news", json=response) + async def get_webhook(self) -> discord.Webhook: + """Get #python-news channel webhook.""" + return await self.bot.fetch_webhook(constants.Webhooks.python_news) + def setup(bot: Bot) -> None: """Add `News` cog.""" -- cgit v1.2.3 From e5f30076304eac16d76b0daabead346253c7d9b4 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 20 Apr 2020 13:08:43 +0300 Subject: Added new category `python_news` to config, that hold mail lists, channel and webhook. This use local dev environment IDs. --- config-default.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/config-default.yml b/config-default.yml index f2b0bfa9f..553afaa33 100644 --- a/config-default.yml +++ b/config-default.yml @@ -122,6 +122,7 @@ guild: channels: announcements: 354619224620138496 user_event_announcements: &USER_EVENT_A 592000283102674944 + python_news: &PYNEWS_CHANNEL 701667765102051398 # Development dev_contrib: &DEV_CONTRIB 635950537262759947 @@ -231,11 +232,12 @@ guild: - *HELPERS_ROLE webhooks: - talent_pool: 569145364800602132 - big_brother: 569133704568373283 - reddit: 635408384794951680 - duck_pond: 637821475327311927 - dev_log: 680501655111729222 + talent_pool: 569145364800602132 + big_brother: 569133704568373283 + reddit: 635408384794951680 + duck_pond: 637821475327311927 + dev_log: 680501655111729222 + python_news: &PYNEWS_WEBHOOK 701731296342179850 filter: @@ -568,5 +570,13 @@ duck_pond: - *DUCKY_MAUL - *DUCKY_SANTA +python_news: + mail_lists: + - 'python-ideas' + - 'python-announce-list' + - 'pypi-announce' + channel: *PYNEWS_CHANNEL + webhook: *PYNEWS_WEBHOOK + config: required_keys: ['bot.token'] -- cgit v1.2.3 From f9dac725a5cac4dbe725aa86d1fbcee5e3a9b5af Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 20 Apr 2020 13:14:46 +0300 Subject: Applied Python News config changes Removed Webhook and Channel from their listings, created new class `PythonNews` that hold them + mail lists. --- bot/constants.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bot/constants.py b/bot/constants.py index 4c2f22741..202a17d71 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -394,7 +394,6 @@ class Channels(metaclass=YAMLGetter): off_topic_2: int organisation: int python_discussion: int - python_news: int reddit: int talent_pool: int user_event_announcements: int @@ -412,7 +411,6 @@ class Webhooks(metaclass=YAMLGetter): reddit: int duck_pond: int dev_log: int - python_news: int class Roles(metaclass=YAMLGetter): @@ -571,6 +569,14 @@ class Sync(metaclass=YAMLGetter): max_diff: int +class PythonNews(metaclass=YAMLGetter): + section = 'python_news' + + mail_lists: List[str] + channel: int + webhook: int + + class Event(Enum): """ Event names. This does not include every event (for example, raw -- cgit v1.2.3 From 6abfc324bb81f7eb7224da913a62ae28cc49f674 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 20 Apr 2020 13:16:57 +0300 Subject: Applied constant changes to News Replaced in-file mail lists with constants.py's, replaced webhook ID getting. --- bot/cogs/news.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index 69305c93d..3aa57442a 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -4,12 +4,6 @@ from discord.ext.commands import Cog from bot import constants from bot.bot import Bot -MAIL_LISTS = [ - "python-ideas", - "python-announce-list", - "pypi-announce" -] - class News(Cog): """Post new PEPs and Python News to `#python-news`.""" @@ -25,7 +19,7 @@ class News(Cog): await self.bot.wait_until_guild_available() response = await self.bot.api_client.get("bot/bot-settings/news") - for mail in MAIL_LISTS: + for mail in constants.PythonNews.mail_lists: if mail not in response["data"]: response["data"][mail] = [] @@ -37,7 +31,7 @@ class News(Cog): async def get_webhook(self) -> discord.Webhook: """Get #python-news channel webhook.""" - return await self.bot.fetch_webhook(constants.Webhooks.python_news) + return await self.bot.fetch_webhook(constants.PythonNews.webhook) def setup(bot: Bot) -> None: -- cgit v1.2.3 From c8c30f3df673975f6d22a14c4658598921c15254 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 20 Apr 2020 14:17:52 +0300 Subject: Created PEP news task + minor changes in `News` - Created task `post_pep_news` that pull existing news message IDs from API, do checks and send new PEP when it's not already sent. - Removed `get_webhook` - Removed `self.webhook` --- bot/cogs/news.py | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index 3aa57442a..6e9441997 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -1,9 +1,18 @@ +import logging +from datetime import datetime + import discord +import feedparser from discord.ext.commands import Cog +from discord.ext.tasks import loop from bot import constants from bot.bot import Bot +PEPS_RSS_URL = "https://www.python.org/dev/peps/peps.rss/" + +log = logging.getLogger(__name__) + class News(Cog): """Post new PEPs and Python News to `#python-news`.""" @@ -11,7 +20,8 @@ class News(Cog): def __init__(self, bot: Bot): self.bot = bot self.bot.loop.create_task(self.sync_maillists()) - self.webhook = self.bot.loop.create_task(self.get_webhook()) + + self.post_pep_news.start() async def sync_maillists(self) -> None: """Sync currently in-use maillists with API.""" @@ -29,9 +39,64 @@ class News(Cog): await self.bot.api_client.put("bot/bot-settings/news", json=response) - async def get_webhook(self) -> discord.Webhook: - """Get #python-news channel webhook.""" - return await self.bot.fetch_webhook(constants.PythonNews.webhook) + @loop(minutes=20) + async def post_pep_news(self) -> None: + """Fetch new PEPs and when they don't have announcement in #python-news, create it.""" + # Wait until everything is ready and http_session available + await self.bot.wait_until_guild_available() + + async with self.bot.http_session.get(PEPS_RSS_URL) as resp: + data = feedparser.parse(await resp.text()) + + news_channel = self.bot.get_channel(constants.PythonNews.channel) + webhook = await self.bot.fetch_webhook(constants.PythonNews.webhook) + + news_listing = await self.bot.api_client.get("bot/bot-settings/news") + payload = news_listing.copy() + pep_news_ids = news_listing["data"]["pep"] + pep_news = [] + + for pep_id in pep_news_ids: + message = discord.utils.get(self.bot.cached_messages, id=pep_id) + if message is None: + message = await news_channel.fetch_message(pep_id) + if message is None: + log.warning(f"Can't fetch news message with ID {pep_id}. Deleting it entry from DB.") + payload["data"]["pep"].remove(pep_id) + pep_news.append((message.embeds[0].title, message.embeds[0].timestamp)) + + # Reverse entries to send oldest first + data["entries"].reverse() + for new in data["entries"]: + try: + new_datetime = datetime.strptime(new["published"], "%a, %d %b %Y %X %Z") + except ValueError: + log.warning(f"Wrong datetime format passed in PEP new: {new['published']}") + continue + if ( + any(pep_new[0] == new["title"] for pep_new in pep_news) + and any(pep_new[1] == new_datetime for pep_new in pep_news) + ): + continue + + embed = discord.Embed( + title=new["title"], + description=new["summary"], + timestamp=new_datetime, + url=new["link"], + colour=constants.Colours.soft_green + ) + + pep_msg = await webhook.send( + embed=embed, + username=data["feed"]["title"], + avatar_url="https://www.python.org/static/opengraph-icon-200x200.png", + wait=True + ) + payload["data"]["pep"].append(pep_msg.id) + + # Apply new sent news to DB to avoid duplicate sending + await self.bot.api_client.put("bot/bot-settings/news", json=payload) def setup(bot: Bot) -> None: -- cgit v1.2.3 From d3a1e346ab65f65e8addda68a2e5dc6860739448 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 20 Apr 2020 15:10:13 +0300 Subject: Added new function `News.get_webhook_names` + new variable `News.webhook_names` Function fetch display names of these mail lists, that bot will post. These names will be used on Webhook author names. `News.webhook_names` storage these name and display name pairs. --- bot/cogs/news.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index 6e9441997..878e533ef 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -19,7 +19,9 @@ class News(Cog): def __init__(self, bot: Bot): self.bot = bot + self.webhook_names = {} self.bot.loop.create_task(self.sync_maillists()) + self.bot.loop.create_task(self.get_webhook_names()) self.post_pep_news.start() @@ -39,6 +41,17 @@ class News(Cog): await self.bot.api_client.put("bot/bot-settings/news", json=response) + async def get_webhook_names(self) -> None: + """Get webhook author names from maillist API.""" + await self.bot.wait_until_guild_available() + + async with self.bot.http_session.get("https://mail.python.org/archives/api/lists") as resp: + lists = await resp.json() + + for mail in lists: + if mail["name"].split("@")[0] in constants.PythonNews.mail_lists: + self.webhook_names[mail["name"].split("@")[0]] = mail["display_name"] + @loop(minutes=20) async def post_pep_news(self) -> None: """Fetch new PEPs and when they don't have announcement in #python-news, create it.""" -- cgit v1.2.3 From c2eac1f8b2a424dd018909b0e4084e730e029210 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 20 Apr 2020 15:19:46 +0300 Subject: Added new dependency `beatifulsoup4` for Python news HTML parsing --- Pipfile | 1 + Pipfile.lock | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index 9994f58e9..14c9ef926 100644 --- a/Pipfile +++ b/Pipfile @@ -22,6 +22,7 @@ coloredlogs = "~=14.0" colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} statsd = "~=3.3" feedparser = "~=5.2" +beautifulsoup4 = "~=4.9" [dev-packages] coverage = "~=5.0" diff --git a/Pipfile.lock b/Pipfile.lock index 5aae9e1b6..4e7050a13 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6a53e10f1f1bf5348da7675113ca2be2667960b7ba65630650e54e7d920d9269" + "sha256": "64620e7e825c74fd3010821fb30843b19f5dafb2b5a1f6eafedc0a5febd99b69" }, "pipfile-spec": 6, "requires": { @@ -91,6 +91,7 @@ "sha256:a4bbe77fd30670455c5296242967a123ec28c37e9702a8a81bd2f20a4baf0368", "sha256:d4e96ac9b0c3a6d3f0caae2e4124e6055c5dcafde8e2f831ff194c104f0775a0" ], + "index": "pypi", "version": "==4.9.0" }, "certifi": { -- cgit v1.2.3 From 866240620827623e9a9a813a38e2ad097fc1a783 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 20 Apr 2020 20:06:57 +0300 Subject: Defined `chardet` log level to warning to avoid spam --- bot/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/__init__.py b/bot/__init__.py index 2dd4af225..344afdf15 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -58,4 +58,5 @@ coloredlogs.install(logger=root_log, stream=sys.stdout) logging.getLogger("discord").setLevel(logging.WARNING) logging.getLogger("websockets").setLevel(logging.WARNING) +logging.getLogger("chardet").setLevel(logging.WARNING) logging.getLogger(__name__) -- cgit v1.2.3 From 7348afc0be7d6f5d3939b13c124e616051fc6170 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 20 Apr 2020 20:10:54 +0300 Subject: Implemented maillists news posting, created helper functions + added date check - Created helper function `News.get_thread_and_first_mail` - Created helper function `News.send_webhook` - Created helper function `News.check_new_exist` - Task `post_maillist_news`, that send latest maillist threads to news, when they don't exist. - Implemented helper functions to PEP news - Added date check --- bot/cogs/news.py | 150 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 131 insertions(+), 19 deletions(-) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index 878e533ef..52c36da2e 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -1,8 +1,10 @@ import logging -from datetime import datetime +import typing as t +from datetime import date, datetime import discord import feedparser +from bs4 import BeautifulSoup from discord.ext.commands import Cog from discord.ext.tasks import loop @@ -11,6 +13,13 @@ from bot.bot import Bot PEPS_RSS_URL = "https://www.python.org/dev/peps/peps.rss/" +RECENT_THREADS_TEMPLATE = "https://mail.python.org/archives/list/{name}@python.org/recent-threads" +THREAD_TEMPLATE_URL = "https://mail.python.org/archives/api/list/{name}@python.org/thread/{id}/" +MAILMAN_PROFILE_URL = "https://mail.python.org/archives/users/{id}/" +THREAD_URL = "https://mail.python.org/archives/list/{list}@python.org/thread/{id}/" + +AVATAR_URL = "https://www.python.org/static/opengraph-icon-200x200.png" + log = logging.getLogger(__name__) @@ -24,6 +33,7 @@ class News(Cog): self.bot.loop.create_task(self.get_webhook_names()) self.post_pep_news.start() + self.post_maillist_news.start() async def sync_maillists(self) -> None: """Sync currently in-use maillists with API.""" @@ -74,8 +84,8 @@ class News(Cog): if message is None: message = await news_channel.fetch_message(pep_id) if message is None: - log.warning(f"Can't fetch news message with ID {pep_id}. Deleting it entry from DB.") - payload["data"]["pep"].remove(pep_id) + log.warning("Can't fetch PEP new message ID.") + continue pep_news.append((message.embeds[0].title, message.embeds[0].timestamp)) # Reverse entries to send oldest first @@ -87,30 +97,132 @@ class News(Cog): log.warning(f"Wrong datetime format passed in PEP new: {new['published']}") continue if ( - any(pep_new[0] == new["title"] for pep_new in pep_news) - and any(pep_new[1] == new_datetime for pep_new in pep_news) + (any(pep_new[0] == new["title"] for pep_new in pep_news) + and any(pep_new[1] == new_datetime for pep_new in pep_news)) + or new_datetime.date() < date.today() ): continue - embed = discord.Embed( - title=new["title"], - description=new["summary"], - timestamp=new_datetime, - url=new["link"], - colour=constants.Colours.soft_green - ) - - pep_msg = await webhook.send( - embed=embed, - username=data["feed"]["title"], - avatar_url="https://www.python.org/static/opengraph-icon-200x200.png", - wait=True + msg_id = await self.send_webhook( + webhook, + new["title"], + new["summary"], + new_datetime, + new["link"], + None, + None, + data["feed"]["title"] ) - payload["data"]["pep"].append(pep_msg.id) + payload["data"]["pep"].append(msg_id) # Apply new sent news to DB to avoid duplicate sending await self.bot.api_client.put("bot/bot-settings/news", json=payload) + @loop(minutes=20) + async def post_maillist_news(self) -> None: + """Send new maillist threads to #python-news that is listed in configuration.""" + await self.bot.wait_until_guild_available() + webhook = await self.bot.fetch_webhook(constants.PythonNews.webhook) + existing_news = await self.bot.api_client.get("bot/bot-settings/news") + payload = existing_news.copy() + + for maillist in constants.PythonNews.mail_lists: + async with self.bot.http_session.get(RECENT_THREADS_TEMPLATE.format(name=maillist)) as resp: + recents = BeautifulSoup(await resp.text()) + + for thread in recents.html.body.div.find_all("a", href=True): + # We want only these threads that have identifiers + if "latest" in thread["href"]: + continue + + thread_information, email_information = await self.get_thread_and_first_mail( + maillist, thread["href"].split("/")[-2] + ) + + try: + new_date = datetime.strptime(email_information["date"], "%Y-%m-%dT%X%z") + except ValueError: + log.warning(f"Invalid datetime from Thread email: {email_information['date']}") + continue + + if ( + await self.check_new_exist(thread_information["subject"], new_date, maillist, existing_news) + or new_date.date() < date.today() + ): + continue + + content = email_information["content"] + link = THREAD_URL.format(id=thread["href"].split("/")[-2], list=maillist) + msg_id = await self.send_webhook( + webhook, + thread_information["subject"], + content[:500] + f"... [continue reading]({link})" if len(content) > 500 else content, + new_date, + link, + f"{email_information['sender_name']} ({email_information['sender']['address']})", + MAILMAN_PROFILE_URL.format(id=email_information["sender"]["mailman_id"]), + self.webhook_names[maillist] + ) + payload["data"][maillist].append(msg_id) + + await self.bot.api_client.put("bot/bot-settings/news", json=payload) + + async def check_new_exist(self, title: str, timestamp: datetime, maillist: str, news: t.Dict[str, t.Any]) -> bool: + """Check does this new title + timestamp already exist in #python-news.""" + channel = await self.bot.fetch_channel(constants.PythonNews.channel) + + for new in news["data"][maillist]: + message = discord.utils.get(self.bot.cached_messages, id=new) + if message is None: + message = await channel.fetch_message(new) + if message is None: + return False + + if message.embeds[0].title == title and message.embeds[0].timestamp == timestamp: + return True + return False + + async def send_webhook(self, + webhook: discord.Webhook, + title: str, + description: str, + timestamp: datetime, + url: str, + author: str, + author_url: str, + webhook_profile_name: str + ) -> int: + """Send webhook entry and return ID of message.""" + embed = discord.Embed( + title=title, + description=description, + timestamp=timestamp, + url=url, + colour=constants.Colours.soft_green + ) + embed.set_author( + name=author, + url=author_url + ) + msg = await webhook.send( + embed=embed, + username=webhook_profile_name, + avatar_url=AVATAR_URL, + wait=True + ) + return msg.id + + async def get_thread_and_first_mail(self, maillist: str, thread_identifier: str) -> t.Tuple[t.Any, t.Any]: + """Get mail thread and first mail from mail.python.org based on `maillist` and `thread_identifier`.""" + async with self.bot.http_session.get( + THREAD_TEMPLATE_URL.format(name=maillist, id=thread_identifier) + ) as resp: + thread_information = await resp.json() + + async with self.bot.http_session.get(thread_information["starting_email"]) as resp: + email_information = await resp.json() + return thread_information, email_information + def setup(bot: Bot) -> None: """Add `News` cog.""" -- cgit v1.2.3 From 47fc4dbbcb288f5757b85ed1e0a385048b708c34 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 21 Apr 2020 08:30:41 +0300 Subject: `News` Cog improvisations - Created new helper function `News.get_webhook_and_channel` to will be run in Cog loading and will fetch #python-news channel and webhook. - Fixed `News.send_webhook` when you pass `None` as author, this will not add author. - Replaced individual channel and webhook fetches with `News.webhook` and `News.channel`. - Replaced positional arguments with kwargs in `send_webhook` uses. - Moved maillists syncing from `News.__init__` to `News.post_maillist_news`. - Simplified `News.post_pep_news` already exist checks. --- bot/cogs/news.py | 73 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index 52c36da2e..21ddb6128 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -29,8 +29,11 @@ class News(Cog): def __init__(self, bot: Bot): self.bot = bot self.webhook_names = {} - self.bot.loop.create_task(self.sync_maillists()) + self.webhook: t.Optional[discord.Webhook] = None + self.channel: t.Optional[discord.TextChannel] = None + self.bot.loop.create_task(self.get_webhook_names()) + self.bot.loop.create_task(self.get_webhook_and_channel()) self.post_pep_news.start() self.post_maillist_news.start() @@ -71,9 +74,6 @@ class News(Cog): async with self.bot.http_session.get(PEPS_RSS_URL) as resp: data = feedparser.parse(await resp.text()) - news_channel = self.bot.get_channel(constants.PythonNews.channel) - webhook = await self.bot.fetch_webhook(constants.PythonNews.webhook) - news_listing = await self.bot.api_client.get("bot/bot-settings/news") payload = news_listing.copy() pep_news_ids = news_listing["data"]["pep"] @@ -82,11 +82,11 @@ class News(Cog): for pep_id in pep_news_ids: message = discord.utils.get(self.bot.cached_messages, id=pep_id) if message is None: - message = await news_channel.fetch_message(pep_id) + message = await self.channel.fetch_message(pep_id) if message is None: log.warning("Can't fetch PEP new message ID.") continue - pep_news.append((message.embeds[0].title, message.embeds[0].timestamp)) + pep_news.append(message.embeds[0].title) # Reverse entries to send oldest first data["entries"].reverse() @@ -97,21 +97,17 @@ class News(Cog): log.warning(f"Wrong datetime format passed in PEP new: {new['published']}") continue if ( - (any(pep_new[0] == new["title"] for pep_new in pep_news) - and any(pep_new[1] == new_datetime for pep_new in pep_news)) + any(pep_new == new["title"] for pep_new in pep_news) or new_datetime.date() < date.today() ): continue msg_id = await self.send_webhook( - webhook, - new["title"], - new["summary"], - new_datetime, - new["link"], - None, - None, - data["feed"]["title"] + title=new["title"], + description=new["summary"], + timestamp=new_datetime, + url=new["link"], + webhook_profile_name=data["feed"]["title"] ) payload["data"]["pep"].append(msg_id) @@ -122,7 +118,7 @@ class News(Cog): async def post_maillist_news(self) -> None: """Send new maillist threads to #python-news that is listed in configuration.""" await self.bot.wait_until_guild_available() - webhook = await self.bot.fetch_webhook(constants.PythonNews.webhook) + await self.sync_maillists() existing_news = await self.bot.api_client.get("bot/bot-settings/news") payload = existing_news.copy() @@ -154,14 +150,13 @@ class News(Cog): content = email_information["content"] link = THREAD_URL.format(id=thread["href"].split("/")[-2], list=maillist) msg_id = await self.send_webhook( - webhook, - thread_information["subject"], - content[:500] + f"... [continue reading]({link})" if len(content) > 500 else content, - new_date, - link, - f"{email_information['sender_name']} ({email_information['sender']['address']})", - MAILMAN_PROFILE_URL.format(id=email_information["sender"]["mailman_id"]), - self.webhook_names[maillist] + title=thread_information["subject"], + description=content[:500] + f"... [continue reading]({link})" if len(content) > 500 else content, + timestamp=new_date, + url=link, + author=f"{email_information['sender_name']} ({email_information['sender']['address']})", + author_url=MAILMAN_PROFILE_URL.format(id=email_information["sender"]["mailman_id"]), + webhook_profile_name=self.webhook_names[maillist] ) payload["data"][maillist].append(msg_id) @@ -169,12 +164,10 @@ class News(Cog): async def check_new_exist(self, title: str, timestamp: datetime, maillist: str, news: t.Dict[str, t.Any]) -> bool: """Check does this new title + timestamp already exist in #python-news.""" - channel = await self.bot.fetch_channel(constants.PythonNews.channel) - for new in news["data"][maillist]: message = discord.utils.get(self.bot.cached_messages, id=new) if message is None: - message = await channel.fetch_message(new) + message = await self.channel.fetch_message(new) if message is None: return False @@ -183,14 +176,13 @@ class News(Cog): return False async def send_webhook(self, - webhook: discord.Webhook, title: str, description: str, timestamp: datetime, url: str, - author: str, - author_url: str, - webhook_profile_name: str + webhook_profile_name: str, + author: t.Optional[str] = None, + author_url: t.Optional[str] = None, ) -> int: """Send webhook entry and return ID of message.""" embed = discord.Embed( @@ -200,11 +192,12 @@ class News(Cog): url=url, colour=constants.Colours.soft_green ) - embed.set_author( - name=author, - url=author_url - ) - msg = await webhook.send( + if author and author_url: + embed.set_author( + name=author, + url=author_url + ) + msg = await self.webhook.send( embed=embed, username=webhook_profile_name, avatar_url=AVATAR_URL, @@ -223,6 +216,12 @@ class News(Cog): email_information = await resp.json() return thread_information, email_information + async def get_webhook_and_channel(self) -> None: + """Storage #python-news channel Webhook and `TextChannel` to `News.webhook` and `channel`.""" + await self.bot.wait_until_guild_available() + self.webhook = await self.bot.fetch_webhook(constants.PythonNews.webhook) + self.channel = await self.bot.fetch_channel(constants.PythonNews.channel) + def setup(bot: Bot) -> None: """Add `News` cog.""" -- cgit v1.2.3 From cb3b2de26c654fa05816b72291e54762c42fad2c Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 21 Apr 2020 15:33:31 +0300 Subject: Simplified title check even more in PEP news --- bot/cogs/news.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index 21ddb6128..83b4989b3 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -97,7 +97,7 @@ class News(Cog): log.warning(f"Wrong datetime format passed in PEP new: {new['published']}") continue if ( - any(pep_new == new["title"] for pep_new in pep_news) + new["title"] in pep_news or new_datetime.date() < date.today() ): continue -- cgit v1.2.3 From 547de1af19038470e5c5a8f2120be40e197a97a8 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 27 Apr 2020 08:21:04 +0300 Subject: Improved `News` cog - Added footer to webhook sent message - Made `send_webhook` return `discord.Message` instead ID of message - Added waiting for Webhook on `send_webhook` - Added message publishing in new loops --- bot/cogs/news.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index 83b4989b3..be1284ca4 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -102,14 +102,17 @@ class News(Cog): ): continue - msg_id = await self.send_webhook( + msg = await self.send_webhook( title=new["title"], description=new["summary"], timestamp=new_datetime, url=new["link"], - webhook_profile_name=data["feed"]["title"] + webhook_profile_name=data["feed"]["title"], + footer=data["feed"]["title"] ) - payload["data"]["pep"].append(msg_id) + payload["data"]["pep"].append(msg.id) + + await msg.publish() # Apply new sent news to DB to avoid duplicate sending await self.bot.api_client.put("bot/bot-settings/news", json=payload) @@ -149,16 +152,19 @@ class News(Cog): content = email_information["content"] link = THREAD_URL.format(id=thread["href"].split("/")[-2], list=maillist) - msg_id = await self.send_webhook( + msg = await self.send_webhook( title=thread_information["subject"], description=content[:500] + f"... [continue reading]({link})" if len(content) > 500 else content, timestamp=new_date, url=link, author=f"{email_information['sender_name']} ({email_information['sender']['address']})", author_url=MAILMAN_PROFILE_URL.format(id=email_information["sender"]["mailman_id"]), - webhook_profile_name=self.webhook_names[maillist] + webhook_profile_name=self.webhook_names[maillist], + footer=f"Posted to {self.webhook_names[maillist]}" ) - payload["data"][maillist].append(msg_id) + payload["data"][maillist].append(msg.id) + + await msg.publish() await self.bot.api_client.put("bot/bot-settings/news", json=payload) @@ -181,10 +187,11 @@ class News(Cog): timestamp: datetime, url: str, webhook_profile_name: str, + footer: str, author: t.Optional[str] = None, author_url: t.Optional[str] = None, - ) -> int: - """Send webhook entry and return ID of message.""" + ) -> discord.Message: + """Send webhook entry and return sent message.""" embed = discord.Embed( title=title, description=description, @@ -197,13 +204,18 @@ class News(Cog): name=author, url=author_url ) - msg = await self.webhook.send( + embed.set_footer(text=footer, icon_url=AVATAR_URL) + + # Wait until Webhook is available + while not self.webhook: + pass + + return await self.webhook.send( embed=embed, username=webhook_profile_name, avatar_url=AVATAR_URL, wait=True ) - return msg.id async def get_thread_and_first_mail(self, maillist: str, thread_identifier: str) -> t.Tuple[t.Any, t.Any]: """Get mail thread and first mail from mail.python.org based on `maillist` and `thread_identifier`.""" -- cgit v1.2.3 From 07808f816aaf59beb2a3da6f115cd4b6577ea9c6 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 27 Apr 2020 09:17:23 +0300 Subject: Fixed `BeautifulSoup` parsing warning Added `features="lxml"` to `BeautifulSoup` class creating to avoid warning. --- bot/cogs/news.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index be1284ca4..db273d68d 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -127,7 +127,7 @@ class News(Cog): for maillist in constants.PythonNews.mail_lists: async with self.bot.http_session.get(RECENT_THREADS_TEMPLATE.format(name=maillist)) as resp: - recents = BeautifulSoup(await resp.text()) + recents = BeautifulSoup(await resp.text(), features="lxml") for thread in recents.html.body.div.find_all("a", href=True): # We want only these threads that have identifiers -- cgit v1.2.3 From 6ba5999089ca1a9d79e32dd7ceefbf3d865c35f9 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 27 Apr 2020 20:21:35 +0300 Subject: Add Python News channel and webhook ID to config-default.yml Co-Authored-By: Joseph --- config-default.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config-default.yml b/config-default.yml index 553afaa33..2cc15c370 100644 --- a/config-default.yml +++ b/config-default.yml @@ -122,7 +122,7 @@ guild: channels: announcements: 354619224620138496 user_event_announcements: &USER_EVENT_A 592000283102674944 - python_news: &PYNEWS_CHANNEL 701667765102051398 + python_news: &PYNEWS_CHANNEL 704372456592506880 # Development dev_contrib: &DEV_CONTRIB 635950537262759947 @@ -237,7 +237,7 @@ guild: reddit: 635408384794951680 duck_pond: 637821475327311927 dev_log: 680501655111729222 - python_news: &PYNEWS_WEBHOOK 701731296342179850 + python_news: &PYNEWS_WEBHOOK 704381182279942324 filter: -- cgit v1.2.3 From 12a7dc28589d2e26e2c843ee1364e9c183ec0035 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Mon, 27 Apr 2020 20:31:55 +0100 Subject: Make some fixes to ensure data is persisted and the bot does not hang --- bot/cogs/news.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index db273d68d..aa2b2ab8c 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -5,6 +5,7 @@ from datetime import date, datetime import discord import feedparser from bs4 import BeautifulSoup +from dateutil import tz from discord.ext.commands import Cog from discord.ext.tasks import loop @@ -35,6 +36,8 @@ class News(Cog): self.bot.loop.create_task(self.get_webhook_names()) self.bot.loop.create_task(self.get_webhook_and_channel()) + async def start_tasks(self) -> None: + """Start the tasks for fetching new PEPs and mailing list messages.""" self.post_pep_news.start() self.post_maillist_news.start() @@ -70,6 +73,7 @@ class News(Cog): """Fetch new PEPs and when they don't have announcement in #python-news, create it.""" # Wait until everything is ready and http_session available await self.bot.wait_until_guild_available() + await self.sync_maillists() async with self.bot.http_session.get(PEPS_RSS_URL) as resp: data = feedparser.parse(await resp.text()) @@ -112,7 +116,9 @@ class News(Cog): ) payload["data"]["pep"].append(msg.id) - await msg.publish() + if msg.channel.type is discord.ChannelType.news: + log.trace("Publishing PEP annnouncement because it was in a news channel") + await msg.publish() # Apply new sent news to DB to avoid duplicate sending await self.bot.api_client.put("bot/bot-settings/news", json=payload) @@ -164,7 +170,9 @@ class News(Cog): ) payload["data"][maillist].append(msg.id) - await msg.publish() + if msg.channel.type is discord.ChannelType.news: + log.trace("Publishing PEP annnouncement because it was in a news channel") + await msg.publish() await self.bot.api_client.put("bot/bot-settings/news", json=payload) @@ -175,10 +183,19 @@ class News(Cog): if message is None: message = await self.channel.fetch_message(new) if message is None: + log.trace(f"Could not find message for {new} on mailing list {maillist}") return False - if message.embeds[0].title == title and message.embeds[0].timestamp == timestamp: + embed_time = message.embeds[0].timestamp.replace(tzinfo=tz.gettz("UTC")) + + if ( + message.embeds[0].title == title + and embed_time == timestamp.astimezone(tz.gettz("UTC")) + ): + log.trace(f"Found existing message for '{title}'") return True + + log.trace(f"Found no existing message for '{title}'") return False async def send_webhook(self, @@ -234,6 +251,8 @@ class News(Cog): self.webhook = await self.bot.fetch_webhook(constants.PythonNews.webhook) self.channel = await self.bot.fetch_channel(constants.PythonNews.channel) + await self.start_tasks() + def setup(bot: Bot) -> None: """Add `News` cog.""" -- cgit v1.2.3 From 2cc1d3fc04a989fce1fd6da7d49c1c105678ef68 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Mon, 27 Apr 2020 20:33:28 +0100 Subject: Minor terminology change on a log --- bot/cogs/news.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index aa2b2ab8c..d7b2bcabb 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -171,7 +171,7 @@ class News(Cog): payload["data"][maillist].append(msg.id) if msg.channel.type is discord.ChannelType.news: - log.trace("Publishing PEP annnouncement because it was in a news channel") + log.trace("Publishing mailing list message because it was in a news channel") await msg.publish() await self.bot.api_client.put("bot/bot-settings/news", json=payload) -- cgit v1.2.3 From d49516c3d4231569f2f2ec6bde84299ded6fc2f4 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 28 Apr 2020 18:44:26 +0300 Subject: Simplified New publishing check + removed unnecessary Webhook check - Replaced type checking with `TextChannel.is_news()` for simplification to check is possible to publish new - Removed unnecessary `while` loop on `send_webhook` that check is webhook available. No need for this after starting ordering modification. --- bot/cogs/news.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index d7b2bcabb..66645bca7 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -116,7 +116,7 @@ class News(Cog): ) payload["data"]["pep"].append(msg.id) - if msg.channel.type is discord.ChannelType.news: + if msg.channel.is_news(): log.trace("Publishing PEP annnouncement because it was in a news channel") await msg.publish() @@ -170,7 +170,7 @@ class News(Cog): ) payload["data"][maillist].append(msg.id) - if msg.channel.type is discord.ChannelType.news: + if msg.channel.is_news(): log.trace("Publishing mailing list message because it was in a news channel") await msg.publish() @@ -223,10 +223,6 @@ class News(Cog): ) embed.set_footer(text=footer, icon_url=AVATAR_URL) - # Wait until Webhook is available - while not self.webhook: - pass - return await self.webhook.send( embed=embed, username=webhook_profile_name, -- cgit v1.2.3 From d498dd612f5f8252de6c09da045d7d91e2103555 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 1 May 2020 15:46:34 +0300 Subject: Replace message ID storage to new specific ID storage in `News` cog - Removed (now) unnecessary helper function `News.check_new_exist`. - Use thread IDs instead message IDs on maillists checking to avoid Discord API calls. - Use PEP number instead message IDs on PEP news checking to avoid Discord API calls. --- bot/cogs/news.py | 44 ++++++-------------------------------------- 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index 66645bca7..c5b89cf57 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -5,7 +5,6 @@ from datetime import date, datetime import discord import feedparser from bs4 import BeautifulSoup -from dateutil import tz from discord.ext.commands import Cog from discord.ext.tasks import loop @@ -80,17 +79,7 @@ class News(Cog): news_listing = await self.bot.api_client.get("bot/bot-settings/news") payload = news_listing.copy() - pep_news_ids = news_listing["data"]["pep"] - pep_news = [] - - for pep_id in pep_news_ids: - message = discord.utils.get(self.bot.cached_messages, id=pep_id) - if message is None: - message = await self.channel.fetch_message(pep_id) - if message is None: - log.warning("Can't fetch PEP new message ID.") - continue - pep_news.append(message.embeds[0].title) + pep_numbers = news_listing["data"]["pep"] # Reverse entries to send oldest first data["entries"].reverse() @@ -100,8 +89,9 @@ class News(Cog): except ValueError: log.warning(f"Wrong datetime format passed in PEP new: {new['published']}") continue + pep_nr = new["title"].split(":")[0].split()[1] if ( - new["title"] in pep_news + pep_nr in pep_numbers or new_datetime.date() < date.today() ): continue @@ -114,7 +104,7 @@ class News(Cog): webhook_profile_name=data["feed"]["title"], footer=data["feed"]["title"] ) - payload["data"]["pep"].append(msg.id) + payload["data"]["pep"].append(pep_nr) if msg.channel.is_news(): log.trace("Publishing PEP annnouncement because it was in a news channel") @@ -151,7 +141,7 @@ class News(Cog): continue if ( - await self.check_new_exist(thread_information["subject"], new_date, maillist, existing_news) + thread_information["thread_id"] in existing_news["data"][maillist] or new_date.date() < date.today() ): continue @@ -168,7 +158,7 @@ class News(Cog): webhook_profile_name=self.webhook_names[maillist], footer=f"Posted to {self.webhook_names[maillist]}" ) - payload["data"][maillist].append(msg.id) + payload["data"][maillist].append(thread_information["thread_id"]) if msg.channel.is_news(): log.trace("Publishing mailing list message because it was in a news channel") @@ -176,28 +166,6 @@ class News(Cog): await self.bot.api_client.put("bot/bot-settings/news", json=payload) - async def check_new_exist(self, title: str, timestamp: datetime, maillist: str, news: t.Dict[str, t.Any]) -> bool: - """Check does this new title + timestamp already exist in #python-news.""" - for new in news["data"][maillist]: - message = discord.utils.get(self.bot.cached_messages, id=new) - if message is None: - message = await self.channel.fetch_message(new) - if message is None: - log.trace(f"Could not find message for {new} on mailing list {maillist}") - return False - - embed_time = message.embeds[0].timestamp.replace(tzinfo=tz.gettz("UTC")) - - if ( - message.embeds[0].title == title - and embed_time == timestamp.astimezone(tz.gettz("UTC")) - ): - log.trace(f"Found existing message for '{title}'") - return True - - log.trace(f"Found no existing message for '{title}'") - return False - async def send_webhook(self, title: str, description: str, -- cgit v1.2.3 From 28fb3b83461f9375133ae8cfed6018f7b84c4a7e Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 1 May 2020 15:51:08 +0300 Subject: Added on cog unload news posting tasks canceling on `News` cog --- bot/cogs/news.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index c5b89cf57..ecc8edaf3 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -217,6 +217,11 @@ class News(Cog): await self.start_tasks() + def cog_unload(self) -> None: + """Stop news posting tasks on cog unload.""" + self.post_pep_news.cancel() + self.post_maillist_news.cancel() + def setup(bot: Bot) -> None: """Add `News` cog.""" -- cgit v1.2.3 From 5e55a34f3a3edcb041e6ea876055c7e593c707cc Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 1 May 2020 17:26:56 +0300 Subject: Added ignoring maillist when no recent threads (this month) in `News` cog --- bot/cogs/news.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index ecc8edaf3..ff2277283 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -125,6 +125,10 @@ class News(Cog): async with self.bot.http_session.get(RECENT_THREADS_TEMPLATE.format(name=maillist)) as resp: recents = BeautifulSoup(await resp.text(), features="lxml") + # When response have

, this mean that no threads available + if recents.p: + continue + for thread in recents.html.body.div.find_all("a", href=True): # We want only these threads that have identifiers if "latest" in thread["href"]: -- cgit v1.2.3 From 8647fd856fc23cf5f4162498f44bbcd9c576de44 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Fri, 1 May 2020 15:54:18 +0100 Subject: Merge the two asynchronous tasks into one to prevent race conditions --- bot/cogs/news.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index ff2277283..a81a50f21 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -37,8 +37,13 @@ class News(Cog): async def start_tasks(self) -> None: """Start the tasks for fetching new PEPs and mailing list messages.""" - self.post_pep_news.start() - self.post_maillist_news.start() + self.fetch_new_media.start() + + @loop(minutes=20) + async def fetch_new_media(self) -> None: + """Fetch new mailing list messages and then new PEPs.""" + await self.post_maillist_news() + await self.post_pep_news() async def sync_maillists(self) -> None: """Sync currently in-use maillists with API.""" @@ -67,7 +72,6 @@ class News(Cog): if mail["name"].split("@")[0] in constants.PythonNews.mail_lists: self.webhook_names[mail["name"].split("@")[0]] = mail["display_name"] - @loop(minutes=20) async def post_pep_news(self) -> None: """Fetch new PEPs and when they don't have announcement in #python-news, create it.""" # Wait until everything is ready and http_session available @@ -113,7 +117,6 @@ class News(Cog): # Apply new sent news to DB to avoid duplicate sending await self.bot.api_client.put("bot/bot-settings/news", json=payload) - @loop(minutes=20) async def post_maillist_news(self) -> None: """Send new maillist threads to #python-news that is listed in configuration.""" await self.bot.wait_until_guild_available() @@ -223,8 +226,7 @@ class News(Cog): def cog_unload(self) -> None: """Stop news posting tasks on cog unload.""" - self.post_pep_news.cancel() - self.post_maillist_news.cancel() + self.fetch_new_media.cancel() def setup(bot: Bot) -> None: -- cgit v1.2.3 From 79b6b1519c0c4b4bc46de12e76257d31338d0678 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 2 May 2020 08:59:56 +0300 Subject: Define encoding in `News` cog `await resp.text()` using In `News` cog PEP news posting, define `utf-8` as encoding on response parsing to avoid the error. Co-authored-by: Joseph Banks --- bot/cogs/news.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index a81a50f21..c716f662b 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -79,7 +79,7 @@ class News(Cog): await self.sync_maillists() async with self.bot.http_session.get(PEPS_RSS_URL) as resp: - data = feedparser.parse(await resp.text()) + data = feedparser.parse(await resp.text("utf-8")) news_listing = await self.bot.api_client.get("bot/bot-settings/news") payload = news_listing.copy() -- cgit v1.2.3 From 3a446f4419d8812df1d3892e43b50dd87fc26708 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sun, 3 May 2020 07:07:13 +0300 Subject: Fix `News` cog maillist news posting no threads check comment Co-authored-by: Joseph Banks --- bot/cogs/news.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bot/cogs/news.py b/bot/cogs/news.py index c716f662b..fc79f2fdc 100644 --- a/bot/cogs/news.py +++ b/bot/cogs/news.py @@ -128,7 +128,9 @@ class News(Cog): async with self.bot.http_session.get(RECENT_THREADS_TEMPLATE.format(name=maillist)) as resp: recents = BeautifulSoup(await resp.text(), features="lxml") - # When response have

, this mean that no threads available + # When a

element is present in the response then the mailing list + # has not had any activity during the current month, so therefore it + # can be ignored. if recents.p: continue -- cgit v1.2.3 From 2f924baafe3ae44b71bd60d1bac2c7f1cd98988b Mon Sep 17 00:00:00 2001 From: Daniel Brown Date: Mon, 4 May 2020 13:42:05 -0500 Subject: Perma Bans now Overwrite Temp Bans - Changed `has_active_infraction` to `get_active_infractions` in order to add additional logic in `apply_ban`. - Added `send_msg` parameters to `pardon_infraction` and `get_active_infractions` so that multi-step checks and actions don't need to send additional messages unless told to do so. Signed-off-by: Daniel Brown --- bot/cogs/moderation/infractions.py | 30 +++++++++++++++++++++++++++--- bot/cogs/moderation/scheduler.py | 17 ++++++++++++----- bot/cogs/moderation/superstarify.py | 2 +- bot/cogs/moderation/utils.py | 24 +++++++++++++++--------- 4 files changed, 55 insertions(+), 18 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index efa19f59e..29b4db20e 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -199,7 +199,7 @@ class Infractions(InfractionScheduler, commands.Cog): async def apply_mute(self, ctx: Context, user: Member, reason: str, **kwargs) -> None: """Apply a mute infraction with kwargs passed to `post_infraction`.""" - if await utils.has_active_infraction(ctx, user, "mute"): + if await utils.get_active_infractions(ctx, user, "mute"): return infraction = await utils.post_infraction(ctx, user, "mute", reason, active=True, **kwargs) @@ -235,8 +235,32 @@ class Infractions(InfractionScheduler, commands.Cog): Will also remove the banned user from the Big Brother watch list if applicable. """ - if await utils.has_active_infraction(ctx, user, "ban"): - return + # In the case of a permanent ban, we don't need get_active_infractions to tell us if one is active + send_msg = "expires_at" in kwargs + active_infraction = await utils.get_active_infractions(ctx, user, "ban", send_msg) + + if active_infraction: + log.trace("Active infractions found.") + if ( + active_infraction.get('expires_at') is not None + and kwargs.get('expires_at') is None + ): + log.trace("Active ban is a temporary and being called by a perma. Removing temporary.") + await self.pardon_infraction(ctx, "ban", user, send_msg) + + elif ( + active_infraction.get('expires_at') is None + and kwargs.get('expires_at') is None + ): + log.trace("Active ban is a perma ban and being called by a perma. Send bounce back message.") + await ctx.send( + f":x: According to my records, this user is already permanently banned. " + f"See infraction **#{active_infraction['id']}**." + ) + return + else: + log.trace("Active ban is a temp ban being called by a temp or a perma being called by a temp. Ignore.") + return infraction = await utils.post_infraction(ctx, user, "ban", reason, active=True, **kwargs) if infraction is None: diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 917697be9..413717fb6 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -190,7 +190,13 @@ class InfractionScheduler(Scheduler): log.info(f"Applied {infr_type} infraction #{id_} to {user}.") - async def pardon_infraction(self, ctx: Context, infr_type: str, user: UserSnowflake) -> None: + async def pardon_infraction( + self, + ctx: Context, + infr_type: str, + user: UserSnowflake, + send_msg: bool = True + ) -> None: """Prematurely end an infraction for a user and log the action in the mod log.""" log.trace(f"Pardoning {infr_type} infraction for {user}.") @@ -277,10 +283,11 @@ class InfractionScheduler(Scheduler): # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} pardon confirmation message.") - await ctx.send( - f"{dm_emoji}{confirm_msg} infraction **{infr_type}** for {user.mention}. " - f"{log_text.get('Failure', '')}" - ) + if send_msg: + await ctx.send( + f"{dm_emoji}{confirm_msg} infraction **{infr_type}** for {user.mention}. " + f"{log_text.get('Failure', '')}" + ) # Send a log message to the mod log. await self.mod_log.send_log_message( diff --git a/bot/cogs/moderation/superstarify.py b/bot/cogs/moderation/superstarify.py index ca3dc4202..272f7c4f0 100644 --- a/bot/cogs/moderation/superstarify.py +++ b/bot/cogs/moderation/superstarify.py @@ -130,7 +130,7 @@ class Superstarify(InfractionScheduler, Cog): An optional reason can be provided. If no reason is given, the original name will be shown in a generated reason. """ - if await utils.has_active_infraction(ctx, member, "superstar"): + if await utils.get_active_infractions(ctx, member, "superstar"): return # Post the infraction to the API diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py index 3598f3b1f..406f9d08a 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/cogs/moderation/utils.py @@ -97,8 +97,13 @@ async def post_infraction( return -async def has_active_infraction(ctx: Context, user: UserSnowflake, infr_type: str) -> bool: - """Checks if a user already has an active infraction of the given type.""" +async def get_active_infractions( + ctx: Context, + user: UserSnowflake, + infr_type: str, + send_msg: bool = True +) -> t.Optional[dict]: + """Retrieves active infractions of the given type for the user.""" log.trace(f"Checking if {user} has active infractions of type {infr_type}.") active_infractions = await ctx.bot.api_client.get( @@ -110,15 +115,16 @@ async def has_active_infraction(ctx: Context, user: UserSnowflake, infr_type: st } ) if active_infractions: - log.trace(f"{user} has active infractions of type {infr_type}.") - await ctx.send( - f":x: According to my records, this user already has a {infr_type} infraction. " - f"See infraction **#{active_infractions[0]['id']}**." - ) - return True + # Checks to see if the moderator should be told there is an active infraction + if send_msg: + log.trace(f"{user} has active infractions of type {infr_type}.") + await ctx.send( + f":x: According to my records, this user already has a {infr_type} infraction. " + f"See infraction **#{active_infractions[0]['id']}**." + ) + return active_infractions[0] else: log.trace(f"{user} does not have active infractions of type {infr_type}.") - return False async def notify_infraction( -- cgit v1.2.3 From 2368ab4d6f107868c40036438a7320a10d3c0184 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 6 May 2020 19:06:34 +0300 Subject: Fix config Webhook IDs formatting Co-authored-by: Sebastiaan Zeeff --- config-default.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config-default.yml b/config-default.yml index 2cc15c370..90d5b1d29 100644 --- a/config-default.yml +++ b/config-default.yml @@ -232,12 +232,12 @@ guild: - *HELPERS_ROLE webhooks: - talent_pool: 569145364800602132 - big_brother: 569133704568373283 - reddit: 635408384794951680 - duck_pond: 637821475327311927 - dev_log: 680501655111729222 - python_news: &PYNEWS_WEBHOOK 704381182279942324 + talent_pool: 569145364800602132 + big_brother: 569133704568373283 + reddit: 635408384794951680 + duck_pond: 637821475327311927 + dev_log: 680501655111729222 + python_news: &PYNEWS_WEBHOOK 704381182279942324 filter: -- cgit v1.2.3 From bcb360b262531e14eb00bfdc36d9da9b260fdcff Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 6 May 2020 19:18:23 +0300 Subject: Renamed `news.py` to `python_news.py` and `News` to `PythonNews` to avoid confusion --- bot/cogs/news.py | 232 ------------------------------------------------ bot/cogs/python_news.py | 232 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 232 insertions(+), 232 deletions(-) delete mode 100644 bot/cogs/news.py create mode 100644 bot/cogs/python_news.py diff --git a/bot/cogs/news.py b/bot/cogs/news.py deleted file mode 100644 index ff2277283..000000000 --- a/bot/cogs/news.py +++ /dev/null @@ -1,232 +0,0 @@ -import logging -import typing as t -from datetime import date, datetime - -import discord -import feedparser -from bs4 import BeautifulSoup -from discord.ext.commands import Cog -from discord.ext.tasks import loop - -from bot import constants -from bot.bot import Bot - -PEPS_RSS_URL = "https://www.python.org/dev/peps/peps.rss/" - -RECENT_THREADS_TEMPLATE = "https://mail.python.org/archives/list/{name}@python.org/recent-threads" -THREAD_TEMPLATE_URL = "https://mail.python.org/archives/api/list/{name}@python.org/thread/{id}/" -MAILMAN_PROFILE_URL = "https://mail.python.org/archives/users/{id}/" -THREAD_URL = "https://mail.python.org/archives/list/{list}@python.org/thread/{id}/" - -AVATAR_URL = "https://www.python.org/static/opengraph-icon-200x200.png" - -log = logging.getLogger(__name__) - - -class News(Cog): - """Post new PEPs and Python News to `#python-news`.""" - - def __init__(self, bot: Bot): - self.bot = bot - self.webhook_names = {} - self.webhook: t.Optional[discord.Webhook] = None - self.channel: t.Optional[discord.TextChannel] = None - - self.bot.loop.create_task(self.get_webhook_names()) - self.bot.loop.create_task(self.get_webhook_and_channel()) - - async def start_tasks(self) -> None: - """Start the tasks for fetching new PEPs and mailing list messages.""" - self.post_pep_news.start() - self.post_maillist_news.start() - - async def sync_maillists(self) -> None: - """Sync currently in-use maillists with API.""" - # Wait until guild is available to avoid running before everything is ready - await self.bot.wait_until_guild_available() - - response = await self.bot.api_client.get("bot/bot-settings/news") - for mail in constants.PythonNews.mail_lists: - if mail not in response["data"]: - response["data"][mail] = [] - - # Because we are handling PEPs differently, we don't include it to mail lists - if "pep" not in response["data"]: - response["data"]["pep"] = [] - - await self.bot.api_client.put("bot/bot-settings/news", json=response) - - async def get_webhook_names(self) -> None: - """Get webhook author names from maillist API.""" - await self.bot.wait_until_guild_available() - - async with self.bot.http_session.get("https://mail.python.org/archives/api/lists") as resp: - lists = await resp.json() - - for mail in lists: - if mail["name"].split("@")[0] in constants.PythonNews.mail_lists: - self.webhook_names[mail["name"].split("@")[0]] = mail["display_name"] - - @loop(minutes=20) - async def post_pep_news(self) -> None: - """Fetch new PEPs and when they don't have announcement in #python-news, create it.""" - # Wait until everything is ready and http_session available - await self.bot.wait_until_guild_available() - await self.sync_maillists() - - async with self.bot.http_session.get(PEPS_RSS_URL) as resp: - data = feedparser.parse(await resp.text()) - - news_listing = await self.bot.api_client.get("bot/bot-settings/news") - payload = news_listing.copy() - pep_numbers = news_listing["data"]["pep"] - - # Reverse entries to send oldest first - data["entries"].reverse() - for new in data["entries"]: - try: - new_datetime = datetime.strptime(new["published"], "%a, %d %b %Y %X %Z") - except ValueError: - log.warning(f"Wrong datetime format passed in PEP new: {new['published']}") - continue - pep_nr = new["title"].split(":")[0].split()[1] - if ( - pep_nr in pep_numbers - or new_datetime.date() < date.today() - ): - continue - - msg = await self.send_webhook( - title=new["title"], - description=new["summary"], - timestamp=new_datetime, - url=new["link"], - webhook_profile_name=data["feed"]["title"], - footer=data["feed"]["title"] - ) - payload["data"]["pep"].append(pep_nr) - - if msg.channel.is_news(): - log.trace("Publishing PEP annnouncement because it was in a news channel") - await msg.publish() - - # Apply new sent news to DB to avoid duplicate sending - await self.bot.api_client.put("bot/bot-settings/news", json=payload) - - @loop(minutes=20) - async def post_maillist_news(self) -> None: - """Send new maillist threads to #python-news that is listed in configuration.""" - await self.bot.wait_until_guild_available() - await self.sync_maillists() - existing_news = await self.bot.api_client.get("bot/bot-settings/news") - payload = existing_news.copy() - - for maillist in constants.PythonNews.mail_lists: - async with self.bot.http_session.get(RECENT_THREADS_TEMPLATE.format(name=maillist)) as resp: - recents = BeautifulSoup(await resp.text(), features="lxml") - - # When response have

, this mean that no threads available - if recents.p: - continue - - for thread in recents.html.body.div.find_all("a", href=True): - # We want only these threads that have identifiers - if "latest" in thread["href"]: - continue - - thread_information, email_information = await self.get_thread_and_first_mail( - maillist, thread["href"].split("/")[-2] - ) - - try: - new_date = datetime.strptime(email_information["date"], "%Y-%m-%dT%X%z") - except ValueError: - log.warning(f"Invalid datetime from Thread email: {email_information['date']}") - continue - - if ( - thread_information["thread_id"] in existing_news["data"][maillist] - or new_date.date() < date.today() - ): - continue - - content = email_information["content"] - link = THREAD_URL.format(id=thread["href"].split("/")[-2], list=maillist) - msg = await self.send_webhook( - title=thread_information["subject"], - description=content[:500] + f"... [continue reading]({link})" if len(content) > 500 else content, - timestamp=new_date, - url=link, - author=f"{email_information['sender_name']} ({email_information['sender']['address']})", - author_url=MAILMAN_PROFILE_URL.format(id=email_information["sender"]["mailman_id"]), - webhook_profile_name=self.webhook_names[maillist], - footer=f"Posted to {self.webhook_names[maillist]}" - ) - payload["data"][maillist].append(thread_information["thread_id"]) - - if msg.channel.is_news(): - log.trace("Publishing mailing list message because it was in a news channel") - await msg.publish() - - await self.bot.api_client.put("bot/bot-settings/news", json=payload) - - async def send_webhook(self, - title: str, - description: str, - timestamp: datetime, - url: str, - webhook_profile_name: str, - footer: str, - author: t.Optional[str] = None, - author_url: t.Optional[str] = None, - ) -> discord.Message: - """Send webhook entry and return sent message.""" - embed = discord.Embed( - title=title, - description=description, - timestamp=timestamp, - url=url, - colour=constants.Colours.soft_green - ) - if author and author_url: - embed.set_author( - name=author, - url=author_url - ) - embed.set_footer(text=footer, icon_url=AVATAR_URL) - - return await self.webhook.send( - embed=embed, - username=webhook_profile_name, - avatar_url=AVATAR_URL, - wait=True - ) - - async def get_thread_and_first_mail(self, maillist: str, thread_identifier: str) -> t.Tuple[t.Any, t.Any]: - """Get mail thread and first mail from mail.python.org based on `maillist` and `thread_identifier`.""" - async with self.bot.http_session.get( - THREAD_TEMPLATE_URL.format(name=maillist, id=thread_identifier) - ) as resp: - thread_information = await resp.json() - - async with self.bot.http_session.get(thread_information["starting_email"]) as resp: - email_information = await resp.json() - return thread_information, email_information - - async def get_webhook_and_channel(self) -> None: - """Storage #python-news channel Webhook and `TextChannel` to `News.webhook` and `channel`.""" - await self.bot.wait_until_guild_available() - self.webhook = await self.bot.fetch_webhook(constants.PythonNews.webhook) - self.channel = await self.bot.fetch_channel(constants.PythonNews.channel) - - await self.start_tasks() - - def cog_unload(self) -> None: - """Stop news posting tasks on cog unload.""" - self.post_pep_news.cancel() - self.post_maillist_news.cancel() - - -def setup(bot: Bot) -> None: - """Add `News` cog.""" - bot.add_cog(News(bot)) diff --git a/bot/cogs/python_news.py b/bot/cogs/python_news.py new file mode 100644 index 000000000..092ee3cff --- /dev/null +++ b/bot/cogs/python_news.py @@ -0,0 +1,232 @@ +import logging +import typing as t +from datetime import date, datetime + +import discord +import feedparser +from bs4 import BeautifulSoup +from discord.ext.commands import Cog +from discord.ext.tasks import loop + +from bot import constants +from bot.bot import Bot + +PEPS_RSS_URL = "https://www.python.org/dev/peps/peps.rss/" + +RECENT_THREADS_TEMPLATE = "https://mail.python.org/archives/list/{name}@python.org/recent-threads" +THREAD_TEMPLATE_URL = "https://mail.python.org/archives/api/list/{name}@python.org/thread/{id}/" +MAILMAN_PROFILE_URL = "https://mail.python.org/archives/users/{id}/" +THREAD_URL = "https://mail.python.org/archives/list/{list}@python.org/thread/{id}/" + +AVATAR_URL = "https://www.python.org/static/opengraph-icon-200x200.png" + +log = logging.getLogger(__name__) + + +class PythonNews(Cog): + """Post new PEPs and Python News to `#python-news`.""" + + def __init__(self, bot: Bot): + self.bot = bot + self.webhook_names = {} + self.webhook: t.Optional[discord.Webhook] = None + self.channel: t.Optional[discord.TextChannel] = None + + self.bot.loop.create_task(self.get_webhook_names()) + self.bot.loop.create_task(self.get_webhook_and_channel()) + + async def start_tasks(self) -> None: + """Start the tasks for fetching new PEPs and mailing list messages.""" + self.post_pep_news.start() + self.post_maillist_news.start() + + async def sync_maillists(self) -> None: + """Sync currently in-use maillists with API.""" + # Wait until guild is available to avoid running before everything is ready + await self.bot.wait_until_guild_available() + + response = await self.bot.api_client.get("bot/bot-settings/news") + for mail in constants.PythonNews.mail_lists: + if mail not in response["data"]: + response["data"][mail] = [] + + # Because we are handling PEPs differently, we don't include it to mail lists + if "pep" not in response["data"]: + response["data"]["pep"] = [] + + await self.bot.api_client.put("bot/bot-settings/news", json=response) + + async def get_webhook_names(self) -> None: + """Get webhook author names from maillist API.""" + await self.bot.wait_until_guild_available() + + async with self.bot.http_session.get("https://mail.python.org/archives/api/lists") as resp: + lists = await resp.json() + + for mail in lists: + if mail["name"].split("@")[0] in constants.PythonNews.mail_lists: + self.webhook_names[mail["name"].split("@")[0]] = mail["display_name"] + + @loop(minutes=20) + async def post_pep_news(self) -> None: + """Fetch new PEPs and when they don't have announcement in #python-news, create it.""" + # Wait until everything is ready and http_session available + await self.bot.wait_until_guild_available() + await self.sync_maillists() + + async with self.bot.http_session.get(PEPS_RSS_URL) as resp: + data = feedparser.parse(await resp.text()) + + news_listing = await self.bot.api_client.get("bot/bot-settings/news") + payload = news_listing.copy() + pep_numbers = news_listing["data"]["pep"] + + # Reverse entries to send oldest first + data["entries"].reverse() + for new in data["entries"]: + try: + new_datetime = datetime.strptime(new["published"], "%a, %d %b %Y %X %Z") + except ValueError: + log.warning(f"Wrong datetime format passed in PEP new: {new['published']}") + continue + pep_nr = new["title"].split(":")[0].split()[1] + if ( + pep_nr in pep_numbers + or new_datetime.date() < date.today() + ): + continue + + msg = await self.send_webhook( + title=new["title"], + description=new["summary"], + timestamp=new_datetime, + url=new["link"], + webhook_profile_name=data["feed"]["title"], + footer=data["feed"]["title"] + ) + payload["data"]["pep"].append(pep_nr) + + if msg.channel.is_news(): + log.trace("Publishing PEP annnouncement because it was in a news channel") + await msg.publish() + + # Apply new sent news to DB to avoid duplicate sending + await self.bot.api_client.put("bot/bot-settings/news", json=payload) + + @loop(minutes=20) + async def post_maillist_news(self) -> None: + """Send new maillist threads to #python-news that is listed in configuration.""" + await self.bot.wait_until_guild_available() + await self.sync_maillists() + existing_news = await self.bot.api_client.get("bot/bot-settings/news") + payload = existing_news.copy() + + for maillist in constants.PythonNews.mail_lists: + async with self.bot.http_session.get(RECENT_THREADS_TEMPLATE.format(name=maillist)) as resp: + recents = BeautifulSoup(await resp.text(), features="lxml") + + # When response have

, this mean that no threads available + if recents.p: + continue + + for thread in recents.html.body.div.find_all("a", href=True): + # We want only these threads that have identifiers + if "latest" in thread["href"]: + continue + + thread_information, email_information = await self.get_thread_and_first_mail( + maillist, thread["href"].split("/")[-2] + ) + + try: + new_date = datetime.strptime(email_information["date"], "%Y-%m-%dT%X%z") + except ValueError: + log.warning(f"Invalid datetime from Thread email: {email_information['date']}") + continue + + if ( + thread_information["thread_id"] in existing_news["data"][maillist] + or new_date.date() < date.today() + ): + continue + + content = email_information["content"] + link = THREAD_URL.format(id=thread["href"].split("/")[-2], list=maillist) + msg = await self.send_webhook( + title=thread_information["subject"], + description=content[:500] + f"... [continue reading]({link})" if len(content) > 500 else content, + timestamp=new_date, + url=link, + author=f"{email_information['sender_name']} ({email_information['sender']['address']})", + author_url=MAILMAN_PROFILE_URL.format(id=email_information["sender"]["mailman_id"]), + webhook_profile_name=self.webhook_names[maillist], + footer=f"Posted to {self.webhook_names[maillist]}" + ) + payload["data"][maillist].append(thread_information["thread_id"]) + + if msg.channel.is_news(): + log.trace("Publishing mailing list message because it was in a news channel") + await msg.publish() + + await self.bot.api_client.put("bot/bot-settings/news", json=payload) + + async def send_webhook(self, + title: str, + description: str, + timestamp: datetime, + url: str, + webhook_profile_name: str, + footer: str, + author: t.Optional[str] = None, + author_url: t.Optional[str] = None, + ) -> discord.Message: + """Send webhook entry and return sent message.""" + embed = discord.Embed( + title=title, + description=description, + timestamp=timestamp, + url=url, + colour=constants.Colours.soft_green + ) + if author and author_url: + embed.set_author( + name=author, + url=author_url + ) + embed.set_footer(text=footer, icon_url=AVATAR_URL) + + return await self.webhook.send( + embed=embed, + username=webhook_profile_name, + avatar_url=AVATAR_URL, + wait=True + ) + + async def get_thread_and_first_mail(self, maillist: str, thread_identifier: str) -> t.Tuple[t.Any, t.Any]: + """Get mail thread and first mail from mail.python.org based on `maillist` and `thread_identifier`.""" + async with self.bot.http_session.get( + THREAD_TEMPLATE_URL.format(name=maillist, id=thread_identifier) + ) as resp: + thread_information = await resp.json() + + async with self.bot.http_session.get(thread_information["starting_email"]) as resp: + email_information = await resp.json() + return thread_information, email_information + + async def get_webhook_and_channel(self) -> None: + """Storage #python-news channel Webhook and `TextChannel` to `News.webhook` and `channel`.""" + await self.bot.wait_until_guild_available() + self.webhook = await self.bot.fetch_webhook(constants.PythonNews.webhook) + self.channel = await self.bot.fetch_channel(constants.PythonNews.channel) + + await self.start_tasks() + + def cog_unload(self) -> None: + """Stop news posting tasks on cog unload.""" + self.post_pep_news.cancel() + self.post_maillist_news.cancel() + + +def setup(bot: Bot) -> None: + """Add `News` cog.""" + bot.add_cog(PythonNews(bot)) -- cgit v1.2.3 From 442d7199eef085910370506f91fda5bf0ef737a5 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 6 May 2020 19:20:51 +0300 Subject: Remove `PythonNews.channel` because this is unnecessary --- bot/cogs/python_news.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bot/cogs/python_news.py b/bot/cogs/python_news.py index 092ee3cff..e3c8b1e1e 100644 --- a/bot/cogs/python_news.py +++ b/bot/cogs/python_news.py @@ -30,7 +30,6 @@ class PythonNews(Cog): self.bot = bot self.webhook_names = {} self.webhook: t.Optional[discord.Webhook] = None - self.channel: t.Optional[discord.TextChannel] = None self.bot.loop.create_task(self.get_webhook_names()) self.bot.loop.create_task(self.get_webhook_and_channel()) @@ -217,7 +216,6 @@ class PythonNews(Cog): """Storage #python-news channel Webhook and `TextChannel` to `News.webhook` and `channel`.""" await self.bot.wait_until_guild_available() self.webhook = await self.bot.fetch_webhook(constants.PythonNews.webhook) - self.channel = await self.bot.fetch_channel(constants.PythonNews.channel) await self.start_tasks() -- cgit v1.2.3 From 601ff03823deb842d74f4689fecb68f7ce1693e6 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 18:11:44 +0200 Subject: AntiMalware Tests - Added unittest for message without attachment --- tests/bot/cogs/test_antimalware.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/bot/cogs/test_antimalware.py diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py new file mode 100644 index 000000000..41ca19e17 --- /dev/null +++ b/tests/bot/cogs/test_antimalware.py @@ -0,0 +1,20 @@ +import asyncio +import unittest + +from bot.cogs import antimalware +from tests.helpers import MockBot, MockMessage + + +class AntiMalwareCogTests(unittest.TestCase): + """Test the AntiMalware cog.""" + + def setUp(self): + """Sets up fresh objects for each test.""" + self.bot = MockBot() + self.cog = antimalware.AntiMalware(self.bot) + self.message = MockMessage() + + def test_message_without_attachment(self): + """Messages without attachments should result in no action.""" + coroutine = self.cog.on_message(self.message) + self.assertIsNone(asyncio.run(coroutine)) -- cgit v1.2.3 From 9889f0fdd1ba403ae50ba20be38feca0932d1dda Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 19:06:46 +0200 Subject: AntiMalware Tests - Added unittests for deletion of message and ignoring of dms --- tests/bot/cogs/test_antimalware.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 41ca19e17..ebf3a1277 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,8 +1,9 @@ import asyncio import unittest +from unittest.mock import AsyncMock from bot.cogs import antimalware -from tests.helpers import MockBot, MockMessage +from tests.helpers import MockAttachment, MockBot, MockMessage class AntiMalwareCogTests(unittest.TestCase): @@ -13,8 +14,27 @@ class AntiMalwareCogTests(unittest.TestCase): self.bot = MockBot() self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() + self.message.delete = AsyncMock() def test_message_without_attachment(self): """Messages without attachments should result in no action.""" coroutine = self.cog.on_message(self.message) self.assertIsNone(asyncio.run(coroutine)) + self.message.delete.assert_not_called() + + def test_direct_message_with_attachment(self): + """Direct messages should have no action taken.""" + attachment = MockAttachment(filename="python.asdfsff") + self.message.attachments = [attachment] + self.message.guild = None + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + self.message.delete.assert_not_called() + + def test_message_with_illegal_extension_gets_deleted(self): + """A message containing an illegal extension should send an embed.""" + attachment = MockAttachment(filename="python.asdfsff") + self.message.attachments = [attachment] + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + self.message.delete.assert_called_once() -- cgit v1.2.3 From 90d2ce0e3717d4ddf79eb986e22f7542ca1770e1 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 19:13:45 +0200 Subject: AntiMalware Tests - Added unittest for messages send by staff --- tests/bot/cogs/test_antimalware.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index ebf3a1277..e3fd477fa 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -3,7 +3,8 @@ import unittest from unittest.mock import AsyncMock from bot.cogs import antimalware -from tests.helpers import MockAttachment, MockBot, MockMessage +from bot.constants import Roles +from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole class AntiMalwareCogTests(unittest.TestCase): @@ -38,3 +39,13 @@ class AntiMalwareCogTests(unittest.TestCase): coroutine = self.cog.on_message(self.message) asyncio.run(coroutine) self.message.delete.assert_called_once() + + def test_message_send_by_staff(self): + """A message send by a member of staff should be ignored.""" + moderator_role = MockRole(name="Moderator", id=Roles.moderators) + self.message.author.roles.append(moderator_role) + attachment = MockAttachment(filename="python.asdfsff") + self.message.attachments = [attachment] + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + self.message.delete.assert_not_called() -- cgit v1.2.3 From 3913a8eba46bf98bd09e13145da33f7a09f77960 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 19:45:57 +0200 Subject: AntiMalware Tests - Added unittest for the embed for a python file. --- tests/bot/cogs/test_antimalware.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index e3fd477fa..0bb5af943 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -3,7 +3,7 @@ import unittest from unittest.mock import AsyncMock from bot.cogs import antimalware -from bot.constants import Roles +from bot.constants import Roles, URLs from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole @@ -28,16 +28,20 @@ class AntiMalwareCogTests(unittest.TestCase): attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] self.message.guild = None + coroutine = self.cog.on_message(self.message) asyncio.run(coroutine) + self.message.delete.assert_not_called() def test_message_with_illegal_extension_gets_deleted(self): """A message containing an illegal extension should send an embed.""" attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] + coroutine = self.cog.on_message(self.message) asyncio.run(coroutine) + self.message.delete.assert_called_once() def test_message_send_by_staff(self): @@ -46,6 +50,25 @@ class AntiMalwareCogTests(unittest.TestCase): self.message.author.roles.append(moderator_role) attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] + coroutine = self.cog.on_message(self.message) asyncio.run(coroutine) + self.message.delete.assert_not_called() + + def test_python_file_redirect_embed(self): + """A message containing a .python file should result in an embed redirecting the user to our paste site""" + attachment = MockAttachment(filename="python.py") + self.message.attachments = [attachment] + self.message.channel.send = AsyncMock() + + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + args, kwargs = self.message.channel.send.call_args + embed = kwargs.pop("embed") + + self.assertEqual(args[0], f"Hey {self.message.author.mention}!") + self.assertEqual(embed.description, ( + "It looks like you tried to attach a Python file - " + f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" + )) -- cgit v1.2.3 From 19c15d957040b6857a4141e15c32fd0526f9920d Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 20:15:17 +0200 Subject: AntiMalware Tests - Added unittest for messages that were deleted in the meantime. --- tests/bot/cogs/test_antimalware.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 0bb5af943..da5cd9d11 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,6 +1,9 @@ import asyncio +import logging import unittest -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, Mock + +from discord import NotFound from bot.cogs import antimalware from bot.constants import Roles, URLs @@ -72,3 +75,18 @@ class AntiMalwareCogTests(unittest.TestCase): "It looks like you tried to attach a Python file - " f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" )) + + def test_removing_deleted_message_logs(self): + """Removing an already deleted message logs the correct message""" + attachment = MockAttachment(filename="python.py") + self.message.attachments = [attachment] + self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) + + coroutine = self.cog.on_message(self.message) + logger = logging.getLogger("bot.cogs.antimalware") + + with self.assertLogs(logger=logger, level="INFO") as logs: + asyncio.run(coroutine) + self.assertIn( + f"INFO:bot.cogs.antimalware:Tried to delete message `{self.message.id}`, but message could not be found.", + logs.output) -- cgit v1.2.3 From 4a0b3ea1ef182ddbbb1f9d731b28768a049a531d Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 20:23:00 +0200 Subject: AntiMalware Tests - Added unittest for cog setup --- tests/bot/cogs/test_antimalware.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index da5cd9d11..67c640d23 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -90,3 +90,13 @@ class AntiMalwareCogTests(unittest.TestCase): self.assertIn( f"INFO:bot.cogs.antimalware:Tried to delete message `{self.message.id}`, but message could not be found.", logs.output) + + +class AntiMalwareSetupTests(unittest.TestCase): + """Tests setup of the `AntiMalware` cog.""" + + def test_setup(self): + """Setup of the extension should call add_cog.""" + bot = MockBot() + antimalware.setup(bot) + bot.add_cog.assert_called_once() -- cgit v1.2.3 From 3090141f673279f2836cb3aca95397eb9950ad0f Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 20:41:31 +0200 Subject: AntiMalware Tests - Added unittest message deletion log --- tests/bot/cogs/test_antimalware.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 67c640d23..b4e31b5ce 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,14 +1,17 @@ import asyncio import logging import unittest +from os.path import splitext from unittest.mock import AsyncMock, Mock from discord import NotFound from bot.cogs import antimalware -from bot.constants import Roles, URLs +from bot.constants import AntiMalware as AntiMalwareConfig, Roles, URLs from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole +MODULE = "bot.cogs.antimalware" + class AntiMalwareCogTests(unittest.TestCase): """Test the AntiMalware cog.""" @@ -78,17 +81,38 @@ class AntiMalwareCogTests(unittest.TestCase): def test_removing_deleted_message_logs(self): """Removing an already deleted message logs the correct message""" - attachment = MockAttachment(filename="python.py") + attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) coroutine = self.cog.on_message(self.message) - logger = logging.getLogger("bot.cogs.antimalware") + logger = logging.getLogger(MODULE) with self.assertLogs(logger=logger, level="INFO") as logs: asyncio.run(coroutine) self.assertIn( - f"INFO:bot.cogs.antimalware:Tried to delete message `{self.message.id}`, but message could not be found.", + f"INFO:{MODULE}:Tried to delete message `{self.message.id}`, but message could not be found.", + logs.output) + + def test_message_with_illegal_attachment_logs(self): + """Deleting a message with an illegal attachment should result in a log.""" + attachment = MockAttachment(filename="python.asdfsff") + self.message.attachments = [attachment] + + coroutine = self.cog.on_message(self.message) + file_extensions = {splitext(attachment.filename.lower())[1] for attachment in self.message.attachments} + extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) + blocked_extensions_str = ', '.join(extensions_blocked) + logger = logging.getLogger(MODULE) + + with self.assertLogs(logger=logger, level="INFO") as logs: + asyncio.run(coroutine) + self.assertEqual( + [ + f"INFO:{MODULE}:" + f"User '{self.message.author}' ({self.message.author.id}) " + f"uploaded blacklisted file(s): {blocked_extensions_str}" + ], logs.output) -- cgit v1.2.3 From f0bc9d800dd141b9126c48251a80618e138d61f1 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 20:46:15 +0200 Subject: AntiMalware Tests - Added unittest for valid attachment --- tests/bot/cogs/test_antimalware.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index b4e31b5ce..407fa05c1 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -23,6 +23,15 @@ class AntiMalwareCogTests(unittest.TestCase): self.message = MockMessage() self.message.delete = AsyncMock() + def test_message_with_allowed_attachment(self): + """Messages with allowed extensions should not be deleted""" + attachment = MockAttachment(filename=f"python.{AntiMalwareConfig.whitelist[0]}") + self.message.attachments = [attachment] + + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + self.message.delete.assert_not_called() + def test_message_without_attachment(self): """Messages without attachments should result in no action.""" coroutine = self.cog.on_message(self.message) -- cgit v1.2.3 From 5d0cecf514701e9e300174e9d3050bd772f3f96f Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 7 May 2020 21:47:20 +0300 Subject: Update Python News extension name in __main__.py Co-authored-by: Joseph Banks --- bot/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/__main__.py b/bot/__main__.py index 42c1a4f3a..aa1d1aee8 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -51,7 +51,7 @@ bot.load_extension("bot.cogs.eval") bot.load_extension("bot.cogs.information") bot.load_extension("bot.cogs.jams") bot.load_extension("bot.cogs.moderation") -bot.load_extension("bot.cogs.news") +bot.load_extension("bot.cogs.python_news") bot.load_extension("bot.cogs.off_topic_names") bot.load_extension("bot.cogs.reddit") bot.load_extension("bot.cogs.reminders") -- cgit v1.2.3 From 75f6ca6bd9b695a5deb4a4d78311bc63eb2a74d0 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 21:04:47 +0200 Subject: AntiMalware Tests - Added unittest for txt file attachment --- tests/bot/cogs/test_antimalware.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 407fa05c1..eba439afb 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -7,7 +7,7 @@ from unittest.mock import AsyncMock, Mock from discord import NotFound from bot.cogs import antimalware -from bot.constants import AntiMalware as AntiMalwareConfig, Roles, URLs +from bot.constants import AntiMalware as AntiMalwareConfig, Channels, Roles, URLs from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole MODULE = "bot.cogs.antimalware" @@ -21,7 +21,6 @@ class AntiMalwareCogTests(unittest.TestCase): self.bot = MockBot() self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() - self.message.delete = AsyncMock() def test_message_with_allowed_attachment(self): """Messages with allowed extensions should not be deleted""" @@ -88,6 +87,28 @@ class AntiMalwareCogTests(unittest.TestCase): f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" )) + def test_txt_file_redirect_embed(self): + attachment = MockAttachment(filename="python.txt") + self.message.attachments = [attachment] + self.message.channel.send = AsyncMock() + + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + args, kwargs = self.message.channel.send.call_args + embed = kwargs.pop("embed") + cmd_channel = self.bot.get_channel(Channels.bot_commands) + + self.assertEqual(args[0], f"Hey {self.message.author.mention}!") + self.assertEqual(embed.description, ( + "**Uh-oh!** It looks like your message got zapped by our spam filter. " + "We currently don't allow `.txt` attachments, so here are some tips to help you travel safely: \n\n" + "• If you attempted to send a message longer than 2000 characters, try shortening your message " + "to fit within the character limit or use a pasting service (see below) \n\n" + "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " + f"{cmd_channel.mention} for more information) or use a pasting service like: " + f"\n\n{URLs.site_schema}{URLs.site_paste}" + )) + def test_removing_deleted_message_logs(self): """Removing an already deleted message logs the correct message""" attachment = MockAttachment(filename="python.asdfsff") -- cgit v1.2.3 From c8bf44e30c286b27768601d5a04cd2459f170d4c Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 21:29:15 +0200 Subject: AntiMalware Tests - Switched to unittest.IsolatedAsyncioTestCase --- tests/bot/cogs/test_antimalware.py | 48 +++++++++++++++----------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index eba439afb..6fb7b399e 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,4 +1,3 @@ -import asyncio import logging import unittest from os.path import splitext @@ -13,7 +12,7 @@ from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole MODULE = "bot.cogs.antimalware" -class AntiMalwareCogTests(unittest.TestCase): +class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): """Test the AntiMalware cog.""" def setUp(self): @@ -22,62 +21,56 @@ class AntiMalwareCogTests(unittest.TestCase): self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() - def test_message_with_allowed_attachment(self): + async def test_message_with_allowed_attachment(self): """Messages with allowed extensions should not be deleted""" attachment = MockAttachment(filename=f"python.{AntiMalwareConfig.whitelist[0]}") self.message.attachments = [attachment] - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.message.delete.assert_not_called() - def test_message_without_attachment(self): + async def test_message_without_attachment(self): """Messages without attachments should result in no action.""" - coroutine = self.cog.on_message(self.message) - self.assertIsNone(asyncio.run(coroutine)) + self.assertIsNone(await self.cog.on_message(self.message)) self.message.delete.assert_not_called() - def test_direct_message_with_attachment(self): + async def test_direct_message_with_attachment(self): """Direct messages should have no action taken.""" attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] self.message.guild = None - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.message.delete.assert_not_called() - def test_message_with_illegal_extension_gets_deleted(self): + async def test_message_with_illegal_extension_gets_deleted(self): """A message containing an illegal extension should send an embed.""" attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.message.delete.assert_called_once() - def test_message_send_by_staff(self): + async def test_message_send_by_staff(self): """A message send by a member of staff should be ignored.""" moderator_role = MockRole(name="Moderator", id=Roles.moderators) self.message.author.roles.append(moderator_role) attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.message.delete.assert_not_called() - def test_python_file_redirect_embed(self): + async def test_python_file_redirect_embed(self): """A message containing a .python file should result in an embed redirecting the user to our paste site""" attachment = MockAttachment(filename="python.py") self.message.attachments = [attachment] self.message.channel.send = AsyncMock() - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) args, kwargs = self.message.channel.send.call_args embed = kwargs.pop("embed") @@ -87,13 +80,12 @@ class AntiMalwareCogTests(unittest.TestCase): f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" )) - def test_txt_file_redirect_embed(self): + async def test_txt_file_redirect_embed(self): attachment = MockAttachment(filename="python.txt") self.message.attachments = [attachment] self.message.channel.send = AsyncMock() - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) args, kwargs = self.message.channel.send.call_args embed = kwargs.pop("embed") cmd_channel = self.bot.get_channel(Channels.bot_commands) @@ -109,34 +101,32 @@ class AntiMalwareCogTests(unittest.TestCase): f"\n\n{URLs.site_schema}{URLs.site_paste}" )) - def test_removing_deleted_message_logs(self): + async def test_removing_deleted_message_logs(self): """Removing an already deleted message logs the correct message""" attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) - coroutine = self.cog.on_message(self.message) logger = logging.getLogger(MODULE) with self.assertLogs(logger=logger, level="INFO") as logs: - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.assertIn( f"INFO:{MODULE}:Tried to delete message `{self.message.id}`, but message could not be found.", logs.output) - def test_message_with_illegal_attachment_logs(self): + async def test_message_with_illegal_attachment_logs(self): """Deleting a message with an illegal attachment should result in a log.""" attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] - coroutine = self.cog.on_message(self.message) file_extensions = {splitext(attachment.filename.lower())[1] for attachment in self.message.attachments} extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) blocked_extensions_str = ', '.join(extensions_blocked) logger = logging.getLogger(MODULE) with self.assertLogs(logger=logger, level="INFO") as logs: - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.assertEqual( [ f"INFO:{MODULE}:" -- cgit v1.2.3 From 8a64763bb138119f2dea7db0a997d380f48865fc Mon Sep 17 00:00:00 2001 From: Daniel Brown Date: Thu, 7 May 2020 15:23:18 -0500 Subject: Addressing Review Changes - Changed docstring explanation and function name of `get_active_infractions` to `get_active_infraction()` to better convey that only one infraction is returned. Also changed all relevant uses to reflect that change. - Added explanation of parameter `send_msg` to the doc strings of `pardon_infraction()` and `get_active_infraction()` - Adjusted placement of `log.trace()` in `pardon_infraction()` - Adjusted logic in `apply_ban()` to remove redundant check. - Adjusted logic in `apply_ban()` to be consistent with other checks. Signed-off-by: Daniel Brown --- bot/cogs/moderation/infractions.py | 35 +++++++++++++++-------------------- bot/cogs/moderation/scheduler.py | 9 +++++++-- bot/cogs/moderation/superstarify.py | 2 +- bot/cogs/moderation/utils.py | 10 ++++++++-- 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 29b4db20e..89f72ade7 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -199,7 +199,7 @@ class Infractions(InfractionScheduler, commands.Cog): async def apply_mute(self, ctx: Context, user: Member, reason: str, **kwargs) -> None: """Apply a mute infraction with kwargs passed to `post_infraction`.""" - if await utils.get_active_infractions(ctx, user, "mute"): + if await utils.get_active_infraction(ctx, user, "mute"): return infraction = await utils.post_infraction(ctx, user, "mute", reason, active=True, **kwargs) @@ -236,28 +236,23 @@ class Infractions(InfractionScheduler, commands.Cog): Will also remove the banned user from the Big Brother watch list if applicable. """ # In the case of a permanent ban, we don't need get_active_infractions to tell us if one is active - send_msg = "expires_at" in kwargs - active_infraction = await utils.get_active_infractions(ctx, user, "ban", send_msg) + send_msg = kwargs.get("expires_at") is None + active_infraction = await utils.get_active_infraction(ctx, user, "ban", send_msg) if active_infraction: log.trace("Active infractions found.") - if ( - active_infraction.get('expires_at') is not None - and kwargs.get('expires_at') is None - ): - log.trace("Active ban is a temporary and being called by a perma. Removing temporary.") - await self.pardon_infraction(ctx, "ban", user, send_msg) - - elif ( - active_infraction.get('expires_at') is None - and kwargs.get('expires_at') is None - ): - log.trace("Active ban is a perma ban and being called by a perma. Send bounce back message.") - await ctx.send( - f":x: According to my records, this user is already permanently banned. " - f"See infraction **#{active_infraction['id']}**." - ) - return + if kwargs.get('expires_at') is None: + if active_infraction.get('expires_at') is not None: + log.trace("Active ban is a temporary and being called by a perma. Removing temporary.") + await self.pardon_infraction(ctx, "ban", user, send_msg) + + elif active_infraction.get('expires_at') is None: + log.trace("Active ban is a perma ban and being called by a perma. Send bounce back message.") + await ctx.send( + f":x: According to my records, this user is already permanently banned. " + f"See infraction **#{active_infraction['id']}**." + ) + return else: log.trace("Active ban is a temp ban being called by a temp or a perma being called by a temp. Ignore.") return diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 413717fb6..dc42bee2e 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -197,7 +197,12 @@ class InfractionScheduler(Scheduler): user: UserSnowflake, send_msg: bool = True ) -> None: - """Prematurely end an infraction for a user and log the action in the mod log.""" + """ + Prematurely end an infraction for a user and log the action in the mod log. + + If `send_msg` is True, then a pardoning confirmation message will be sent to + the context channel. Otherwise, no such message will be sent. + """ log.trace(f"Pardoning {infr_type} infraction for {user}.") # Check the current active infraction @@ -282,8 +287,8 @@ class InfractionScheduler(Scheduler): log.info(f"Pardoned {infr_type} infraction #{id_} for {user}.") # Send a confirmation message to the invoking context. - log.trace(f"Sending infraction #{id_} pardon confirmation message.") if send_msg: + log.trace(f"Sending infraction #{id_} pardon confirmation message.") await ctx.send( f"{dm_emoji}{confirm_msg} infraction **{infr_type}** for {user.mention}. " f"{log_text.get('Failure', '')}" diff --git a/bot/cogs/moderation/superstarify.py b/bot/cogs/moderation/superstarify.py index 272f7c4f0..29855c325 100644 --- a/bot/cogs/moderation/superstarify.py +++ b/bot/cogs/moderation/superstarify.py @@ -130,7 +130,7 @@ class Superstarify(InfractionScheduler, Cog): An optional reason can be provided. If no reason is given, the original name will be shown in a generated reason. """ - if await utils.get_active_infractions(ctx, member, "superstar"): + if await utils.get_active_infraction(ctx, member, "superstar"): return # Post the infraction to the API diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py index 406f9d08a..e4e0f1ec2 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/cogs/moderation/utils.py @@ -97,13 +97,19 @@ async def post_infraction( return -async def get_active_infractions( +async def get_active_infraction( ctx: Context, user: UserSnowflake, infr_type: str, send_msg: bool = True ) -> t.Optional[dict]: - """Retrieves active infractions of the given type for the user.""" + """ + Retrieves an active infraction of the given type for the user. + + If `send_msg` is True and the user has an active infraction matching the `infr_type` parameter, + then a message for the moderator will be sent to the context channel letting them know. + Otherwise, no message will be sent. + """ log.trace(f"Checking if {user} has active infractions of type {infr_type}.") active_infractions = await ctx.bot.api_client.get( -- cgit v1.2.3 From 79791326bbea2fee3ab06e1055dfe1897e050c51 Mon Sep 17 00:00:00 2001 From: Daniel Brown Date: Thu, 7 May 2020 15:58:09 -0500 Subject: apply_ban() logic refined - Refined the logic for `apply_ban()` even further to be cleaner. (Thanks, @MarkKoz!) Signed-off-by: Daniel Brown --- bot/cogs/moderation/infractions.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 89f72ade7..19a3176d9 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -236,27 +236,27 @@ class Infractions(InfractionScheduler, commands.Cog): Will also remove the banned user from the Big Brother watch list if applicable. """ # In the case of a permanent ban, we don't need get_active_infractions to tell us if one is active - send_msg = kwargs.get("expires_at") is None - active_infraction = await utils.get_active_infraction(ctx, user, "ban", send_msg) + is_temporary = kwargs.get("expires_at") is not None + active_infraction = await utils.get_active_infraction(ctx, user, "ban", is_temporary) if active_infraction: log.trace("Active infractions found.") - if kwargs.get('expires_at') is None: - if active_infraction.get('expires_at') is not None: - log.trace("Active ban is a temporary and being called by a perma. Removing temporary.") - await self.pardon_infraction(ctx, "ban", user, send_msg) - - elif active_infraction.get('expires_at') is None: - log.trace("Active ban is a perma ban and being called by a perma. Send bounce back message.") - await ctx.send( - f":x: According to my records, this user is already permanently banned. " - f"See infraction **#{active_infraction['id']}**." - ) - return - else: + if is_temporary: log.trace("Active ban is a temp ban being called by a temp or a perma being called by a temp. Ignore.") return + if active_infraction.get('expires_at') is not None: + log.trace("Active ban is a temporary and being called by a perma. Removing temporary.") + await self.pardon_infraction(ctx, "ban", user, is_temporary) + + else: + log.trace("Active ban is a perma ban and being called by a perma. Send bounce back message.") + await ctx.send( + f":x: According to my records, this user is already permanently banned. " + f"See infraction **#{active_infraction['id']}**." + ) + return + infraction = await utils.post_infraction(ctx, user, "ban", reason, active=True, **kwargs) if infraction is None: return -- cgit v1.2.3 From bd9537ba85154ece1dca39ec03d36dd7d39a8388 Mon Sep 17 00:00:00 2001 From: MrGrote Date: Fri, 8 May 2020 22:11:54 +0200 Subject: Update tests/bot/cogs/test_antimalware.py Co-authored-by: Mark --- tests/bot/cogs/test_antimalware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 6fb7b399e..e0aa9d6d2 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -65,7 +65,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.message.delete.assert_not_called() async def test_python_file_redirect_embed(self): - """A message containing a .python file should result in an embed redirecting the user to our paste site""" + """A message containing a .py file should result in an embed redirecting the user to our paste site""" attachment = MockAttachment(filename="python.py") self.message.attachments = [attachment] self.message.channel.send = AsyncMock() -- cgit v1.2.3 From b06c60bc3456c36583d3d58cbf62e9ecd14e5f94 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 9 May 2020 23:19:39 -0700 Subject: Filtering: don't delete messages in DMs Bots are incapable of deleting direct messages authored by others. --- bot/cogs/filtering.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 6a703f5a1..1e21a4ce3 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -4,7 +4,7 @@ from typing import Optional, Union import discord.errors from dateutil.relativedelta import relativedelta -from discord import Colour, DMChannel, Member, Message, TextChannel +from discord import Colour, Member, Message, TextChannel from discord.ext.commands import Cog from discord.utils import escape_markdown @@ -161,8 +161,10 @@ class Filtering(Cog): match = await _filter["function"](msg) if match: - # If this is a filter (not a watchlist), we should delete the message. - if _filter["type"] == "filter": + is_private = msg.channel.type is discord.ChannelType.private + + # If this is a filter (not a watchlist) and not in a DM, delete the message. + if _filter["type"] == "filter" and not is_private: try: # Embeds (can?) trigger both the `on_message` and `on_message_edit` # event handlers, triggering filtering twice for the same message. @@ -181,7 +183,7 @@ class Filtering(Cog): if _filter["user_notification"]: await self.notify_member(msg.author, _filter["notification_msg"], msg.channel) - if isinstance(msg.channel, DMChannel): + if is_private: channel_str = "via DM" else: channel_str = f"in {msg.channel.mention}" -- cgit v1.2.3 From 55b8aed9dc8d4f38824c8b5642a74df6a9948799 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 12:58:38 -0700 Subject: Filtering: don't attempt to send additional embeds for invalid invites Invalid invites won't have data available to put in the embeds. Fixes #929 Fixes BOT-3Z --- bot/cogs/filtering.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 6a703f5a1..0ad534741 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -212,7 +212,9 @@ class Filtering(Cog): additional_embeds = None additional_embeds_msg = None - if filter_name == "filter_invites": + # The function returns True for invalid invites. + # They have no data so additional embeds can't be created for them. + if filter_name == "filter_invites" and match is not True: additional_embeds = [] for invite, data in match.items(): embed = discord.Embed(description=( -- cgit v1.2.3 From 4a4ca1a168c5130d3627d3c4dbc8bfe39119cc22 Mon Sep 17 00:00:00 2001 From: Suhail Date: Sun, 10 May 2020 23:04:04 +0100 Subject: Add remindme alias for the remind command --- bot/cogs/reminders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/reminders.py b/bot/cogs/reminders.py index 24c279357..8b6457cbb 100644 --- a/bot/cogs/reminders.py +++ b/bot/cogs/reminders.py @@ -158,7 +158,7 @@ class Reminders(Scheduler, Cog): ) await self._delete_reminder(reminder["id"]) - @group(name="remind", aliases=("reminder", "reminders"), invoke_without_command=True) + @group(name="remind", aliases=("reminder", "reminders", "remindme"), invoke_without_command=True) async def remind_group(self, ctx: Context, expiration: Duration, *, content: str) -> None: """Commands for managing your reminders.""" await ctx.invoke(self.new_reminder, expiration=expiration, content=content) -- cgit v1.2.3 From d1af9cda00d18d0fee679964ba177f6a3f7ec196 Mon Sep 17 00:00:00 2001 From: Daniel Brown Date: Mon, 11 May 2020 12:39:49 -0500 Subject: Restructure `apply_ban()` logic Another refactor/cleaning to make the logic clearer and easier to understand. Also cleaned up the trace logs to be shorter and more concise. Thanks, @scragly! Co-authored-by: scragly <29337040+scragly@users.noreply.github.com> --- bot/cogs/moderation/infractions.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 19a3176d9..e62a36c43 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -240,23 +240,18 @@ class Infractions(InfractionScheduler, commands.Cog): active_infraction = await utils.get_active_infraction(ctx, user, "ban", is_temporary) if active_infraction: - log.trace("Active infractions found.") if is_temporary: - log.trace("Active ban is a temp ban being called by a temp or a perma being called by a temp. Ignore.") + log.trace("Tempban ignored as it cannot overwrite an active ban.") return - if active_infraction.get('expires_at') is not None: - log.trace("Active ban is a temporary and being called by a perma. Removing temporary.") - await self.pardon_infraction(ctx, "ban", user, is_temporary) - - else: - log.trace("Active ban is a perma ban and being called by a perma. Send bounce back message.") - await ctx.send( - f":x: According to my records, this user is already permanently banned. " - f"See infraction **#{active_infraction['id']}**." - ) + if active_infraction.get('expires_at') is None: + log.trace("Permaban already exists, notify.") + await ctx.send(f":x: User is already permanently banned (#{active_infraction['id']}).") return + log.trace("Old tempban is being replaced by new permaban.") + await self.pardon_infraction(ctx, "ban", user, is_temporary) + infraction = await utils.post_infraction(ctx, user, "ban", reason, active=True, **kwargs) if infraction is None: return -- cgit v1.2.3 From 847a78a76c08a670e85d926e3afa43e1cc3180f4 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 19:41:46 +0200 Subject: AntiMalware Tests - implemented minor feedback --- tests/bot/cogs/test_antimalware.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index e0aa9d6d2..6e06df0a8 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,4 +1,3 @@ -import logging import unittest from os.path import splitext from unittest.mock import AsyncMock, Mock @@ -6,7 +5,7 @@ from unittest.mock import AsyncMock, Mock from discord import NotFound from bot.cogs import antimalware -from bot.constants import AntiMalware as AntiMalwareConfig, Channels, Roles, URLs +from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES, URLs from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole MODULE = "bot.cogs.antimalware" @@ -31,7 +30,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_message_without_attachment(self): """Messages without attachments should result in no action.""" - self.assertIsNone(await self.cog.on_message(self.message)) + await self.cog.on_message(self.message) self.message.delete.assert_not_called() async def test_direct_message_with_attachment(self): @@ -55,8 +54,8 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_message_send_by_staff(self): """A message send by a member of staff should be ignored.""" - moderator_role = MockRole(name="Moderator", id=Roles.moderators) - self.message.author.roles.append(moderator_role) + staff_role = MockRole(id=STAFF_ROLES[0]) + self.message.author.roles.append(staff_role) attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] @@ -71,6 +70,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.message.channel.send = AsyncMock() await self.cog.on_message(self.message) + self.message.channel.send.assert_called_once() args, kwargs = self.message.channel.send.call_args embed = kwargs.pop("embed") @@ -107,13 +107,13 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.message.attachments = [attachment] self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) - logger = logging.getLogger(MODULE) - - with self.assertLogs(logger=logger, level="INFO") as logs: + with self.assertLogs(logger=antimalware.log, level="INFO") as logs: await self.cog.on_message(self.message) + self.message.delete.assert_called_once() self.assertIn( f"INFO:{MODULE}:Tried to delete message `{self.message.id}`, but message could not be found.", - logs.output) + logs.output + ) async def test_message_with_illegal_attachment_logs(self): """Deleting a message with an illegal attachment should result in a log.""" @@ -123,9 +123,8 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): file_extensions = {splitext(attachment.filename.lower())[1] for attachment in self.message.attachments} extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) blocked_extensions_str = ', '.join(extensions_blocked) - logger = logging.getLogger(MODULE) - with self.assertLogs(logger=logger, level="INFO") as logs: + with self.assertLogs(logger=antimalware.log, level="INFO") as logs: await self.cog.on_message(self.message) self.assertEqual( [ @@ -133,7 +132,8 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): f"User '{self.message.author}' ({self.message.author.id}) " f"uploaded blacklisted file(s): {blocked_extensions_str}" ], - logs.output) + logs.output + ) class AntiMalwareSetupTests(unittest.TestCase): -- cgit v1.2.3 From ba71ac5b002dd3e1ee6a916ba2705a7cff697a66 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 20:24:20 +0200 Subject: AntiMalware Tests - extracted the method for determining disallowed extensions and added a test for it. --- tests/bot/cogs/test_antimalware.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 6e06df0a8..78ad996f2 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -19,10 +19,11 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.bot = MockBot() self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() + AntiMalwareConfig.whitelist = [".first", ".second", ".third"] async def test_message_with_allowed_attachment(self): """Messages with allowed extensions should not be deleted""" - attachment = MockAttachment(filename=f"python.{AntiMalwareConfig.whitelist[0]}") + attachment = MockAttachment(filename=f"python{AntiMalwareConfig.whitelist[0]}") self.message.attachments = [attachment] await self.cog.on_message(self.message) @@ -35,7 +36,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_direct_message_with_attachment(self): """Direct messages should have no action taken.""" - attachment = MockAttachment(filename="python.asdfsff") + attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] self.message.guild = None @@ -45,7 +46,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_message_with_illegal_extension_gets_deleted(self): """A message containing an illegal extension should send an embed.""" - attachment = MockAttachment(filename="python.asdfsff") + attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] await self.cog.on_message(self.message) @@ -56,7 +57,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): """A message send by a member of staff should be ignored.""" staff_role = MockRole(id=STAFF_ROLES[0]) self.message.author.roles.append(staff_role) - attachment = MockAttachment(filename="python.asdfsff") + attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] await self.cog.on_message(self.message) @@ -103,7 +104,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_removing_deleted_message_logs(self): """Removing an already deleted message logs the correct message""" - attachment = MockAttachment(filename="python.asdfsff") + attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) @@ -117,7 +118,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_message_with_illegal_attachment_logs(self): """Deleting a message with an illegal attachment should result in a log.""" - attachment = MockAttachment(filename="python.asdfsff") + attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] file_extensions = {splitext(attachment.filename.lower())[1] for attachment in self.message.attachments} @@ -135,6 +136,22 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): logs.output ) + async def test_get_disallowed_extensions(self): + """The return value should include all non-whitelisted extensions.""" + test_values = ( + (AntiMalwareConfig.whitelist, []), + ([".first"], []), + ([".first", ".disallowed"], [".disallowed"]), + ([".disallowed"], [".disallowed"]), + ([".disallowed", ".illegal"], [".disallowed", ".illegal"]), + ) + + for extensions, expected_disallowed_extensions in test_values: + with self.subTest(extensions=extensions, expected_disallowed_extensions=expected_disallowed_extensions): + self.message.attachments = [MockAttachment(filename=f"filename{extension}") for extension in extensions] + disallowed_extensions = self.cog.get_disallowed_extensions(self.message) + self.assertCountEqual(disallowed_extensions, expected_disallowed_extensions) + class AntiMalwareSetupTests(unittest.TestCase): """Tests setup of the `AntiMalware` cog.""" -- cgit v1.2.3 From 148b12603f4ad8799d135ec9956d1841cf1c7bf7 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 20:24:39 +0200 Subject: AntiMalware Tests - extracted the method for determining disallowed extensions and added a test for it. --- bot/cogs/antimalware.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bot/cogs/antimalware.py b/bot/cogs/antimalware.py index 66b5073e8..f5fd5e2d9 100644 --- a/bot/cogs/antimalware.py +++ b/bot/cogs/antimalware.py @@ -1,4 +1,5 @@ import logging +import typing as t from os.path import splitext from discord import Embed, Message, NotFound @@ -29,8 +30,7 @@ class AntiMalware(Cog): return embed = Embed() - file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} - extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) + extensions_blocked = self.get_disallowed_extensions(message) blocked_extensions_str = ', '.join(extensions_blocked) if ".py" in extensions_blocked: # Short-circuit on *.py files to provide a pastebin link @@ -73,6 +73,13 @@ class AntiMalware(Cog): except NotFound: log.info(f"Tried to delete message `{message.id}`, but message could not be found.") + @classmethod + def get_disallowed_extensions(cls, message: Message) -> t.Iterable[str]: + """Get an iterable containing all the disallowed extensions of attachments.""" + file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} + extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) + return extensions_blocked + def setup(bot: Bot) -> None: """Load the AntiMalware cog.""" -- cgit v1.2.3 From ecaddcedab6946ac4650b699a790471ef2a898c9 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 20:39:25 +0200 Subject: AntiMalware Tests - added a missing case for no extensions in test_get_disallowed_extensions --- tests/bot/cogs/test_antimalware.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 78ad996f2..7caee6f3c 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -139,6 +139,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_get_disallowed_extensions(self): """The return value should include all non-whitelisted extensions.""" test_values = ( + ([], []), (AntiMalwareConfig.whitelist, []), ([".first"], []), ([".first", ".disallowed"], [".disallowed"]), -- cgit v1.2.3 From fa467e4ef133186ff462b0178bcab08e8a3d6b2d Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 20:58:51 +0200 Subject: AntiMalware Tests - Removed exact log content checks --- tests/bot/cogs/test_antimalware.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 7caee6f3c..a2ce9a740 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,5 +1,4 @@ import unittest -from os.path import splitext from unittest.mock import AsyncMock, Mock from discord import NotFound @@ -108,33 +107,17 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.message.attachments = [attachment] self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) - with self.assertLogs(logger=antimalware.log, level="INFO") as logs: + with self.assertLogs(logger=antimalware.log, level="INFO"): await self.cog.on_message(self.message) self.message.delete.assert_called_once() - self.assertIn( - f"INFO:{MODULE}:Tried to delete message `{self.message.id}`, but message could not be found.", - logs.output - ) async def test_message_with_illegal_attachment_logs(self): """Deleting a message with an illegal attachment should result in a log.""" attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] - file_extensions = {splitext(attachment.filename.lower())[1] for attachment in self.message.attachments} - extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) - blocked_extensions_str = ', '.join(extensions_blocked) - - with self.assertLogs(logger=antimalware.log, level="INFO") as logs: + with self.assertLogs(logger=antimalware.log, level="INFO"): await self.cog.on_message(self.message) - self.assertEqual( - [ - f"INFO:{MODULE}:" - f"User '{self.message.author}' ({self.message.author.id}) " - f"uploaded blacklisted file(s): {blocked_extensions_str}" - ], - logs.output - ) async def test_get_disallowed_extensions(self): """The return value should include all non-whitelisted extensions.""" -- cgit v1.2.3 From ddfe583d0b1e72f98855f628ff01b72c82fa491d Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 21:56:39 +0200 Subject: AntiMalware Refactor - Moved embed descriptions into constants, added tests for embed descriptions --- bot/cogs/antimalware.py | 44 ++++++++++++++++++++-------------- tests/bot/cogs/test_antimalware.py | 48 ++++++++++++++++++++++++-------------- 2 files changed, 56 insertions(+), 36 deletions(-) diff --git a/bot/cogs/antimalware.py b/bot/cogs/antimalware.py index f5fd5e2d9..ea257442e 100644 --- a/bot/cogs/antimalware.py +++ b/bot/cogs/antimalware.py @@ -10,6 +10,27 @@ from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLE log = logging.getLogger(__name__) +PY_EMBED_DESCRIPTION = ( + "It looks like you tried to attach a Python file - " + f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" +) + +TXT_EMBED_DESCRIPTION = ( + "**Uh-oh!** It looks like your message got zapped by our spam filter. " + "We currently don't allow `.txt` attachments, so here are some tips to help you travel safely: \n\n" + "• If you attempted to send a message longer than 2000 characters, try shortening your message " + "to fit within the character limit or use a pasting service (see below) \n\n" + "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " + "{cmd_channel_mention} for more information) or use a pasting service like: " + f"\n\n{URLs.site_schema}{URLs.site_paste}" +) + +DISALLOWED_EMBED_DESCRIPTION = ( + "It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " + f"We currently allow the following file types: **{', '.join(AntiMalwareConfig.whitelist)}**.\n\n" + "Feel free to ask in {meta_channel_mention} if you think this is a mistake." +) + class AntiMalware(Cog): """Delete messages which contain attachments with non-whitelisted file extensions.""" @@ -34,29 +55,16 @@ class AntiMalware(Cog): blocked_extensions_str = ', '.join(extensions_blocked) if ".py" in extensions_blocked: # Short-circuit on *.py files to provide a pastebin link - embed.description = ( - "It looks like you tried to attach a Python file - " - f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" - ) + embed.description = PY_EMBED_DESCRIPTION elif ".txt" in extensions_blocked: # Work around Discord AutoConversion of messages longer than 2000 chars to .txt cmd_channel = self.bot.get_channel(Channels.bot_commands) - embed.description = ( - "**Uh-oh!** It looks like your message got zapped by our spam filter. " - "We currently don't allow `.txt` attachments, so here are some tips to help you travel safely: \n\n" - "• If you attempted to send a message longer than 2000 characters, try shortening your message " - "to fit within the character limit or use a pasting service (see below) \n\n" - "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " - f"{cmd_channel.mention} for more information) or use a pasting service like: " - f"\n\n{URLs.site_schema}{URLs.site_paste}" - ) + embed.description = TXT_EMBED_DESCRIPTION.format(cmd_channel_mention=cmd_channel.mention) elif extensions_blocked: - whitelisted_types = ', '.join(AntiMalwareConfig.whitelist) meta_channel = self.bot.get_channel(Channels.meta) - embed.description = ( - f"It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " - f"We currently allow the following file types: **{whitelisted_types}**.\n\n" - f"Feel free to ask in {meta_channel.mention} if you think this is a mistake." + embed.description = DISALLOWED_EMBED_DESCRIPTION.format( + blocked_extensions_str=blocked_extensions_str, + meta_channel_mention=meta_channel.mention, ) if embed.description: diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index a2ce9a740..fab063201 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -4,7 +4,7 @@ from unittest.mock import AsyncMock, Mock from discord import NotFound from bot.cogs import antimalware -from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES, URLs +from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole MODULE = "bot.cogs.antimalware" @@ -63,7 +63,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.message.delete.assert_not_called() - async def test_python_file_redirect_embed(self): + async def test_python_file_redirect_embed_description(self): """A message containing a .py file should result in an embed redirecting the user to our paste site""" attachment = MockAttachment(filename="python.py") self.message.attachments = [attachment] @@ -74,32 +74,44 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): args, kwargs = self.message.channel.send.call_args embed = kwargs.pop("embed") - self.assertEqual(args[0], f"Hey {self.message.author.mention}!") - self.assertEqual(embed.description, ( - "It looks like you tried to attach a Python file - " - f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" - )) + self.assertEqual(embed.description, antimalware.PY_EMBED_DESCRIPTION) - async def test_txt_file_redirect_embed(self): + async def test_txt_file_redirect_embed_description(self): + """A message containing a .txt file should result in the correct embed.""" attachment = MockAttachment(filename="python.txt") self.message.attachments = [attachment] self.message.channel.send = AsyncMock() + antimalware.TXT_EMBED_DESCRIPTION = Mock() + antimalware.TXT_EMBED_DESCRIPTION.format.return_value = "test" await self.cog.on_message(self.message) + self.message.channel.send.assert_called_once() args, kwargs = self.message.channel.send.call_args embed = kwargs.pop("embed") cmd_channel = self.bot.get_channel(Channels.bot_commands) - self.assertEqual(args[0], f"Hey {self.message.author.mention}!") - self.assertEqual(embed.description, ( - "**Uh-oh!** It looks like your message got zapped by our spam filter. " - "We currently don't allow `.txt` attachments, so here are some tips to help you travel safely: \n\n" - "• If you attempted to send a message longer than 2000 characters, try shortening your message " - "to fit within the character limit or use a pasting service (see below) \n\n" - "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " - f"{cmd_channel.mention} for more information) or use a pasting service like: " - f"\n\n{URLs.site_schema}{URLs.site_paste}" - )) + self.assertEqual(embed.description, antimalware.TXT_EMBED_DESCRIPTION.format.return_value) + antimalware.TXT_EMBED_DESCRIPTION.format.assert_called_with(cmd_channel_mention=cmd_channel.mention) + + async def test_other_disallowed_extention_embed_description(self): + """Test the description for a non .py/.txt disallowed extension.""" + attachment = MockAttachment(filename="python.disallowed") + self.message.attachments = [attachment] + self.message.channel.send = AsyncMock() + antimalware.DISALLOWED_EMBED_DESCRIPTION = Mock() + antimalware.DISALLOWED_EMBED_DESCRIPTION.format.return_value = "test" + + await self.cog.on_message(self.message) + self.message.channel.send.assert_called_once() + args, kwargs = self.message.channel.send.call_args + embed = kwargs.pop("embed") + meta_channel = self.bot.get_channel(Channels.meta) + + self.assertEqual(embed.description, antimalware.DISALLOWED_EMBED_DESCRIPTION.format.return_value) + antimalware.DISALLOWED_EMBED_DESCRIPTION.format.assert_called_with( + blocked_extensions_str=".disallowed", + meta_channel_mention=meta_channel.mention + ) async def test_removing_deleted_message_logs(self): """Removing an already deleted message logs the correct message""" -- cgit v1.2.3 From 0c552b2dc3e88b5e13278cb705c371db48c72646 Mon Sep 17 00:00:00 2001 From: "S. Co1" Date: Tue, 12 May 2020 21:29:35 -0400 Subject: Expand guild whitelist --- config-default.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config-default.yml b/config-default.yml index ff6790423..6d97b7f33 100644 --- a/config-default.yml +++ b/config-default.yml @@ -263,7 +263,8 @@ filter: guild_invite_whitelist: - 280033776820813825 # Functional Programming - 267624335836053506 # Python Discord - - 440186186024222721 # Python Discord: ModLog Emojis + - 440186186024222721 # Python Discord: Emojis 1 + - 578587418123304970 # Python Discord: Emojis 2 - 273944235143593984 # STEM - 348658686962696195 # RLBot - 531221516914917387 # Pallets @@ -280,6 +281,11 @@ filter: - 336642139381301249 # discord.py - 405403391410438165 # Sentdex - 172018499005317120 # The Coding Den + - 666560367173828639 # PyWeek + - 702724176489873509 # Microsoft Python + - 81384788765712384 # Discord API + - 613425648685547541 # Discord Developers + - 185590609631903755 # Blender Hub domain_blacklist: - pornhub.com -- cgit v1.2.3 From d7e6bed7b5b0f61312165e5b0b2b9291cd8df0c9 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 13 May 2020 14:17:51 +0300 Subject: Add message publishing to `Reddit` cog --- bot/cogs/reddit.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index 5a7fa100f..371b65434 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -218,7 +218,10 @@ class Reddit(Cog): for subreddit in RedditConfig.subreddits: top_posts = await self.get_top_posts(subreddit=subreddit, time="day") - await self.webhook.send(username=f"{subreddit} Top Daily Posts", embed=top_posts) + message = await self.webhook.send(username=f"{subreddit} Top Daily Posts", embed=top_posts) + + if message.channel.is_news(): + await message.publish() async def top_weekly_posts(self) -> None: """Post a summary of the top posts.""" @@ -242,6 +245,9 @@ class Reddit(Cog): await message.pin() + if message.channel.is_news(): + await message.publish() + @group(name="reddit", invoke_without_command=True) async def reddit_group(self, ctx: Context) -> None: """View the top posts from various subreddits.""" -- cgit v1.2.3 From 8d595467aba4bcde76caa6f6b9eccf8a9b73b0c5 Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff <33516116+SebastiaanZ@users.noreply.github.com> Date: Thu, 14 May 2020 09:56:55 +0200 Subject: Remove everyone-ping from mentions alert The mentions alert that is sent out by the Verification cog currently pings `@everyone` despite being quite unactionable by most people receiving the ping. As it happens frequently, especially with the recent uptick in joins, I'm removing that ping to not bother our moderators as much. --- bot/cogs/verification.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 388b7a338..b1ab7be4d 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -92,7 +92,6 @@ class Verification(Cog): text=embed_text, thumbnail=message.author.avatar_url_as(static_format="png"), channel_id=constants.Channels.mod_alerts, - ping_everyone=constants.Filter.ping_everyone, ) ctx: Context = await self.bot.get_context(message) -- cgit v1.2.3 From 4480e1d5f7d2df5f9a2f8b57b1b9c4b26e1c8ea6 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 14 May 2020 10:03:30 +0200 Subject: Remove @Admins ping from the #verification message This probably isn't necessary anymore. We get so many new users that someone is going to DM us very soon when something breaks. We've outgrown this, and it just adds noise to the #verification channel in the form of pings. --- bot/cogs/verification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index b1ab7be4d..77e8b5706 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -40,7 +40,7 @@ else: PERIODIC_PING = ( f"@everyone To verify that you have read our rules, please type `{constants.Bot.prefix}accept`." " If you encounter any problems during the verification process, " - f"ping the <@&{constants.Roles.admins}> role in this channel." + f"send a direct message to a staff member." ) BOT_MESSAGE_DELETE_DELAY = 10 -- cgit v1.2.3 From 206aed70f6185057ccbe4f8478ed456e5ab0c197 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 14 May 2020 16:58:55 +0300 Subject: Python News: Implement stats Add stat increaser to PEP and maillist posting. --- bot/cogs/python_news.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/cogs/python_news.py b/bot/cogs/python_news.py index 57ce61638..d28af4a0b 100644 --- a/bot/cogs/python_news.py +++ b/bot/cogs/python_news.py @@ -109,6 +109,9 @@ class PythonNews(Cog): ) payload["data"]["pep"].append(pep_nr) + # Increase overall PEP new stat + self.bot.stats.incr("python_news.posted.pep") + if msg.channel.is_news(): log.trace("Publishing PEP annnouncement because it was in a news channel") await msg.publish() @@ -168,6 +171,9 @@ class PythonNews(Cog): ) payload["data"][maillist].append(thread_information["thread_id"]) + # Increase this specific maillist counter in stats + self.bot.stats.incr(f"python_news.posted.{maillist.replace('-', '_')}") + if msg.channel.is_news(): log.trace("Publishing mailing list message because it was in a news channel") await msg.publish() -- cgit v1.2.3 From d2c538e23c20c5c4b22d7b2eb2bcf03067593374 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Fri, 15 May 2020 19:33:24 +0200 Subject: Increase snekbox re eval timeout. --- bot/cogs/snekbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 8d4688114..611e80f61 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -227,7 +227,7 @@ class Snekbox(Cog): _, new_message = await self.bot.wait_for( 'message_edit', check=_predicate_eval_message_edit, - timeout=10 + timeout=30 ) await ctx.message.add_reaction(REEVAL_EMOJI) await self.bot.wait_for( -- cgit v1.2.3 From 1c06d2a9d873ced2e54bf16a96573a46c583c12f Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Fri, 15 May 2020 19:51:19 +0200 Subject: Move the re eval timeout to a module constant --- bot/cogs/snekbox.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 611e80f61..9fa75a929 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -47,6 +47,7 @@ EVAL_ROLES = (Roles.helpers, Roles.moderators, Roles.admins, Roles.owners, Roles SIGKILL = 9 REEVAL_EMOJI = '\U0001f501' # :repeat: +REEVAL_TIMEOUT = 30 class Snekbox(Cog): @@ -227,7 +228,7 @@ class Snekbox(Cog): _, new_message = await self.bot.wait_for( 'message_edit', check=_predicate_eval_message_edit, - timeout=30 + timeout=REEVAL_TIMEOUT ) await ctx.message.add_reaction(REEVAL_EMOJI) await self.bot.wait_for( -- cgit v1.2.3 From 5a48ed0d60ebc9984cae27b19953b50b52df83d9 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Fri, 15 May 2020 19:52:26 +0200 Subject: Change tests to use the new timeout constant --- tests/bot/cogs/test_snekbox.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_snekbox.py b/tests/bot/cogs/test_snekbox.py index 1dec0ccaf..ccc090f02 100644 --- a/tests/bot/cogs/test_snekbox.py +++ b/tests/bot/cogs/test_snekbox.py @@ -291,7 +291,11 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(actual, expected) self.bot.wait_for.assert_has_awaits( ( - call('message_edit', check=partial_mock(snekbox.predicate_eval_message_edit, ctx), timeout=10), + call( + 'message_edit', + check=partial_mock(snekbox.predicate_eval_message_edit, ctx), + timeout=snekbox.REEVAL_TIMEOUT, + ), call('reaction_add', check=partial_mock(snekbox.predicate_eval_emoji_reaction, ctx), timeout=10) ) ) -- cgit v1.2.3 From c71c538c265e325f7aa4ee9cb9f298b2f50d38f1 Mon Sep 17 00:00:00 2001 From: mathsman5133 Date: Sat, 16 May 2020 13:11:25 +1000 Subject: fix redirect_output decorator; remove ninja code - Lots of instance of `for c in ...` or `for a in ...` or `fmt` which are non-descriptive and sometimes cryptic. - Ves suggested running the command in an asyncio task for `@redirect_output`, rather than making a workaround which only applies to the help command. This fixes a fundamental flaw where the redirection message wouldn't be deleted until a further 60sec after the command has finished, which for `!help` could be up to 5min, meaning the invocation message could be sitting there for 6min, not the intended 60sec. --- bot/cogs/help.py | 107 ++++++++++++++++++++++++++---------------------------- bot/decorators.py | 33 +++++++---------- bot/pagination.py | 2 +- 3 files changed, 66 insertions(+), 76 deletions(-) diff --git a/bot/cogs/help.py b/bot/cogs/help.py index 6fc4d83f8..542f19139 100644 --- a/bot/cogs/help.py +++ b/bot/cogs/help.py @@ -3,7 +3,7 @@ import logging from asyncio import TimeoutError from collections import namedtuple from contextlib import suppress -from typing import List +from typing import List, Union from discord import Colour, Embed, Member, Message, NotFound, Reaction, User from discord.ext.commands import Bot, Cog, Command, Context, Group, HelpCommand @@ -30,9 +30,9 @@ async def help_cleanup(bot: Bot, author: Member, message: Message) -> None: Adds the :trashcan: reaction that, when clicked, will delete the help message. After a 300 second timeout, the reaction will be removed. """ - def check(r: Reaction, u: User) -> bool: + def check(reaction: Reaction, user: User) -> bool: """Checks the reaction is :trashcan:, the author is original author and messages are the same.""" - return str(r) == DELETE_EMOJI and u.id == author.id and r.message.id == message.id + return str(reaction) == DELETE_EMOJI and user.id == author.id and reaction.message.id == message.id await message.add_reaction(DELETE_EMOJI) @@ -75,17 +75,11 @@ class CustomHelpCommand(HelpCommand): super().__init__(command_attrs={"help": "Shows help for bot commands"}) @redirect_output(destination_channel=Channels.bot_commands, bypass_roles=STAFF_ROLES) - async def prepare_help_command(self, ctx: Context, command: str = None) -> None: - """Adjust context to redirect to a new channel if required.""" - self.context = ctx - async def command_callback(self, ctx: Context, *, command: str = None) -> None: """Attempts to match the provided query with a valid command or cog.""" # the only reason we need to tamper with this is because d.py does not support "categories", # so we need to deal with them ourselves. - # handle any command redirection and adjust context channel accordingly. - await self.prepare_help_command(ctx, command=command) bot = ctx.bot if command is None: @@ -107,7 +101,7 @@ class CustomHelpCommand(HelpCommand): await self.send_category_help(category) return - # it's either a cog, group, command or subcommand, let super deal with it + # it's either a cog, group, command or subcommand; let the parent class deal with it await super().command_callback(ctx, command=command) async def get_all_help_choices(self) -> set: @@ -127,22 +121,22 @@ class CustomHelpCommand(HelpCommand): """ # first get all commands including subcommands and full command name aliases choices = set() - for c in await self.filter_commands(self.context.bot.walk_commands()): + for command in await self.filter_commands(self.context.bot.walk_commands()): # the the command or group name - choices.add(str(c)) + choices.add(str(command)) - if isinstance(c, Command): + if isinstance(command, Command): # all aliases if it's just a command - choices.update(c.aliases) + choices.update(command.aliases) else: # otherwise we need to add the parent name in - choices.update(f"{c.full_parent_name} {a}" for a in c.aliases) + choices.update(f"{command.full_parent_name} {alias}" for alias in command.aliases) # all cog names choices.update(self.context.bot.cogs) # all category names - choices.update(n.category for n in self.context.bot.cogs.values() if hasattr(n, "category")) + choices.update(cog.category for cog in self.context.bot.cogs.values() if hasattr(cog, "category")) return choices async def command_not_found(self, string: str) -> "HelpQueryNotFound": @@ -169,7 +163,7 @@ class CustomHelpCommand(HelpCommand): embed = Embed(colour=Colour.red(), title=str(error)) if getattr(error, "possible_matches", None): - matches = "\n".join(f"`{n}`" for n in error.possible_matches) + matches = "\n".join(f"`{match}`" for match in error.possible_matches) embed.description = f"**Did you mean:**\n{matches}" await self.context.send(embed=embed) @@ -186,19 +180,19 @@ class CustomHelpCommand(HelpCommand): parent = command.full_parent_name name = str(command) if not parent else f"{parent} {command.name}" - fmt = f"**```{PREFIX}{name} {command.signature}```**\n" + command_details = f"**```{PREFIX}{name} {command.signature}```**\n" # show command aliases - aliases = ", ".join(f"`{a}`" if not parent else f"`{parent} {a}`" for a in command.aliases) + aliases = ", ".join(f"`{alias}`" if not parent else f"`{parent} {alias}`" for alias in command.aliases) if aliases: - fmt += f"**Can also use:** {aliases}\n\n" + command_details += f"**Can also use:** {aliases}\n\n" # check if the user is allowed to run this command if not await command.can_run(self.context): - fmt += "***You cannot run this command.***\n\n" + command_details += "***You cannot run this command.***\n\n" - fmt += f"*{command.help or 'No details provided.'}*\n" - embed.description = fmt + command_details += f"*{command.help or 'No details provided.'}*\n" + embed.description = command_details return embed @@ -209,14 +203,22 @@ class CustomHelpCommand(HelpCommand): await help_cleanup(self.context.bot, self.context.author, message) @staticmethod - def get_commands_brief_details(commands_: List[Command]) -> str: - """Formats the prefix, command name and signature, and short doc for an iterable of commands.""" - details = "" - for c in commands_: - signature = f" {c.signature}" if c.signature else "" - details += f"\n**`{PREFIX}{c.qualified_name}{signature}`**\n*{c.short_doc or 'No details provided'}*" + def get_commands_brief_details(commands_: List[Command], return_as_list: bool = False) -> Union[List[str], str]: + """ + Formats the prefix, command name and signature, and short doc for an iterable of commands. - return details + return_as_list is helpful for passing these command details into the paginator as a list of command details. + """ + details = [] + for command in commands_: + signature = f" {command.signature}" if command.signature else "" + details.append( + f"\n**`{PREFIX}{command.qualified_name}{signature}`**\n*{command.short_doc or 'No details provided'}*" + ) + if return_as_list: + return details + else: + return "".join(details) async def send_group_help(self, group: Group) -> None: """Sends help for a group command.""" @@ -256,17 +258,17 @@ class CustomHelpCommand(HelpCommand): await help_cleanup(self.context.bot, self.context.author, message) @staticmethod - def _category_key(cmd: Command) -> str: + def _category_key(command: Command) -> str: """ Returns a cog name of a given command for use as a key for `sorted` and `groupby`. A zero width space is used as a prefix for results with no cogs to force them last in ordering. """ - if cmd.cog: + if command.cog: with suppress(AttributeError): - if cmd.cog.category: - return f"**{cmd.cog.category}**" - return f"**{cmd.cog_name}**" + if command.cog.category: + return f"**{command.cog.category}**" + return f"**{command.cog_name}**" else: return "**\u200bNo Category:**" @@ -280,28 +282,24 @@ class CustomHelpCommand(HelpCommand): embed.set_author(name="Command Help", icon_url=constants.Icons.questionmark) all_commands = [] - for c in category.cogs: - all_commands.extend(c.get_commands()) + for cog in category.cogs: + all_commands.extend(cog.get_commands()) filtered_commands = await self.filter_commands(all_commands, sort=True) - lines = [ - f"`{PREFIX}{c.qualified_name}{f' {c.signature}' if c.signature else ''}`" - f"\n*{c.short_doc or 'No details provided.'}*" for c in filtered_commands - ] - + command_detail_lines = self.get_commands_brief_details(filtered_commands, return_as_list=True) description = f"**{category.name}**\n*{category.description}*" - if lines: + if command_detail_lines: description += "\n\n**Commands:**" await LinePaginator.paginate( - lines, + command_detail_lines, self.context, embed, prefix=description, max_lines=COMMANDS_PER_PAGE, - max_size=2040 + max_size=2040, ) async def send_bot_help(self, mapping: dict) -> None: @@ -321,31 +319,28 @@ class CustomHelpCommand(HelpCommand): if len(sorted_commands) == 0: continue - command_details = [ - f"`{PREFIX}{c.qualified_name}{f' {c.signature}' if c.signature else ''}`" - f"\n*{c.short_doc or 'No details provided.'}*" for c in sorted_commands - ] + command_detail_lines = self.get_commands_brief_details(sorted_commands, return_as_list=True) # Split cogs or categories which have too many commands to fit in one page. # The length of commands is included for later use when aggregating into pages for the paginator. - for i in range(0, len(sorted_commands), COMMANDS_PER_PAGE): - truncated_fmt = command_details[i:i + COMMANDS_PER_PAGE] - joined_fmt = "\n".join(truncated_fmt) - cog_or_category_pages.append((f"**{cog_or_category}**\n{joined_fmt}", len(truncated_fmt))) + for index in range(0, len(sorted_commands), COMMANDS_PER_PAGE): + truncated_lines = command_detail_lines[index:index + COMMANDS_PER_PAGE] + joined_lines = "".join(truncated_lines) + cog_or_category_pages.append((f"**{cog_or_category}**{joined_lines}", len(truncated_lines))) pages = [] counter = 0 page = "" - for fmt, length in cog_or_category_pages: + for page_details, length in cog_or_category_pages: counter += length if counter > COMMANDS_PER_PAGE: # force a new page on paginator even if it falls short of the max pages # since we still want to group categories/cogs. counter = length pages.append(page) - page = f"{fmt}\n\n" + page = f"{page_details}\n\n" else: - page += f"{fmt}\n\n" + page += f"{page_details}\n\n" if page: # add any remaining command help that didn't get added in the last iteration above. diff --git a/bot/decorators.py b/bot/decorators.py index ce38a5e76..306f0830c 100644 --- a/bot/decorators.py +++ b/bot/decorators.py @@ -1,12 +1,12 @@ import logging import random -from asyncio import Lock, sleep +from asyncio import Lock, create_task, sleep from contextlib import suppress from functools import wraps from typing import Callable, Container, Optional, Union from weakref import WeakValueDictionary -from discord import Colour, Embed, Member, Message +from discord import Colour, Embed, Member from discord.errors import NotFound from discord.ext import commands from discord.ext.commands import CheckFailure, Cog, Context @@ -135,20 +135,6 @@ def locked() -> Callable: return wrap -async def delete_invocation(ctx: Context, message: Message) -> None: - """Task to delete the invocation and user redirection messages.""" - if RedirectOutput.delete_invocation: - await sleep(RedirectOutput.delete_delay) - - with suppress(NotFound): - await message.delete() - log.trace("Redirect output: Deleted user redirection message") - - with suppress(NotFound): - await ctx.message.delete() - log.trace("Redirect output: Deleted invocation message") - - def redirect_output(destination_channel: int, bypass_roles: Container[int] = None) -> Callable: """ Changes the channel in the context of the command to redirect the output to a certain channel. @@ -176,14 +162,23 @@ def redirect_output(destination_channel: int, bypass_roles: Container[int] = Non log.trace(f"Redirecting output of {ctx.author}'s command '{ctx.command.name}' to {redirect_channel.name}") ctx.channel = redirect_channel await ctx.channel.send(f"Here's the output of your command, {ctx.author.mention}") - await func(self, ctx, *args, **kwargs) + create_task(func(self, ctx, *args, **kwargs)) message = await old_channel.send( f"Hey, {ctx.author.mention}, you can find the output of your command here: " f"{redirect_channel.mention}" ) - # we need to run it in a task for the help command - which gets held up if waiting for invocation deletion. - ctx.bot.loop.create_task(delete_invocation(ctx, message)) + if RedirectOutput.delete_invocation: + await sleep(RedirectOutput.delete_delay) + + with suppress(NotFound): + await message.delete() + log.trace("Redirect output: Deleted user redirection message") + + with suppress(NotFound): + await ctx.message.delete() + log.trace("Redirect output: Deleted invocation message") + return inner return wrap diff --git a/bot/pagination.py b/bot/pagination.py index 90c8f849c..b0c4b70e2 100644 --- a/bot/pagination.py +++ b/bot/pagination.py @@ -102,7 +102,7 @@ class LinePaginator(Paginator): timeout: int = 300, footer_text: str = None, url: str = None, - exception_on_empty_embed: bool = False + exception_on_empty_embed: bool = False, ) -> t.Optional[discord.Message]: """ Use a paginator and set of reactions to provide pagination over a set of lines. -- cgit v1.2.3 From 3aeb9e6ac6258fe3446788fc8a731fc8bb5922d4 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 12:23:52 +0200 Subject: Adding redis to docker-compose file. This is almost hilariously easy since we can just use the official image for it. --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 11deceae8..1bcf1008e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,9 @@ services: POSTGRES_PASSWORD: pysite POSTGRES_USER: pysite + redis: + image: redis:5.0.9 + web: image: pythondiscord/site:latest command: ["run", "--debug"] @@ -41,6 +44,7 @@ services: tty: true depends_on: - web + - redis environment: BOT_TOKEN: ${BOT_TOKEN} BOT_API_KEY: badbot13m0n8f570f942013fc818f234916ca531 -- cgit v1.2.3 From 5382dd80611dcf24124477f0e09dfde668df1ace Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 12:24:34 +0200 Subject: Adding redis-py to the Pipfile This is the module we will be using to interface with Redis. --- Pipfile | 1 + Pipfile.lock | 131 ++++++++++++++++++++++++++++++----------------------------- 2 files changed, 67 insertions(+), 65 deletions(-) diff --git a/Pipfile b/Pipfile index 14c9ef926..5f85b1e51 100644 --- a/Pipfile +++ b/Pipfile @@ -23,6 +23,7 @@ colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} statsd = "~=3.3" feedparser = "~=5.2" beautifulsoup4 = "~=4.9" +redis = ">=3.5" [dev-packages] coverage = "~=5.0" diff --git a/Pipfile.lock b/Pipfile.lock index 4e7050a13..1a420182d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "64620e7e825c74fd3010821fb30843b19f5dafb2b5a1f6eafedc0a5febd99b69" + "sha256": "c6b4d38c4034e55a4bd598399f2e1f48b70a76693c986d0db0fae7442e224d41" }, "pipfile-spec": 6, "requires": { @@ -52,10 +52,10 @@ }, "aiormq": { "hashes": [ - "sha256:286e0b0772075580466e45f98f051b9728a9316b9c36f0c14c7bc1409be375b0", - "sha256:7ed7d6df6b57af7f8bce7d1ebcbdfc32b676192e46703e81e9e217316e56b5bd" + "sha256:41a9d4eb17db805f30ed172f3f609fe0c2b16657fb15b1b67df19d251dd93c0d", + "sha256:7c19477a9450824cb79f9949fd238f4148e2c0dca67756a2868863c387209f04" ], - "version": "==3.2.1" + "version": "==3.2.2" }, "alabaster": { "hashes": [ @@ -305,39 +305,39 @@ }, "more-itertools": { "hashes": [ - "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c", - "sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507" + "sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be", + "sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982" ], "index": "pypi", - "version": "==8.2.0" + "version": "==8.3.0" }, "multidict": { "hashes": [ - "sha256:317f96bc0950d249e96d8d29ab556d01dd38888fbe68324f46fd834b430169f1", - "sha256:42f56542166040b4474c0c608ed051732033cd821126493cf25b6c276df7dd35", - "sha256:4b7df040fb5fe826d689204f9b544af469593fb3ff3a069a6ad3409f742f5928", - "sha256:544fae9261232a97102e27a926019100a9db75bec7b37feedd74b3aa82f29969", - "sha256:620b37c3fea181dab09267cd5a84b0f23fa043beb8bc50d8474dd9694de1fa6e", - "sha256:6e6fef114741c4d7ca46da8449038ec8b1e880bbe68674c01ceeb1ac8a648e78", - "sha256:7774e9f6c9af3f12f296131453f7b81dabb7ebdb948483362f5afcaac8a826f1", - "sha256:85cb26c38c96f76b7ff38b86c9d560dea10cf3459bb5f4caf72fc1bb932c7136", - "sha256:a326f4240123a2ac66bb163eeba99578e9d63a8654a59f4688a79198f9aa10f8", - "sha256:ae402f43604e3b2bc41e8ea8b8526c7fa7139ed76b0d64fc48e28125925275b2", - "sha256:aee283c49601fa4c13adc64c09c978838a7e812f85377ae130a24d7198c0331e", - "sha256:b51249fdd2923739cd3efc95a3d6c363b67bbf779208e9f37fd5e68540d1a4d4", - "sha256:bb519becc46275c594410c6c28a8a0adc66fe24fef154a9addea54c1adb006f5", - "sha256:c2c37185fb0af79d5c117b8d2764f4321eeb12ba8c141a95d0aa8c2c1d0a11dd", - "sha256:dc561313279f9d05a3d0ffa89cd15ae477528ea37aa9795c4654588a3287a9ab", - "sha256:e439c9a10a95cb32abd708bb8be83b2134fa93790a4fb0535ca36db3dda94d20", - "sha256:fc3b4adc2ee8474cb3cd2a155305d5f8eda0a9c91320f83e55748e1fcb68f8e3" - ], - "version": "==4.7.5" + "sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a", + "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000", + "sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2", + "sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507", + "sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5", + "sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7", + "sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d", + "sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463", + "sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19", + "sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3", + "sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b", + "sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c", + "sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87", + "sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7", + "sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430", + "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255", + "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d" + ], + "version": "==4.7.6" }, "ordered-set": { "hashes": [ - "sha256:a7bfa858748c73b096e43db14eb23e2bc714a503f990c89fac8fab9b0ee79724" + "sha256:a31008c57f9c9776b12eb8841b1f61d1e4d70dfbbe8875ccfa2403c54af3d51b" ], - "version": "==3.1.1" + "version": "==4.0.1" }, "packaging": { "hashes": [ @@ -418,10 +418,10 @@ }, "pytz": { "hashes": [ - "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", - "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" + "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", + "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048" ], - "version": "==2019.3" + "version": "==2020.1" }, "pyyaml": { "hashes": [ @@ -440,6 +440,14 @@ "index": "pypi", "version": "==5.3.1" }, + "redis": { + "hashes": [ + "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", + "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" + ], + "index": "pypi", + "version": "==3.5.2" + }, "requests": { "hashes": [ "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", @@ -450,11 +458,11 @@ }, "sentry-sdk": { "hashes": [ - "sha256:23808d571d2461a4ce3784ec12bbee5bdb8c026c143fe79d36cef8a6d653e71f", - "sha256:bb90a4e19c7233a580715fc986cc44be2c48fc10b31e71580a2037e1c94b6950" + "sha256:0e5e947d0f7a969314aa23669a94a9712be5a688ff069ff7b9fc36c66adc160c", + "sha256:799a8bf76b012e3030a881be00e97bc0b922ce35dde699c6537122b751d80e2c" ], "index": "pypi", - "version": "==0.14.3" + "version": "==0.14.4" }, "six": { "hashes": [ @@ -595,10 +603,10 @@ "develop": { "appdirs": { "hashes": [ - "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", - "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" ], - "version": "==1.4.3" + "version": "==1.4.4" }, "attrs": { "hashes": [ @@ -657,13 +665,6 @@ ], "version": "==0.3.0" }, - "entrypoints": { - "hashes": [ - "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", - "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" - ], - "version": "==0.3" - }, "filelock": { "hashes": [ "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", @@ -673,11 +674,11 @@ }, "flake8": { "hashes": [ - "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb", - "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca" + "sha256:6c1193b0c3f853ef763969238f6c81e9e63ace9d024518edc020d5f1d6d93195", + "sha256:ea6623797bf9a52f4c9577d780da0bb17d65f870213f7b5bcc9fca82540c31d5" ], "index": "pypi", - "version": "==3.7.9" + "version": "==3.8.1" }, "flake8-annotations": { "hashes": [ @@ -743,10 +744,10 @@ }, "identify": { "hashes": [ - "sha256:2bb8760d97d8df4408f4e805883dad26a2d076f04be92a10a3e43f09c6060742", - "sha256:faffea0fd8ec86bb146ac538ac350ed0c73908326426d387eded0bcc9d077522" + "sha256:23c18d97bb50e05be1a54917ee45cc61d57cb96aedc06aabb2b02331edf0dbf0", + "sha256:88ed90632023e52a6495749c6732e61e08ec9f4f04e95484a5c37b9caf40283c" ], - "version": "==1.4.14" + "version": "==1.4.15" }, "mccabe": { "hashes": [ @@ -771,18 +772,18 @@ }, "pre-commit": { "hashes": [ - "sha256:487c675916e6f99d355ec5595ad77b325689d423ef4839db1ed2f02f639c9522", - "sha256:c0aa11bce04a7b46c5544723aedf4e81a4d5f64ad1205a30a9ea12d5e81969e1" + "sha256:5559e09afcac7808933951ffaf4ff9aac524f31efbc3f24d021540b6c579813c", + "sha256:703e2e34cbe0eedb0d319eff9f7b83e2022bb5a3ab5289a6a8841441076514d0" ], "index": "pypi", - "version": "==2.2.0" + "version": "==2.4.0" }, "pycodestyle": { "hashes": [ - "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", - "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" + "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367", + "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e" ], - "version": "==2.5.0" + "version": "==2.6.0" }, "pydocstyle": { "hashes": [ @@ -793,10 +794,10 @@ }, "pyflakes": { "hashes": [ - "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", - "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" + "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92", + "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8" ], - "version": "==2.1.1" + "version": "==2.2.0" }, "pyyaml": { "hashes": [ @@ -831,10 +832,10 @@ }, "toml": { "hashes": [ - "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", - "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" + "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", + "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88" ], - "version": "==0.10.0" + "version": "==0.10.1" }, "unittest-xml-reporting": { "hashes": [ @@ -846,10 +847,10 @@ }, "virtualenv": { "hashes": [ - "sha256:5021396e8f03d0d002a770da90e31e61159684db2859d0ba4850fbea752aa675", - "sha256:ac53ade75ca189bc97b6c1d9ec0f1a50efe33cbf178ae09452dcd9fd309013c1" + "sha256:b4c14d4d73a0c23db267095383c4276ef60e161f94fde0427f2f21a0132dde74", + "sha256:fd0e54dec8ac96c1c7c87daba85f0a59a7c37fe38748e154306ca21c73244637" ], - "version": "==20.0.18" + "version": "==20.0.20" } } } -- cgit v1.2.3 From 7a501fdecaae186590177fd4ebd6cea64119629e Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 12:40:06 +0200 Subject: Boilerplate for the RedisCacheMixin We're using __init_subclass__ to initialize our RedisDict with the subclass name as a namespace. This will be prefixed to all data that we store, so that there won't be collisions between different subclasses. --- bot/mixins/__init__.py | 3 +++ bot/mixins/redis.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 bot/mixins/__init__.py create mode 100644 bot/mixins/redis.py diff --git a/bot/mixins/__init__.py b/bot/mixins/__init__.py new file mode 100644 index 000000000..ff1f0c50d --- /dev/null +++ b/bot/mixins/__init__.py @@ -0,0 +1,3 @@ +from .redis import RedisCacheMixin + +__all__ = ['RedisCacheMixin'] diff --git a/bot/mixins/redis.py b/bot/mixins/redis.py new file mode 100644 index 000000000..f19108576 --- /dev/null +++ b/bot/mixins/redis.py @@ -0,0 +1,56 @@ +import redis as redis_py + +redis = redis_py.Redis(host="redis") + + +class RedisDict(dict): + """ + A dictionary interface for a Redis database. + + Objects created by this class should mostly behave like a normal dictionary, + but will store all the data in our Redis database for persistence between restarts. + + There are, however, a few limitations to what kinds of data types can be + stored on Redis, so this is a little bit more limited than a regular dict. + """ + + def __init__(self, namespace: str = "global"): + """Initialize the RedisDict with the right namespace.""" + # TODO: Make namespace collision impossible! + # Append a number or something if it exists already. + self.namespace = namespace + + # redis.mset({"firedog": "donkeykong"}) + # + # print(redis.get("firedog").decode("utf-8") + + +class RedisCacheMixin: + """ + A mixin which adds a cls.cache parameter which can be used for persistent caching. + + This adds a dictionary-like object called cache which can be treated like a regular dictionary, + but which can only store simple data types like ints, strings, and floats. + + To use it, simply subclass it into your class like this: + + class MyCog(Cog, RedisCacheMixin): + def some_command(self): + # You can now do this! + self.cache['some_data'] = some_data + + All the data stored in this cache will probably be available permanently, even if the bot restarts or + is updated. However, Redis is not meant to be used for reliable, permanent storage. It may be cleared + from time to time, so please only use it for caching data that you can afford to lose. + + If it's really important that your data should never disappear, please use our postgres database instead. + """ + + def __init_subclass__(cls, **kwargs): + """ + Initialize the cache when subclass is created. + + When this mixin is subclassed, we create a cache using the subclass name as the namespace. + This is to prevent collisions between subclasses. + """ + cls.cache = RedisDict(cls.__name__) -- cgit v1.2.3 From 93cce50846b1cfcf520535d69a9fe223c2cd4d7a Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 16 May 2020 16:49:48 +0300 Subject: Stats: Create guild boost stat collection Collect Guild boost amount + level and post it to StatsD every hour in task. Added starting to cog `__init__.py` and stopping to `cog_unload`. --- bot/cogs/stats.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py index d253db913..acee1f5a9 100644 --- a/bot/cogs/stats.py +++ b/bot/cogs/stats.py @@ -2,8 +2,10 @@ import string from datetime import datetime from discord import Member, Message, Status -from discord.ext.commands import Bot, Cog, Context +from discord.ext.commands import Cog, Context +from discord.ext.tasks import loop +from bot.bot import Bot from bot.constants import Channels, Guild, Stats as StatConf @@ -23,6 +25,7 @@ class Stats(Cog): def __init__(self, bot: Bot): self.bot = bot self.last_presence_update = None + self.update_guild_boost.start() @Cog.listener() async def on_message(self, message: Message) -> None: @@ -101,6 +104,18 @@ class Stats(Cog): self.bot.stats.gauge("guild.status.do_not_disturb", dnd) self.bot.stats.gauge("guild.status.offline", offline) + @loop(hours=1) + async def update_guild_boost(self) -> None: + """Update every hour guild boosts amount + level.""" + await self.bot.wait_until_guild_available() + g = self.bot.get_guild(Guild.id) + self.bot.stats.gauge("boost.amount", g.premium_subscription_count) + self.bot.stats.gauge("boost.tier", g.premium_tier) + + def cog_unload(self) -> None: + """Stop guild boost stat collecting task on Cog unload.""" + self.update_guild_boost.stop() + def setup(bot: Bot) -> None: """Load the stats cog.""" -- cgit v1.2.3 From 1a6abaac12eb2e6ab0d26065108fe1cce9a7be45 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 16 May 2020 17:00:25 +0300 Subject: Stats: Added codeblock correction stats --- bot/cogs/bot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/cogs/bot.py b/bot/cogs/bot.py index a6929b431..67ff8f95d 100644 --- a/bot/cogs/bot.py +++ b/bot/cogs/bot.py @@ -326,6 +326,8 @@ class BotCog(Cog, name="Bot"): log.trace("The code consists only of expressions, not sending instructions") if howto != "": + # Increase amount of codeblock correction in stats + self.bot.stats.incr("codeblock_corrections") howto_embed = Embed(description=howto) bot_message = await msg.channel.send(f"Hey {msg.author.mention}!", embed=howto_embed) self.codeblock_message_ids[msg.id] = bot_message.id -- cgit v1.2.3 From 158e19a6fcb2056c6bcc244a1f02d8b75d7fe503 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 16 May 2020 17:13:54 +0300 Subject: Stats: Added stats for eval successes + fails --- bot/cogs/snekbox.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 8d4688114..1c64c893b 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -205,6 +205,12 @@ class Snekbox(Cog): if paste_link: msg = f"{msg}\nFull output: {paste_link}" + # Collect stats of eval fails + successes + if icon == ":x:": + self.bot.stats.incr("evals.fail") + elif icon in (":warning:", ":white_check_mark:"): + self.bot.stats.incr("evals.success") + response = await ctx.send(msg) self.bot.loop.create_task( wait_for_deletion(response, user_ids=(ctx.author.id,), client=ctx.bot) -- cgit v1.2.3 From 5878ec93c5b883038b0f738e9bbbdee8fd1929ad Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 16 May 2020 17:25:25 +0300 Subject: Stats: Added stats for eval role uses (Helpers/Developers) --- bot/cogs/snekbox.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 1c64c893b..04c0a5ae1 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -298,6 +298,11 @@ class Snekbox(Cog): await ctx.invoke(self.bot.get_command("help"), "eval") return + if Roles.helpers in (role.id for role in ctx.author.roles): + self.bot.stats.incr("evals.roles.helpers") + else: + self.bot.stats.incr("evals.roles.developers") + log.info(f"Received code from {ctx.author} for evaluation:\n{code}") while True: -- cgit v1.2.3 From 613b00a5ec060f409d5838cb1a648d9770cecfde Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 16 May 2020 18:19:18 +0300 Subject: Stats: Added stats for eval channel using (Help/Bot commands/Topical) --- bot/cogs/snekbox.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 04c0a5ae1..1d240d8d8 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -303,6 +303,13 @@ class Snekbox(Cog): else: self.bot.stats.incr("evals.roles.developers") + if ctx.channel.category_id == Categories.help_in_use: + self.bot.stats.incr("evals.channels.help") + elif ctx.channel.id == Channels.bot_commands: + self.bot.stats.incr("evals.channels.bot_commands") + else: + self.bot.stats.incr("evals.channels.topical") + log.info(f"Received code from {ctx.author} for evaluation:\n{code}") while True: -- cgit v1.2.3 From 588521c82403f6d66693512c6d33272cc370d755 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 20:22:05 +0200 Subject: Refactor - no more mixins! It was brought to my attention that we may need several caches per Cog for some of our Cogs. This means that the original approach of having this be a mixin is a little bit problematic. Instead, RedisDict will be instantiated directly inside the class you want it in. By leveraging __set_name__, we can create a namespace containing both the class name and the variable name without the user having to provide anything. For example, if you create an attribute MyClass.cache = RedisDict(), this will be using the redis namespace 'MyClass.cache.' before anything you store in it. With this approach, it is also possible to instantiate a RedisDict with a custom namespace by simply passing it into the constructor. - RedisDict("firedog") will create items with the 'firedog.your_item' prefix. - If there are multiple RedisDicts using the same namespace, an underscore will be appended to the namespace, such that the second RedisDict("firedog") will actually create items in the 'firedog_.your_item' namespace. This is also possible to use outside of classes, so long as you provide a custom namespace when you instantiate it. Custom namespaces will always take precedence over automatic 'Class.attribute_name' ones. --- bot/mixins/__init__.py | 3 --- bot/mixins/redis.py | 56 ---------------------------------------- bot/utils/__init__.py | 4 +++ bot/utils/redis.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 59 deletions(-) delete mode 100644 bot/mixins/__init__.py delete mode 100644 bot/mixins/redis.py create mode 100644 bot/utils/redis.py diff --git a/bot/mixins/__init__.py b/bot/mixins/__init__.py deleted file mode 100644 index ff1f0c50d..000000000 --- a/bot/mixins/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .redis import RedisCacheMixin - -__all__ = ['RedisCacheMixin'] diff --git a/bot/mixins/redis.py b/bot/mixins/redis.py deleted file mode 100644 index f19108576..000000000 --- a/bot/mixins/redis.py +++ /dev/null @@ -1,56 +0,0 @@ -import redis as redis_py - -redis = redis_py.Redis(host="redis") - - -class RedisDict(dict): - """ - A dictionary interface for a Redis database. - - Objects created by this class should mostly behave like a normal dictionary, - but will store all the data in our Redis database for persistence between restarts. - - There are, however, a few limitations to what kinds of data types can be - stored on Redis, so this is a little bit more limited than a regular dict. - """ - - def __init__(self, namespace: str = "global"): - """Initialize the RedisDict with the right namespace.""" - # TODO: Make namespace collision impossible! - # Append a number or something if it exists already. - self.namespace = namespace - - # redis.mset({"firedog": "donkeykong"}) - # - # print(redis.get("firedog").decode("utf-8") - - -class RedisCacheMixin: - """ - A mixin which adds a cls.cache parameter which can be used for persistent caching. - - This adds a dictionary-like object called cache which can be treated like a regular dictionary, - but which can only store simple data types like ints, strings, and floats. - - To use it, simply subclass it into your class like this: - - class MyCog(Cog, RedisCacheMixin): - def some_command(self): - # You can now do this! - self.cache['some_data'] = some_data - - All the data stored in this cache will probably be available permanently, even if the bot restarts or - is updated. However, Redis is not meant to be used for reliable, permanent storage. It may be cleared - from time to time, so please only use it for caching data that you can afford to lose. - - If it's really important that your data should never disappear, please use our postgres database instead. - """ - - def __init_subclass__(cls, **kwargs): - """ - Initialize the cache when subclass is created. - - When this mixin is subclassed, we create a cache using the subclass name as the namespace. - This is to prevent collisions between subclasses. - """ - cls.cache = RedisDict(cls.__name__) diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 9b32e515d..7ae2db8fe 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -2,6 +2,10 @@ from abc import ABCMeta from discord.ext.commands import CogMeta +from bot.utils.redis import RedisDict + +__all__ = ['RedisDict', 'CogABCMeta'] + class CogABCMeta(CogMeta, ABCMeta): """Metaclass for ABCs meant to be implemented as Cogs.""" diff --git a/bot/utils/redis.py b/bot/utils/redis.py new file mode 100644 index 000000000..8b33e8977 --- /dev/null +++ b/bot/utils/redis.py @@ -0,0 +1,70 @@ +from collections.abc import MutableMapping +from typing import Optional + +import redis as redis_py + +redis = redis_py.Redis(host="redis") + + +class RedisDict(MutableMapping): + """ + A dictionary interface for a Redis database. + + Objects created by this class should mostly behave like a normal dictionary, + but will store all the data in our Redis database for persistence between restarts. + + Redis is limited to simple types, so to allow you to store collections like lists + and dictionaries, we JSON deserialize every value. That means that it will not be possible + to store complex objects, only stuff like strings, numbers, and collections of strings and numbers. + + TODO: Implement these: + __delitem__ + __getitem__ + __setitem__ + __iter__ + __len__ + clear (just use DEL and the hash goes) + copy (convert to dict maybe?) + pop + popitem + setdefault + update + + TODO: TEST THESE + .get + .items + .keys + .values + .__eg__ + .__ne__ + """ + + namespaces = [] + + def _set_namespace(self, namespace: str) -> None: + """Try to set the namespace, but do not permit collisions.""" + while namespace in self.namespaces: + namespace = namespace + "_" + + self.namespaces.append(namespace) + self.namespace = namespace + + def __init__(self, namespace: Optional[str] = None) -> None: + """Initialize the RedisDict with the right namespace.""" + super().__init__() + self.has_custom_namespace = namespace is not None + self._set_namespace(namespace) + + def __set_name__(self, owner: object, attribute_name: str) -> None: + """ + Set the namespace to Class.attribute_name. + + Called automatically when this class is constructed inside a class as an attribute, as long as + no custom namespace is provided to the constructor. + """ + if not self.has_custom_namespace: + self._set_namespace(f"{owner.__name__}.{attribute_name}") + + def __repr__(self) -> str: + """Return a beautiful representation of this object instance.""" + return f"RedisDict(namespace={self.namespace!r})" -- cgit v1.2.3 From ee8386e67aa7d298b4761eef79b927c3066fe037 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 23:40:21 +0200 Subject: Add basic dict methods for RedisDict. The rest of the features should be provided by the MutableMapping abc we're interfacing. Specifically, MutableMapping provides these: .pop, .popitem, .clear, .update, .setdefault, __contains__, .keys, .items, .values, .get, __eq__, and __ne__. --- bot/utils/redis.py | 80 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/bot/utils/redis.py b/bot/utils/redis.py index 8b33e8977..1e8c6422d 100644 --- a/bot/utils/redis.py +++ b/bot/utils/redis.py @@ -1,9 +1,12 @@ +import json from collections.abc import MutableMapping -from typing import Optional +from enum import Enum +from typing import Dict, List, Optional, Tuple, Union import redis as redis_py -redis = redis_py.Redis(host="redis") +ValidRedisKey = Union[str, int, float] +JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] class RedisDict(MutableMapping): @@ -16,45 +19,25 @@ class RedisDict(MutableMapping): Redis is limited to simple types, so to allow you to store collections like lists and dictionaries, we JSON deserialize every value. That means that it will not be possible to store complex objects, only stuff like strings, numbers, and collections of strings and numbers. - - TODO: Implement these: - __delitem__ - __getitem__ - __setitem__ - __iter__ - __len__ - clear (just use DEL and the hash goes) - copy (convert to dict maybe?) - pop - popitem - setdefault - update - - TODO: TEST THESE - .get - .items - .keys - .values - .__eg__ - .__ne__ """ - namespaces = [] - - def _set_namespace(self, namespace: str) -> None: - """Try to set the namespace, but do not permit collisions.""" - while namespace in self.namespaces: - namespace = namespace + "_" - - self.namespaces.append(namespace) - self.namespace = namespace + _namespaces = [] + _redis = redis_py.Redis(host="redis") # Can be overridden for testing def __init__(self, namespace: Optional[str] = None) -> None: """Initialize the RedisDict with the right namespace.""" super().__init__() - self.has_custom_namespace = namespace is not None + self._has_custom_namespace = namespace is not None self._set_namespace(namespace) + def _set_namespace(self, namespace: str) -> None: + """Try to set the namespace, but do not permit collisions.""" + while namespace in self._namespaces: + namespace = namespace + "_" + + self._namespaces.append(namespace) + self._namespace = namespace + def __set_name__(self, owner: object, attribute_name: str) -> None: """ Set the namespace to Class.attribute_name. @@ -62,9 +45,36 @@ class RedisDict(MutableMapping): Called automatically when this class is constructed inside a class as an attribute, as long as no custom namespace is provided to the constructor. """ - if not self.has_custom_namespace: + if not self._has_custom_namespace: self._set_namespace(f"{owner.__name__}.{attribute_name}") def __repr__(self) -> str: """Return a beautiful representation of this object instance.""" - return f"RedisDict(namespace={self.namespace!r})" + return f"RedisDict(namespace={self._namespace!r})" + + def __setitem__(self, key: ValidRedisKey, value: JSONSerializableType): + """Store an item in the Redis cache.""" + # JSON serialize the value before storing it. + json_value = json.dumps(value) + self._redis.hset(self._namespace, key, json_value) + + def __getitem__(self, key: ValidRedisKey): + """Get an item from the Redis cache.""" + value = self._redis.hget(self._namespace, key) + return json.loads(value) + + def __delitem__(self, key: ValidRedisKey): + """Delete an item from the Redis cache.""" + self._redis.hdel(self._namespace, key) + + def __iter__(self): + """Iterate all the items in the Redis cache.""" + return iter(self._redis.hkeys(self._namespace)) + + def __len__(self): + """Return the number of items in the Redis cache.""" + return self._redis.hlen(self._namespace) + + def copy(self) -> Dict: + """Convert to dict and return.""" + return dict(self) -- cgit v1.2.3 From 5859711a05e634d7b788944b8292071cb1e7cf72 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 23:43:22 +0200 Subject: copy should dictify the .items(), not just keys. --- bot/utils/redis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/redis.py b/bot/utils/redis.py index 1e8c6422d..470de47b7 100644 --- a/bot/utils/redis.py +++ b/bot/utils/redis.py @@ -77,4 +77,4 @@ class RedisDict(MutableMapping): def copy(self) -> Dict: """Convert to dict and return.""" - return dict(self) + return dict(self.items()) -- cgit v1.2.3 From 9eeee1ce303b7ebac4fa9db37193921d052d0f8d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 00:10:29 +0200 Subject: Implements .clear with hash deletion. This would've been implemented by MutableMapping, but that implementation is O(n) instead of O(1) since it just iterates the entire hash and does HDEL. Feels wasteful. --- bot/utils/redis.py | 80 ---------------------------------------------- bot/utils/redis_dict.py | 84 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 80 deletions(-) delete mode 100644 bot/utils/redis.py create mode 100644 bot/utils/redis_dict.py diff --git a/bot/utils/redis.py b/bot/utils/redis.py deleted file mode 100644 index 470de47b7..000000000 --- a/bot/utils/redis.py +++ /dev/null @@ -1,80 +0,0 @@ -import json -from collections.abc import MutableMapping -from enum import Enum -from typing import Dict, List, Optional, Tuple, Union - -import redis as redis_py - -ValidRedisKey = Union[str, int, float] -JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] - - -class RedisDict(MutableMapping): - """ - A dictionary interface for a Redis database. - - Objects created by this class should mostly behave like a normal dictionary, - but will store all the data in our Redis database for persistence between restarts. - - Redis is limited to simple types, so to allow you to store collections like lists - and dictionaries, we JSON deserialize every value. That means that it will not be possible - to store complex objects, only stuff like strings, numbers, and collections of strings and numbers. - """ - - _namespaces = [] - _redis = redis_py.Redis(host="redis") # Can be overridden for testing - - def __init__(self, namespace: Optional[str] = None) -> None: - """Initialize the RedisDict with the right namespace.""" - super().__init__() - self._has_custom_namespace = namespace is not None - self._set_namespace(namespace) - - def _set_namespace(self, namespace: str) -> None: - """Try to set the namespace, but do not permit collisions.""" - while namespace in self._namespaces: - namespace = namespace + "_" - - self._namespaces.append(namespace) - self._namespace = namespace - - def __set_name__(self, owner: object, attribute_name: str) -> None: - """ - Set the namespace to Class.attribute_name. - - Called automatically when this class is constructed inside a class as an attribute, as long as - no custom namespace is provided to the constructor. - """ - if not self._has_custom_namespace: - self._set_namespace(f"{owner.__name__}.{attribute_name}") - - def __repr__(self) -> str: - """Return a beautiful representation of this object instance.""" - return f"RedisDict(namespace={self._namespace!r})" - - def __setitem__(self, key: ValidRedisKey, value: JSONSerializableType): - """Store an item in the Redis cache.""" - # JSON serialize the value before storing it. - json_value = json.dumps(value) - self._redis.hset(self._namespace, key, json_value) - - def __getitem__(self, key: ValidRedisKey): - """Get an item from the Redis cache.""" - value = self._redis.hget(self._namespace, key) - return json.loads(value) - - def __delitem__(self, key: ValidRedisKey): - """Delete an item from the Redis cache.""" - self._redis.hdel(self._namespace, key) - - def __iter__(self): - """Iterate all the items in the Redis cache.""" - return iter(self._redis.hkeys(self._namespace)) - - def __len__(self): - """Return the number of items in the Redis cache.""" - return self._redis.hlen(self._namespace) - - def copy(self) -> Dict: - """Convert to dict and return.""" - return dict(self.items()) diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py new file mode 100644 index 000000000..b2fd7d2e9 --- /dev/null +++ b/bot/utils/redis_dict.py @@ -0,0 +1,84 @@ +import json +from collections.abc import MutableMapping +from enum import Enum +from typing import Dict, List, Optional, Tuple, Union + +import redis as redis_py + +ValidRedisKey = Union[str, int, float] +JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] + + +class RedisDict(MutableMapping): + """ + A dictionary interface for a Redis database. + + Objects created by this class should mostly behave like a normal dictionary, + but will store all the data in our Redis database for persistence between restarts. + + Redis is limited to simple types, so to allow you to store collections like lists + and dictionaries, we JSON deserialize every value. That means that it will not be possible + to store complex objects, only stuff like strings, numbers, and collections of strings and numbers. + """ + + _namespaces = [] + _redis = redis_py.Redis(host="redis") # Can be overridden for testing + + def __init__(self, namespace: Optional[str] = None) -> None: + """Initialize the RedisDict with the right namespace.""" + super().__init__() + self._has_custom_namespace = namespace is not None + self._set_namespace(namespace) + + def _set_namespace(self, namespace: str) -> None: + """Try to set the namespace, but do not permit collisions.""" + while namespace in self._namespaces: + namespace = namespace + "_" + + self._namespaces.append(namespace) + self._namespace = namespace + + def __set_name__(self, owner: object, attribute_name: str) -> None: + """ + Set the namespace to Class.attribute_name. + + Called automatically when this class is constructed inside a class as an attribute, as long as + no custom namespace is provided to the constructor. + """ + if not self._has_custom_namespace: + self._set_namespace(f"{owner.__name__}.{attribute_name}") + + def __repr__(self) -> str: + """Return a beautiful representation of this object instance.""" + return f"RedisDict(namespace={self._namespace!r})" + + def __setitem__(self, key: ValidRedisKey, value: JSONSerializableType): + """Store an item in the Redis cache.""" + # JSON serialize the value before storing it. + json_value = json.dumps(value) + self._redis.hset(self._namespace, key, json_value) + + def __getitem__(self, key: ValidRedisKey): + """Get an item from the Redis cache.""" + value = self._redis.hget(self._namespace, key) + return json.loads(value) + + def __delitem__(self, key: ValidRedisKey): + """Delete an item from the Redis cache.""" + self._redis.hdel(self._namespace, key) + + def __iter__(self): + """Iterate all the items in the Redis cache.""" + return iter(self._redis.hkeys(self._namespace)) + + def __len__(self): + """Return the number of items in the Redis cache.""" + return self._redis.hlen(self._namespace) + + def copy(self) -> Dict: + """Convert to dict and return.""" + return dict(self.items()) + + def clear(self) -> None: + """Deletes the entire hash from the Redis cache.""" + self._redis.delete(self._namespace) -- cgit v1.2.3 From 3a278c7ff64951e0bb7ed05d822f60e5a70d34d9 Mon Sep 17 00:00:00 2001 From: vivax3794 <51753506+vivax3794@users.noreply.github.com> Date: Sun, 17 May 2020 00:38:17 +0200 Subject: added "solved" as a alias for "closed" --- bot/cogs/help_channels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index b714a1642..1bd1f9d68 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -225,7 +225,7 @@ class HelpChannels(Scheduler, commands.Cog): return role_check - @commands.command(name="close", aliases=["dormant"], enabled=False) + @commands.command(name="close", aliases=["dormant", "solved"], enabled=False) async def close_command(self, ctx: commands.Context) -> None: """ Make the current in-use help channel dormant. -- cgit v1.2.3 From 677a7f755a15f8fdf0cd97e399c4265dd8e702d9 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 02:30:28 +0200 Subject: Implement .get, equality, and membership check This is supposed to be provided by our MutableMapping mixin, but unit tests are demonstrating that these don't really work as intended. --- bot/utils/redis_dict.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py index b2fd7d2e9..35439b2f3 100644 --- a/bot/utils/redis_dict.py +++ b/bot/utils/redis_dict.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json from collections.abc import MutableMapping from enum import Enum @@ -28,7 +30,11 @@ class RedisDict(MutableMapping): """Initialize the RedisDict with the right namespace.""" super().__init__() self._has_custom_namespace = namespace is not None - self._set_namespace(namespace) + + if self._has_custom_namespace: + self._set_namespace(namespace) + else: + self.namespace = "general" def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" @@ -52,6 +58,14 @@ class RedisDict(MutableMapping): """Return a beautiful representation of this object instance.""" return f"RedisDict(namespace={self._namespace!r})" + def __eq__(self, other: RedisDict) -> bool: + """Check equality between two RedisDicts.""" + return self.items() == other.items() and self._namespace == other._namespace + + def __ne__(self, other: RedisDict) -> bool: + """Check inequality between two RedisDicts.""" + return self.items() != other.items() or self._namespace != other._namespace + def __setitem__(self, key: ValidRedisKey, value: JSONSerializableType): """Store an item in the Redis cache.""" # JSON serialize the value before storing it. @@ -61,12 +75,18 @@ class RedisDict(MutableMapping): def __getitem__(self, key: ValidRedisKey): """Get an item from the Redis cache.""" value = self._redis.hget(self._namespace, key) - return json.loads(value) + + if value: + return json.loads(value) def __delitem__(self, key: ValidRedisKey): """Delete an item from the Redis cache.""" self._redis.hdel(self._namespace, key) + def __contains__(self, key: ValidRedisKey): + """Check if a key exists in the Redis cache.""" + return self._redis.hexists(self._namespace, key) + def __iter__(self): """Iterate all the items in the Redis cache.""" return iter(self._redis.hkeys(self._namespace)) @@ -82,3 +102,10 @@ class RedisDict(MutableMapping): def clear(self) -> None: """Deletes the entire hash from the Redis cache.""" self._redis.delete(self._namespace) + + def get(self, key: ValidRedisKey, default: Optional[str] = None) -> JSONSerializableType: + """Get the item, but provide a default if not found.""" + if key in self: + return self[key] + else: + return default -- cgit v1.2.3 From 0843728927fe73be2f2e2c37381ead1497debe11 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 02:40:30 +0200 Subject: Add fakeredis to the Pipfile --- Pipfile | 3 ++- Pipfile.lock | 25 ++++++++++++++++++++++++- bot/utils/__init__.py | 2 +- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/Pipfile b/Pipfile index 5f85b1e51..40ae52761 100644 --- a/Pipfile +++ b/Pipfile @@ -23,10 +23,11 @@ colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} statsd = "~=3.3" feedparser = "~=5.2" beautifulsoup4 = "~=4.9" -redis = ">=3.5" +redis = "~=3.5" [dev-packages] coverage = "~=5.0" +fakeredis = "~=1.4" flake8 = "~=3.7" flake8-annotations = "~=2.0" flake8-bugbear = "~=20.1" diff --git a/Pipfile.lock b/Pipfile.lock index 1a420182d..414f4a053 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c6b4d38c4034e55a4bd598399f2e1f48b70a76693c986d0db0fae7442e224d41" + "sha256": "8ec71e9c46d52bf3b8c72939519e993715c79b4bc9e6ad164c1cf88951dc48b4" }, "pipfile-spec": 6, "requires": { @@ -665,6 +665,14 @@ ], "version": "==0.3.0" }, + "fakeredis": { + "hashes": [ + "sha256:4d170886865a91dbc8b7f8cbd4e5d488f4c5f2f25dfae127f001617bbe9e8f97", + "sha256:647b2593d349d9d4e566c8dadb2e4c71ba35be5bdc4f1f7ac2d565a12a965053" + ], + "index": "pypi", + "version": "==1.4.1" + }, "filelock": { "hashes": [ "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", @@ -816,6 +824,14 @@ "index": "pypi", "version": "==5.3.1" }, + "redis": { + "hashes": [ + "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", + "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" + ], + "index": "pypi", + "version": "==3.5.2" + }, "six": { "hashes": [ "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", @@ -830,6 +846,13 @@ ], "version": "==2.0.0" }, + "sortedcontainers": { + "hashes": [ + "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", + "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" + ], + "version": "==2.1.0" + }, "toml": { "hashes": [ "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 7ae2db8fe..5ce383bf2 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -2,7 +2,7 @@ from abc import ABCMeta from discord.ext.commands import CogMeta -from bot.utils.redis import RedisDict +from bot.utils.redis_dict import RedisDict __all__ = ['RedisDict', 'CogABCMeta'] -- cgit v1.2.3 From 7cf0e83d1079ed34a3839948ce6823d95e0ebb62 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 03:16:03 +0200 Subject: Implement .pop, .popitem and .setdefault. Turns out the MutableMapping class doesn't give us servicable implementations of these, so we need to implement them ourselves. Also, let's not have keys returned as bytestrings. --- bot/utils/redis_dict.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py index 35439b2f3..c89765a24 100644 --- a/bot/utils/redis_dict.py +++ b/bot/utils/redis_dict.py @@ -89,7 +89,8 @@ class RedisDict(MutableMapping): def __iter__(self): """Iterate all the items in the Redis cache.""" - return iter(self._redis.hkeys(self._namespace)) + keys = self._redis.hkeys(self._namespace) + return iter([key.decode('utf-8') for key in keys]) def __len__(self): """Return the number of items in the Redis cache.""" @@ -103,9 +104,28 @@ class RedisDict(MutableMapping): """Deletes the entire hash from the Redis cache.""" self._redis.delete(self._namespace) - def get(self, key: ValidRedisKey, default: Optional[str] = None) -> JSONSerializableType: + def get(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: """Get the item, but provide a default if not found.""" if key in self: return self[key] else: return default + + def pop(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + """Get the item, remove it from the cache, and provide a default if not found.""" + value = self.get(key, default) + del self[key] + return value + + def popitem(self) -> JSONSerializableType: + """Get the last item added to the cache.""" + key = list(self.keys())[-1] + return self.pop(key) + + def setdefault(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + """Try to get the item. If the item does not exist, set it to `default` and return that.""" + value = self.get(key) + + if value is None: + self[key] = default + return default -- cgit v1.2.3 From bf6c113319d47594e103c71f8ff5b0ea48d15b38 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 03:18:19 +0200 Subject: Test suite for the redis dict. --- tests/bot/utils/test_redis_dict.py | 189 +++++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 tests/bot/utils/test_redis_dict.py diff --git a/tests/bot/utils/test_redis_dict.py b/tests/bot/utils/test_redis_dict.py new file mode 100644 index 000000000..f422887ce --- /dev/null +++ b/tests/bot/utils/test_redis_dict.py @@ -0,0 +1,189 @@ +import unittest + +import fakeredis +from redis import DataError + +from bot.utils import RedisDict + +redis_server = fakeredis.FakeServer() +RedisDict._redis = fakeredis.FakeStrictRedis(server=redis_server) + + +class RedisDictTests(unittest.TestCase): + """Tests the RedisDict class from utils.redis_dict.py.""" + + redis = RedisDict() + + def test_class_attribute_namespace(self): + """Test that RedisDict creates a namespace automatically for class attributes.""" + self.assertEqual(self.redis._namespace, "RedisDictTests.redis") + + def test_custom_namespace(self): + """Test that users can set a custom namespaces which never collide.""" + test_cases = ( + (RedisDict("firedog")._namespace, "firedog"), + (RedisDict("firedog")._namespace, "firedog_"), + (RedisDict("firedog")._namespace, "firedog__"), + ) + + for test_case, result in test_cases: + self.assertEqual(test_case, result) + + def test_custom_namespace_takes_precedence(self): + """Test that custom namespaces take precedence over class attribute ones.""" + class LemonJuice: + citrus = RedisDict("citrus") + watercat = RedisDict() + + test_class = LemonJuice() + self.assertEqual(test_class.citrus._namespace, "citrus") + self.assertEqual(test_class.watercat._namespace, "LemonJuice.watercat") + + def test_set_get_item(self): + """Test that users can set and get items from the RedisDict.""" + self.redis['favorite_fruit'] = 'melon' + self.redis['favorite_number'] = 86 + self.assertEqual(self.redis['favorite_fruit'], 'melon') + self.assertEqual(self.redis['favorite_number'], 86) + + def test_set_item_value_types(self): + """Test that setitem rejects values that are not JSON serializable.""" + with self.assertRaises(TypeError): + self.redis['favorite_thing'] = object + self.redis['favorite_stuff'] = RedisDict + + def test_set_item_key_types(self): + """Test that setitem rejects keys that are not strings, ints or floats.""" + fruits = ["lemon", "melon", "apple"] + + with self.assertRaises(DataError): + self.redis[fruits] = "nice" + + def test_get_method(self): + """Test that the .get method works like in a dict.""" + self.redis['favorite_movie'] = 'Code Jam Highlights' + + self.assertEqual(self.redis.get('favorite_movie'), 'Code Jam Highlights') + self.assertEqual(self.redis.get('favorite_youtuber', 'pydis'), 'pydis') + self.assertIsNone(self.redis.get('favorite_dog')) + + def test_membership(self): + """Test that we can reliably use the `in` operator with our RedisDict.""" + self.redis['favorite_country'] = "Burkina Faso" + + self.assertIn('favorite_country', self.redis) + self.assertNotIn('favorite_dentist', self.redis) + + def test_del_item(self): + """Test that users can delete items from the RedisDict.""" + self.redis['favorite_band'] = "Radiohead" + self.assertIn('favorite_band', self.redis) + + del self.redis['favorite_band'] + self.assertNotIn('favorite_band', self.redis) + + def test_iter(self): + """Test that the RedisDict can be iterated.""" + self.redis.clear() + test_cases = ( + ('favorite_turtle', 'Donatello'), + ('second_favorite_turtle', 'Leonardo'), + ('third_favorite_turtle', 'Raphael'), + ) + for key, value in test_cases: + self.redis[key] = value + + # Test regular iteration + for test_case, key in zip(test_cases, self.redis): + value = test_case[1] + self.assertEqual(self.redis[key], value) + + # Test .items iteration + for key, value in self.redis.items(): + self.assertEqual(self.redis[key], value) + + # Test .keys iteration + for test_case, key in zip(test_cases, self.redis.keys()): + value = test_case[1] + self.assertEqual(self.redis[key], value) + + def test_len(self): + """Test that we can get the correct len() from the RedisDict.""" + self.redis.clear() + self.redis['one'] = 1 + self.redis['two'] = 2 + self.redis['three'] = 3 + self.assertEqual(len(self.redis), 3) + + self.redis['four'] = 4 + self.assertEqual(len(self.redis), 4) + + def test_copy(self): + """Test that the .copy method returns a workable dictionary copy.""" + copy = self.redis.copy() + local_copy = dict(self.redis.items()) + self.assertIs(type(copy), dict) + self.assertEqual(copy, local_copy) + + def test_clear(self): + """Test that the .clear method removes the entire hash.""" + self.redis.clear() + self.redis['teddy'] = "with me" + self.redis['in my dreams'] = "you have a weird hat" + self.assertEqual(len(self.redis), 2) + + self.redis.clear() + self.assertEqual(len(self.redis), 0) + + def test_pop(self): + """Test that we can .pop an item from the RedisDict.""" + self.redis.clear() + self.redis['john'] = 'was afraid' + + self.assertEqual(self.redis.pop('john'), 'was afraid') + self.assertEqual(self.redis.pop('pete', 'breakneck'), 'breakneck') + self.assertEqual(len(self.redis), 0) + + def test_popitem(self): + """Test that we can .popitem an item from the RedisDict.""" + self.redis.clear() + self.redis['john'] = 'the revalator' + self.redis['teddy'] = 'big bear' + + self.assertEqual(len(self.redis), 2) + self.assertEqual(self.redis.popitem(), 'big bear') + self.assertEqual(len(self.redis), 1) + + def test_setdefault(self): + """Test that we can .setdefault an item from the RedisDict.""" + self.redis.clear() + self.redis.setdefault('john', 'is yellow and weak') + self.assertEqual(self.redis['john'], 'is yellow and weak') + + with self.assertRaises(TypeError): + self.redis.setdefault('geisha', object) + + def test_update(self): + """Test that we can .update the RedisDict with multiple items.""" + self.redis.clear() + self.redis["reckfried"] = "lona" + self.redis["bel air"] = "prince" + self.redis.update({ + "reckfried": "jona", + "mega": "hungry, though", + }) + + result = { + "reckfried": "jona", + "bel air": "prince", + "mega": "hungry, though", + } + self.assertEqual(self.redis.copy(), result) + + def test_equals(self): + """Test that RedisDicts can be compared with == and !=.""" + new_redis_dict = RedisDict("firedog_the_sequel") + new_new_redis_dict = new_redis_dict + + self.assertEqual(new_redis_dict, new_new_redis_dict) + self.assertNotEqual(new_redis_dict, self.redis) -- cgit v1.2.3 From 258ad9d9c5601e01d135e706c256c0cd7f7fdbe0 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 03:29:53 +0200 Subject: Make redis host and port configurable. --- bot/constants.py | 3 +++ bot/utils/redis_dict.py | 7 ++++++- config-default.yml | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/bot/constants.py b/bot/constants.py index fd280e9de..01e8ac3a3 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -199,6 +199,9 @@ class Bot(metaclass=YAMLGetter): prefix: str token: str sentry_dsn: str + redis_host: str + redis_port: int + class Filter(metaclass=YAMLGetter): section = "filter" diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py index c89765a24..dfb1c7252 100644 --- a/bot/utils/redis_dict.py +++ b/bot/utils/redis_dict.py @@ -7,6 +7,8 @@ from typing import Dict, List, Optional, Tuple, Union import redis as redis_py +from bot import constants + ValidRedisKey = Union[str, int, float] JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] @@ -24,7 +26,10 @@ class RedisDict(MutableMapping): """ _namespaces = [] - _redis = redis_py.Redis(host="redis") # Can be overridden for testing + _redis = redis_py.Redis( + host=constants.Bot.redis_host, + port=constants.Bot.redis_port, + ) # Can be overridden for testing def __init__(self, namespace: Optional[str] = None) -> None: """Initialize the RedisDict with the right namespace.""" diff --git a/config-default.yml b/config-default.yml index 83ea59016..722afa41b 100644 --- a/config-default.yml +++ b/config-default.yml @@ -2,6 +2,8 @@ bot: prefix: "!" token: !ENV "BOT_TOKEN" sentry_dsn: !ENV "BOT_SENTRY_DSN" + redis_host: "redis" + redis_port: 6379 stats: statsd_host: "graphite" -- cgit v1.2.3 From 8456023252c2d4c91c6566ee0a3f83e9033d45d9 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 03:39:52 +0200 Subject: namespace "general" -> "global" --- bot/utils/redis_dict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py index dfb1c7252..47905314a 100644 --- a/bot/utils/redis_dict.py +++ b/bot/utils/redis_dict.py @@ -39,7 +39,7 @@ class RedisDict(MutableMapping): if self._has_custom_namespace: self._set_namespace(namespace) else: - self.namespace = "general" + self.namespace = "global" def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" -- cgit v1.2.3 From e993566fe5d816fadee64aaca454ce6ba463bca2 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 11:35:34 +0200 Subject: Fix linting errors introduced by flake8 3.8 Turns out that bumping the flake8 version up to 3.8 introduces a long list of new linting errors. Since this PR is the one that bumps the version, I suppose we will also fix all the linting errors in this branch. --- bot/cogs/antispam.py | 4 ++-- bot/cogs/defcon.py | 2 +- bot/cogs/duck_pond.py | 2 +- bot/cogs/error_handler.py | 4 ++-- bot/cogs/help_channels.py | 2 +- bot/cogs/moderation/management.py | 10 +++++----- bot/cogs/moderation/scheduler.py | 10 +++++----- bot/cogs/moderation/silence.py | 2 +- bot/cogs/stats.py | 4 ++-- bot/cogs/utils.py | 4 ++-- bot/cogs/watchchannels/talentpool.py | 2 +- bot/cogs/watchchannels/watchchannel.py | 14 +++++++------- bot/decorators.py | 2 +- bot/pagination.py | 4 ++-- bot/utils/messages.py | 2 +- 15 files changed, 34 insertions(+), 34 deletions(-) diff --git a/bot/cogs/antispam.py b/bot/cogs/antispam.py index d63acbc4a..0bcca578d 100644 --- a/bot/cogs/antispam.py +++ b/bot/cogs/antispam.py @@ -94,7 +94,7 @@ class DeletionContext: await modlog.send_log_message( icon_url=Icons.filtering, colour=Colour(Colours.soft_red), - title=f"Spam detected!", + title="Spam detected!", text=mod_alert_message, thumbnail=last_message.author.avatar_url_as(static_format="png"), channel_id=Channels.mod_alerts, @@ -130,7 +130,7 @@ class AntiSpam(Cog): body += "\n\n**The cog has been unloaded.**" await self.mod_log.send_log_message( - title=f"Error: AntiSpam configuration validation failed!", + title="Error: AntiSpam configuration validation failed!", text=body, ping_everyone=True, icon_url=Icons.token_removed, diff --git a/bot/cogs/defcon.py b/bot/cogs/defcon.py index 56fca002a..f4cb0aa58 100644 --- a/bot/cogs/defcon.py +++ b/bot/cogs/defcon.py @@ -81,7 +81,7 @@ class Defcon(Cog): else: self.enabled = False self.days = timedelta(days=0) - log.info(f"DEFCON disabled") + log.info("DEFCON disabled") await self.update_channel_topic() diff --git a/bot/cogs/duck_pond.py b/bot/cogs/duck_pond.py index 1f84a0609..37d1786a2 100644 --- a/bot/cogs/duck_pond.py +++ b/bot/cogs/duck_pond.py @@ -117,7 +117,7 @@ class DuckPond(Cog): avatar_url=message.author.avatar_url ) except discord.HTTPException: - log.exception(f"Failed to send an attachment to the webhook") + log.exception("Failed to send an attachment to the webhook") await message.add_reaction("✅") diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index b2f4c59f6..16790c769 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -173,7 +173,7 @@ class ErrorHandler(Cog): await ctx.invoke(*help_command) self.bot.stats.incr("errors.missing_required_argument") elif isinstance(e, errors.TooManyArguments): - await ctx.send(f"Too many arguments provided.") + await ctx.send("Too many arguments provided.") await ctx.invoke(*help_command) self.bot.stats.incr("errors.too_many_arguments") elif isinstance(e, errors.BadArgument): @@ -213,7 +213,7 @@ class ErrorHandler(Cog): if isinstance(e, bot_missing_errors): ctx.bot.stats.incr("errors.bot_permission_error") await ctx.send( - f"Sorry, it looks like I don't have the permissions or roles I need to do that." + "Sorry, it looks like I don't have the permissions or roles I need to do that." ) elif isinstance(e, (InWhitelistCheckFailure, errors.NoPrivateMessage)): ctx.bot.stats.incr("errors.wrong_channel_or_dm_error") diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 1bd1f9d68..a20fe2b05 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -391,7 +391,7 @@ class HelpChannels(Scheduler, commands.Cog): self.in_use_category = await self.try_get_channel(constants.Categories.help_in_use) self.dormant_category = await self.try_get_channel(constants.Categories.help_dormant) except discord.HTTPException: - log.exception(f"Failed to get a category; cog will be removed") + log.exception("Failed to get a category; cog will be removed") self.bot.remove_cog(self.qualified_name) async def init_cog(self) -> None: diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index 250a24247..82ec6b0d9 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -49,8 +49,8 @@ class ModManagement(commands.Cog): async def infraction_edit( self, ctx: Context, - infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], - duration: t.Union[Expiry, allowed_strings("p", "permanent"), None], + infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], # noqa: F821 + duration: t.Union[Expiry, allowed_strings("p", "permanent"), None], # noqa: F821 *, reason: str = None ) -> None: @@ -83,14 +83,14 @@ class ModManagement(commands.Cog): "actor__id": ctx.author.id, "ordering": "-inserted_at" } - infractions = await self.bot.api_client.get(f"bot/infractions", params=params) + infractions = await self.bot.api_client.get("bot/infractions", params=params) if infractions: old_infraction = infractions[0] infraction_id = old_infraction["id"] else: await ctx.send( - f":x: Couldn't find most recent infraction; you have never given an infraction." + ":x: Couldn't find most recent infraction; you have never given an infraction." ) return else: @@ -224,7 +224,7 @@ class ModManagement(commands.Cog): ) -> None: """Send a paginated embed of infractions for the specified user.""" if not infractions: - await ctx.send(f":warning: No infractions could be found for that query.") + await ctx.send(":warning: No infractions could be found for that query.") return lines = tuple( diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index dc42bee2e..012432e60 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -91,7 +91,7 @@ class InfractionScheduler(Scheduler): log.trace(f"Applying {infr_type} infraction #{id_} to {user}.") # Default values for the confirmation message and mod log. - confirm_msg = f":ok_hand: applied" + confirm_msg = ":ok_hand: applied" # Specifying an expiry for a note or warning makes no sense. if infr_type in ("note", "warning"): @@ -154,7 +154,7 @@ class InfractionScheduler(Scheduler): self.schedule_task(infraction["id"], infraction) except discord.HTTPException as e: # Accordingly display that applying the infraction failed. - confirm_msg = f":x: failed to apply" + confirm_msg = ":x: failed to apply" expiry_msg = "" log_content = ctx.author.mention log_title = "failed to apply" @@ -281,7 +281,7 @@ class InfractionScheduler(Scheduler): log.warning(f"Failed to pardon {infr_type} infraction #{id_} for {user}.") else: - confirm_msg = f":ok_hand: pardoned" + confirm_msg = ":ok_hand: pardoned" log_title = "pardoned" log.info(f"Pardoned {infr_type} infraction #{id_} for {user}.") @@ -353,7 +353,7 @@ class InfractionScheduler(Scheduler): ) except discord.Forbidden: log.warning(f"Failed to deactivate infraction #{id_} ({type_}): bot lacks permissions.") - log_text["Failure"] = f"The bot lacks permissions to do this (role hierarchy?)" + log_text["Failure"] = "The bot lacks permissions to do this (role hierarchy?)" log_content = mod_role.mention except discord.HTTPException as e: log.exception(f"Failed to deactivate infraction #{id_} ({type_})") @@ -402,7 +402,7 @@ class InfractionScheduler(Scheduler): # Send a log message to the mod log. if send_log: - log_title = f"expiration failed" if "Failure" in log_text else "expired" + log_title = "expiration failed" if "Failure" in log_text else "expired" user = self.bot.get_user(user_id) avatar = user.avatar_url_as(static_format="png") if user else None diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index 1ef3967a9..25febfa51 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -91,7 +91,7 @@ class Silence(commands.Cog): await ctx.send(f"{Emojis.check_mark} silenced current channel for {duration} minute(s).") await asyncio.sleep(duration*60) - log.info(f"Unsilencing channel after set delay.") + log.info("Unsilencing channel after set delay.") await ctx.invoke(self.unsilence) @commands.command(aliases=("unhush",)) diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py index d253db913..e088c2b87 100644 --- a/bot/cogs/stats.py +++ b/bot/cogs/stats.py @@ -59,7 +59,7 @@ class Stats(Cog): if member.guild.id != Guild.id: return - self.bot.stats.gauge(f"guild.total_members", len(member.guild.members)) + self.bot.stats.gauge("guild.total_members", len(member.guild.members)) @Cog.listener() async def on_member_leave(self, member: Member) -> None: @@ -67,7 +67,7 @@ class Stats(Cog): if member.guild.id != Guild.id: return - self.bot.stats.gauge(f"guild.total_members", len(member.guild.members)) + self.bot.stats.gauge("guild.total_members", len(member.guild.members)) @Cog.listener() async def on_member_update(self, _before: Member, after: Member) -> None: diff --git a/bot/cogs/utils.py b/bot/cogs/utils.py index 89d556f58..f76daedac 100644 --- a/bot/cogs/utils.py +++ b/bot/cogs/utils.py @@ -253,8 +253,8 @@ class Utils(Cog): async def send_pep_zero(self, ctx: Context) -> None: """Send information about PEP 0.""" pep_embed = Embed( - title=f"**PEP 0 - Index of Python Enhancement Proposals (PEPs)**", - description=f"[Link](https://www.python.org/dev/peps/)" + title="**PEP 0 - Index of Python Enhancement Proposals (PEPs)**", + description="[Link](https://www.python.org/dev/peps/)" ) pep_embed.set_thumbnail(url=ICON_URL) pep_embed.add_field(name="Status", value="Active") diff --git a/bot/cogs/watchchannels/talentpool.py b/bot/cogs/watchchannels/talentpool.py index ad0c51fa6..68b220233 100644 --- a/bot/cogs/watchchannels/talentpool.py +++ b/bot/cogs/watchchannels/talentpool.py @@ -61,7 +61,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): return if isinstance(user, Member) and any(role.id in STAFF_ROLES for role in user.roles): - await ctx.send(f":x: Nominating staff members, eh? Here's a cookie :cookie:") + await ctx.send(":x: Nominating staff members, eh? Here's a cookie :cookie:") return if not await self.fetch_user_cache(): diff --git a/bot/cogs/watchchannels/watchchannel.py b/bot/cogs/watchchannels/watchchannel.py index 479820444..643cd46e4 100644 --- a/bot/cogs/watchchannels/watchchannel.py +++ b/bot/cogs/watchchannels/watchchannel.py @@ -82,7 +82,7 @@ class WatchChannel(metaclass=CogABCMeta): exc = self._consume_task.exception() if exc: self.log.exception( - f"The message queue consume task has failed with:", + "The message queue consume task has failed with:", exc_info=exc ) return False @@ -146,7 +146,7 @@ class WatchChannel(metaclass=CogABCMeta): try: data = await self.bot.api_client.get(self.api_endpoint, params=self.api_default_params) except ResponseCodeError as err: - self.log.exception(f"Failed to fetch the watched users from the API", exc_info=err) + self.log.exception("Failed to fetch the watched users from the API", exc_info=err) return False self.watched_users = defaultdict(dict) @@ -173,7 +173,7 @@ class WatchChannel(metaclass=CogABCMeta): self.log.trace(f"Sleeping {BigBrotherConfig.log_delay} seconds before consuming message queue") await asyncio.sleep(BigBrotherConfig.log_delay) - self.log.trace(f"Started consuming the message queue") + self.log.trace("Started consuming the message queue") # If the previous consumption Task failed, first consume the existing comsumption_queue if not self.consumption_queue: @@ -208,7 +208,7 @@ class WatchChannel(metaclass=CogABCMeta): await self.webhook.send(content=content, username=username, avatar_url=avatar_url, embed=embed) except discord.HTTPException as exc: self.log.exception( - f"Failed to send a message to the webhook", + "Failed to send a message to the webhook", exc_info=exc ) @@ -254,7 +254,7 @@ class WatchChannel(metaclass=CogABCMeta): ) except discord.HTTPException as exc: self.log.exception( - f"Failed to send an attachment to the webhook", + "Failed to send an attachment to the webhook", exc_info=exc ) @@ -326,13 +326,13 @@ class WatchChannel(metaclass=CogABCMeta): def cog_unload(self) -> None: """Takes care of unloading the cog and canceling the consumption task.""" - self.log.trace(f"Unloading the cog") + self.log.trace("Unloading the cog") if self._consume_task and not self._consume_task.done(): self._consume_task.cancel() try: self._consume_task.result() except asyncio.CancelledError as e: self.log.exception( - f"The consume task was canceled. Messages may be lost.", + "The consume task was canceled. Messages may be lost.", exc_info=e ) diff --git a/bot/decorators.py b/bot/decorators.py index 2ee5879f2..dc9c7d439 100644 --- a/bot/decorators.py +++ b/bot/decorators.py @@ -121,7 +121,7 @@ def locked() -> Callable: embed = Embed() embed.colour = Colour.red() - log.debug(f"User tried to invoke a locked command.") + log.debug("User tried to invoke a locked command.") embed.description = ( "You're already using this command. Please wait until it is done before you use it again." ) diff --git a/bot/pagination.py b/bot/pagination.py index 90c8f849c..09759d5be 100644 --- a/bot/pagination.py +++ b/bot/pagination.py @@ -147,7 +147,7 @@ class LinePaginator(Paginator): if not lines: if exception_on_empty_embed: - log.exception(f"Pagination asked for empty lines iterable") + log.exception("Pagination asked for empty lines iterable") raise EmptyPaginatorEmbed("No lines to paginate") log.debug("No lines to add to paginator, adding '(nothing to display)' message") @@ -357,7 +357,7 @@ class ImagePaginator(Paginator): if not pages: if exception_on_empty_embed: - log.exception(f"Pagination asked for empty image list") + log.exception("Pagination asked for empty image list") raise EmptyPaginatorEmbed("No images to paginate") log.debug("No images to add to paginator, adding '(no images to display)' message") diff --git a/bot/utils/messages.py b/bot/utils/messages.py index e969ee590..de8e186f3 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -100,7 +100,7 @@ async def send_attachments( log.warning(f"{failure_msg} with status {e.status}.") if link_large and large: - desc = f"\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large) + desc = "\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large) embed = Embed(description=desc) embed.set_footer(text="Attachments exceed upload size limit.") -- cgit v1.2.3 From 4e24e9c43a331ebc0f9b598f4de6c45e04216782 Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff Date: Sun, 17 May 2020 12:10:56 +0200 Subject: Use `send_help` to invoke command help After the refactoring of the help command, we need to use the built-in method of calling the help command: `Context.send_help`. As an argument, the qualified name (a string containing the full command path, including parents) of the command can be passed. Examples: - await ctx.send_help("reminders edit") This would send a help embed with information on `!reminders edit` to the Context. - await ctx.send_help(ctx.command.qualified_name) This would extract the qualified name of the command, which is the full command path, and send a help embed to Context. - await ctx.send_help() This will send the main "root" help embed to the Context. --- bot/cogs/bot.py | 2 +- bot/cogs/clean.py | 2 +- bot/cogs/defcon.py | 2 +- bot/cogs/error_handler.py | 33 +++++++++++++-------------------- bot/cogs/eval.py | 2 +- bot/cogs/extensions.py | 8 ++++---- bot/cogs/moderation/management.py | 2 +- bot/cogs/off_topic_names.py | 2 +- bot/cogs/reddit.py | 2 +- bot/cogs/reminders.py | 2 +- bot/cogs/site.py | 2 +- bot/cogs/snekbox.py | 2 +- bot/cogs/utils.py | 2 +- bot/cogs/watchchannels/bigbrother.py | 2 +- bot/cogs/watchchannels/talentpool.py | 4 ++-- tests/bot/cogs/test_snekbox.py | 3 +-- 16 files changed, 32 insertions(+), 40 deletions(-) diff --git a/bot/cogs/bot.py b/bot/cogs/bot.py index a6929b431..ae829d5c3 100644 --- a/bot/cogs/bot.py +++ b/bot/cogs/bot.py @@ -41,7 +41,7 @@ class BotCog(Cog, name="Bot"): @with_role(Roles.verified) async def botinfo_group(self, ctx: Context) -> None: """Bot informational commands.""" - await ctx.invoke(self.bot.get_command("help"), "bot") + await ctx.send_help("bot") @botinfo_group.command(name='about', aliases=('info',), hidden=True) @with_role(Roles.verified) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index 5cdf0b048..e9bdbf510 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -180,7 +180,7 @@ class Clean(Cog): @with_role(*MODERATION_ROLES) async def clean_group(self, ctx: Context) -> None: """Commands for cleaning messages in channels.""" - await ctx.invoke(self.bot.get_command("help"), "clean") + await ctx.send_help("clean") @clean_group.command(name="user", aliases=["users"]) @with_role(*MODERATION_ROLES) diff --git a/bot/cogs/defcon.py b/bot/cogs/defcon.py index 56fca002a..71847a441 100644 --- a/bot/cogs/defcon.py +++ b/bot/cogs/defcon.py @@ -122,7 +122,7 @@ class Defcon(Cog): @with_role(Roles.admins, Roles.owners) async def defcon_group(self, ctx: Context) -> None: """Check the DEFCON status or run a subcommand.""" - await ctx.invoke(self.bot.get_command("help"), "defcon") + await ctx.send_help("defcon") async def _defcon_action(self, ctx: Context, days: int, action: Action) -> None: """Providing a structured way to do an defcon action.""" diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index b2f4c59f6..2d6cd85e6 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -2,7 +2,7 @@ import contextlib import logging import typing as t -from discord.ext.commands import Cog, Command, Context, errors +from discord.ext.commands import Cog, Context, errors from sentry_sdk import push_scope from bot.api import ResponseCodeError @@ -79,19 +79,13 @@ class ErrorHandler(Cog): f"{e.__class__.__name__}: {e}" ) - async def get_help_command(self, command: t.Optional[Command]) -> t.Tuple: - """Return the help command invocation args to display help for `command`.""" - parent = None - if command is not None: - parent = command.parent - - # Retrieve the help command for the invoked command. - if parent and command: - return self.bot.get_command("help"), parent.name, command.name - elif command: - return self.bot.get_command("help"), command.name - else: - return self.bot.get_command("help") + @staticmethod + def get_help_command(ctx: Context) -> t.Coroutine: + """Return a prepared `help` command invocation coroutine.""" + if ctx.command: + return ctx.send_help(ctx.command.qualified_name) + + return ctx.send_help() async def try_silence(self, ctx: Context) -> bool: """ @@ -165,20 +159,19 @@ class ErrorHandler(Cog): * ArgumentParsingError: send an error message * Other: send an error message and the help command """ - # TODO: use ctx.send_help() once PR #519 is merged. - help_command = await self.get_help_command(ctx.command) + prepared_help_command = self.get_help_command(ctx) if isinstance(e, errors.MissingRequiredArgument): await ctx.send(f"Missing required argument `{e.param.name}`.") - await ctx.invoke(*help_command) + await prepared_help_command self.bot.stats.incr("errors.missing_required_argument") elif isinstance(e, errors.TooManyArguments): await ctx.send(f"Too many arguments provided.") - await ctx.invoke(*help_command) + await prepared_help_command self.bot.stats.incr("errors.too_many_arguments") elif isinstance(e, errors.BadArgument): await ctx.send(f"Bad argument: {e}\n") - await ctx.invoke(*help_command) + await prepared_help_command self.bot.stats.incr("errors.bad_argument") elif isinstance(e, errors.BadUnionArgument): await ctx.send(f"Bad argument: {e}\n```{e.errors[-1]}```") @@ -188,7 +181,7 @@ class ErrorHandler(Cog): self.bot.stats.incr("errors.argument_parsing_error") else: await ctx.send("Something about your input seems off. Check the arguments:") - await ctx.invoke(*help_command) + await prepared_help_command self.bot.stats.incr("errors.other_user_input_error") @staticmethod diff --git a/bot/cogs/eval.py b/bot/cogs/eval.py index 52136fc8d..2d52197e8 100644 --- a/bot/cogs/eval.py +++ b/bot/cogs/eval.py @@ -178,7 +178,7 @@ async def func(): # (None,) -> Any async def internal_group(self, ctx: Context) -> None: """Internal commands. Top secret!""" if not ctx.invoked_subcommand: - await ctx.invoke(self.bot.get_command("help"), "internal") + await ctx.send_help("internal") @internal_group.command(name='eval', aliases=('e',)) @with_role(Roles.admins, Roles.owners) diff --git a/bot/cogs/extensions.py b/bot/cogs/extensions.py index fb6cd9aa3..4493046e1 100644 --- a/bot/cogs/extensions.py +++ b/bot/cogs/extensions.py @@ -65,7 +65,7 @@ class Extensions(commands.Cog): @group(name="extensions", aliases=("ext", "exts", "c", "cogs"), invoke_without_command=True) async def extensions_group(self, ctx: Context) -> None: """Load, unload, reload, and list loaded extensions.""" - await ctx.invoke(self.bot.get_command("help"), "extensions") + await ctx.send_help("extensions") @extensions_group.command(name="load", aliases=("l",)) async def load_command(self, ctx: Context, *extensions: Extension) -> None: @@ -75,7 +75,7 @@ class Extensions(commands.Cog): If '\*' or '\*\*' is given as the name, all unloaded extensions will be loaded. """ # noqa: W605 if not extensions: - await ctx.invoke(self.bot.get_command("help"), "extensions load") + await ctx.send_help("extensions load") return if "*" in extensions or "**" in extensions: @@ -92,7 +92,7 @@ class Extensions(commands.Cog): If '\*' or '\*\*' is given as the name, all loaded extensions will be unloaded. """ # noqa: W605 if not extensions: - await ctx.invoke(self.bot.get_command("help"), "extensions unload") + await ctx.send_help("extensions unload") return blacklisted = "\n".join(UNLOAD_BLACKLIST & set(extensions)) @@ -118,7 +118,7 @@ class Extensions(commands.Cog): If '\*\*' is given as the name, all extensions, including unloaded ones, will be reloaded. """ # noqa: W605 if not extensions: - await ctx.invoke(self.bot.get_command("help"), "extensions reload") + await ctx.send_help("extensions reload") return if "**" in extensions: diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index 250a24247..5cd59cc07 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -43,7 +43,7 @@ class ModManagement(commands.Cog): @commands.group(name='infraction', aliases=('infr', 'infractions', 'inf'), invoke_without_command=True) async def infraction_group(self, ctx: Context) -> None: """Infraction manipulation commands.""" - await ctx.invoke(self.bot.get_command("help"), "infraction") + await ctx.send_help("infraction") @infraction_group.command(name='edit') async def infraction_edit( diff --git a/bot/cogs/off_topic_names.py b/bot/cogs/off_topic_names.py index 81511f99d..829772f65 100644 --- a/bot/cogs/off_topic_names.py +++ b/bot/cogs/off_topic_names.py @@ -97,7 +97,7 @@ class OffTopicNames(Cog): @with_role(*MODERATION_ROLES) async def otname_group(self, ctx: Context) -> None: """Add or list items from the off-topic channel name rotation.""" - await ctx.invoke(self.bot.get_command("help"), "otname") + await ctx.send_help("otname") @otname_group.command(name='add', aliases=('a',)) @with_role(*MODERATION_ROLES) diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index 5a7fa100f..07a2497be 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -245,7 +245,7 @@ class Reddit(Cog): @group(name="reddit", invoke_without_command=True) async def reddit_group(self, ctx: Context) -> None: """View the top posts from various subreddits.""" - await ctx.invoke(self.bot.get_command("help"), "reddit") + await ctx.send_help("reddit") @reddit_group.command(name="top") async def top_command(self, ctx: Context, subreddit: Subreddit = "r/Python") -> None: diff --git a/bot/cogs/reminders.py b/bot/cogs/reminders.py index 8b6457cbb..e2289c75d 100644 --- a/bot/cogs/reminders.py +++ b/bot/cogs/reminders.py @@ -281,7 +281,7 @@ class Reminders(Scheduler, Cog): @remind_group.group(name="edit", aliases=("change", "modify"), invoke_without_command=True) async def edit_reminder_group(self, ctx: Context) -> None: """Commands for modifying your current reminders.""" - await ctx.invoke(self.bot.get_command("help"), "reminders", "edit") + await ctx.send_help("reminders edit") @edit_reminder_group.command(name="duration", aliases=("time",)) async def edit_reminder_duration(self, ctx: Context, id_: int, expiration: Duration) -> None: diff --git a/bot/cogs/site.py b/bot/cogs/site.py index 853e29568..c17761a2b 100644 --- a/bot/cogs/site.py +++ b/bot/cogs/site.py @@ -21,7 +21,7 @@ class Site(Cog): @group(name="site", aliases=("s",), invoke_without_command=True) async def site_group(self, ctx: Context) -> None: """Commands for getting info about our website.""" - await ctx.invoke(self.bot.get_command("help"), "site") + await ctx.send_help("site") @site_group.command(name="home", aliases=("about",)) async def site_main(self, ctx: Context) -> None: diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 8d4688114..5de978758 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -289,7 +289,7 @@ class Snekbox(Cog): return if not code: # None or empty string - await ctx.invoke(self.bot.get_command("help"), "eval") + await ctx.send_help("eval") return log.info(f"Received code from {ctx.author} for evaluation:\n{code}") diff --git a/bot/cogs/utils.py b/bot/cogs/utils.py index 89d556f58..7350dc2ba 100644 --- a/bot/cogs/utils.py +++ b/bot/cogs/utils.py @@ -55,7 +55,7 @@ class Utils(Cog): if pep_number.isdigit(): pep_number = int(pep_number) else: - await ctx.invoke(self.bot.get_command("help"), "pep") + await ctx.send_help("pep") return # Handle PEP 0 directly because it's not in .rst or .txt so it can't be accessed like other PEPs. diff --git a/bot/cogs/watchchannels/bigbrother.py b/bot/cogs/watchchannels/bigbrother.py index 903c87f85..37f2d2b9d 100644 --- a/bot/cogs/watchchannels/bigbrother.py +++ b/bot/cogs/watchchannels/bigbrother.py @@ -30,7 +30,7 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): @with_role(*MODERATION_ROLES) async def bigbrother_group(self, ctx: Context) -> None: """Monitors users by relaying their messages to the Big Brother watch channel.""" - await ctx.invoke(self.bot.get_command("help"), "bigbrother") + await ctx.send_help("bigbrother") @bigbrother_group.command(name='watched', aliases=('all', 'list')) @with_role(*MODERATION_ROLES) diff --git a/bot/cogs/watchchannels/talentpool.py b/bot/cogs/watchchannels/talentpool.py index ad0c51fa6..b8473963d 100644 --- a/bot/cogs/watchchannels/talentpool.py +++ b/bot/cogs/watchchannels/talentpool.py @@ -34,7 +34,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): @with_role(*MODERATION_ROLES) async def nomination_group(self, ctx: Context) -> None: """Highlights the activity of helper nominees by relaying their messages to the talent pool channel.""" - await ctx.invoke(self.bot.get_command("help"), "talentpool") + await ctx.send_help("talentpool") @nomination_group.command(name='watched', aliases=('all', 'list')) @with_role(*MODERATION_ROLES) @@ -173,7 +173,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): @with_role(*MODERATION_ROLES) async def nomination_edit_group(self, ctx: Context) -> None: """Commands to edit nominations.""" - await ctx.invoke(self.bot.get_command("help"), "talentpool", "edit") + await ctx.send_help("talentpool edit") @nomination_edit_group.command(name='reason') @with_role(*MODERATION_ROLES) diff --git a/tests/bot/cogs/test_snekbox.py b/tests/bot/cogs/test_snekbox.py index 1dec0ccaf..190d41d66 100644 --- a/tests/bot/cogs/test_snekbox.py +++ b/tests/bot/cogs/test_snekbox.py @@ -209,9 +209,8 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): async def test_eval_command_call_help(self): """Test if the eval command call the help command if no code is provided.""" ctx = MockContext() - ctx.invoke = AsyncMock() await self.cog.eval_command.callback(self.cog, ctx=ctx, code='') - ctx.invoke.assert_called_once_with(self.bot.get_command("help"), "eval") + ctx.send_help.assert_called_once_with("eval") async def test_send_eval(self): """Test the send_eval function.""" -- cgit v1.2.3 From a983d49051dad22a041c0c2f886313e38e0eaf61 Mon Sep 17 00:00:00 2001 From: decorator-factory <42166884+decorator-factory@users.noreply.github.com> Date: Sun, 17 May 2020 13:28:41 +0300 Subject: Apply language improvements proposed from kwzrd Co-authored-by: kwzrd <44734341+kwzrd@users.noreply.github.com> --- bot/resources/tags/mutability.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/resources/tags/mutability.md b/bot/resources/tags/mutability.md index 48e5bac74..b37420fc7 100644 --- a/bot/resources/tags/mutability.md +++ b/bot/resources/tags/mutability.md @@ -11,14 +11,14 @@ print(string) # abcd `string` didn't change. Why is that so? -That's because strings in Python are _immutable_. You can't change them, you can only pass around existing strings or create new oness. +That's because strings in Python are _immutable_. You can't change them, you can only pass around existing strings or create new ones. ```python string = "abcd" string = string.upper() ``` -`string.upper()` creates a new string which is like the old one, but with allthe letters turned to upper case. +`string.upper()` creates and returns a new string which is like the old one, but with all the letters turned to upper case. `int`, `float`, `complex`, `tuple`, `frozenset` are other examples of immutable data types in Python. @@ -29,4 +29,4 @@ my_list.append(4) print(my_list) # [1, 2, 3, 4] ``` -`dict` and `set` are other examples of mutable data types in Python. +Other examples of mutable data types in Python are `dict` and `set`. -- cgit v1.2.3 From 1db4fe2cfe63a9199eb84f8c0ee6daff6efa41dc Mon Sep 17 00:00:00 2001 From: decorator-factory <42166884+decorator-factory@users.noreply.github.com> Date: Sun, 17 May 2020 13:30:23 +0300 Subject: Change standalone programs to interactive sessions --- bot/resources/tags/mutability.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/bot/resources/tags/mutability.md b/bot/resources/tags/mutability.md index b37420fc7..8b98da43a 100644 --- a/bot/resources/tags/mutability.md +++ b/bot/resources/tags/mutability.md @@ -4,9 +4,11 @@ Imagine that you want to make all letters in a string upper case. Conveniently, You might think that this would work: ```python -string = "abcd" -string.upper() -print(string) # abcd +>>> string = "abcd" +>>> string.upper() +'ABCD' +>>> string +'abcd' ``` `string` didn't change. Why is that so? @@ -14,8 +16,10 @@ print(string) # abcd That's because strings in Python are _immutable_. You can't change them, you can only pass around existing strings or create new ones. ```python -string = "abcd" -string = string.upper() +>>> string = "abcd" +>>> string = string.upper() +>>> string +'ABCD' ``` `string.upper()` creates and returns a new string which is like the old one, but with all the letters turned to upper case. @@ -24,9 +28,10 @@ string = string.upper() Mutable data types like `list`, on the other hand, can be changed in-place: ```python -my_list = [1, 2, 3] -my_list.append(4) -print(my_list) # [1, 2, 3, 4] +>>> my_list = [1, 2, 3] +>>> my_list.append(4) +>>> my_list +[1, 2, 3, 4] ``` Other examples of mutable data types in Python are `dict` and `set`. -- cgit v1.2.3 From abb9e57f7022d87f762660d10c83b7912483b873 Mon Sep 17 00:00:00 2001 From: decorator-factory <42166884+decorator-factory@users.noreply.github.com> Date: Sun, 17 May 2020 13:34:27 +0300 Subject: Add a note on user-defined classes --- bot/resources/tags/mutability.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/resources/tags/mutability.md b/bot/resources/tags/mutability.md index 8b98da43a..3447109ad 100644 --- a/bot/resources/tags/mutability.md +++ b/bot/resources/tags/mutability.md @@ -34,4 +34,4 @@ Mutable data types like `list`, on the other hand, can be changed in-place: [1, 2, 3, 4] ``` -Other examples of mutable data types in Python are `dict` and `set`. +Other examples of mutable data types in Python are `dict` and `set`. Instances of user-defined classes are also mutable. -- cgit v1.2.3 From 87cec1a863213aa23a07d29ca928766d382ee732 Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff Date: Sun, 17 May 2020 12:37:34 +0200 Subject: Use `Command`-object for `send_help` As @mathsman5133 pointed out, it's better to use the `Command`-instance we typically already have in the current context than to rely on parsing the qualified name again. The invocation is now done as: `await ctx.send_help(ctx.command)` --- bot/cogs/bot.py | 2 +- bot/cogs/clean.py | 2 +- bot/cogs/defcon.py | 2 +- bot/cogs/error_handler.py | 2 +- bot/cogs/eval.py | 2 +- bot/cogs/extensions.py | 8 ++++---- bot/cogs/moderation/management.py | 2 +- bot/cogs/off_topic_names.py | 2 +- bot/cogs/reddit.py | 2 +- bot/cogs/reminders.py | 2 +- bot/cogs/site.py | 2 +- bot/cogs/snekbox.py | 2 +- bot/cogs/utils.py | 2 +- bot/cogs/watchchannels/bigbrother.py | 2 +- bot/cogs/watchchannels/talentpool.py | 4 ++-- tests/bot/cogs/test_snekbox.py | 4 ++-- 16 files changed, 21 insertions(+), 21 deletions(-) diff --git a/bot/cogs/bot.py b/bot/cogs/bot.py index ae829d5c3..f6aea51c5 100644 --- a/bot/cogs/bot.py +++ b/bot/cogs/bot.py @@ -41,7 +41,7 @@ class BotCog(Cog, name="Bot"): @with_role(Roles.verified) async def botinfo_group(self, ctx: Context) -> None: """Bot informational commands.""" - await ctx.send_help("bot") + await ctx.send_help(ctx.command) @botinfo_group.command(name='about', aliases=('info',), hidden=True) @with_role(Roles.verified) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index e9bdbf510..b5d9132cb 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -180,7 +180,7 @@ class Clean(Cog): @with_role(*MODERATION_ROLES) async def clean_group(self, ctx: Context) -> None: """Commands for cleaning messages in channels.""" - await ctx.send_help("clean") + await ctx.send_help(ctx.command) @clean_group.command(name="user", aliases=["users"]) @with_role(*MODERATION_ROLES) diff --git a/bot/cogs/defcon.py b/bot/cogs/defcon.py index 71847a441..25b0a6ad5 100644 --- a/bot/cogs/defcon.py +++ b/bot/cogs/defcon.py @@ -122,7 +122,7 @@ class Defcon(Cog): @with_role(Roles.admins, Roles.owners) async def defcon_group(self, ctx: Context) -> None: """Check the DEFCON status or run a subcommand.""" - await ctx.send_help("defcon") + await ctx.send_help(ctx.command) async def _defcon_action(self, ctx: Context, days: int, action: Action) -> None: """Providing a structured way to do an defcon action.""" diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index 2d6cd85e6..23d1eed82 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -83,7 +83,7 @@ class ErrorHandler(Cog): def get_help_command(ctx: Context) -> t.Coroutine: """Return a prepared `help` command invocation coroutine.""" if ctx.command: - return ctx.send_help(ctx.command.qualified_name) + return ctx.send_help(ctx.command) return ctx.send_help() diff --git a/bot/cogs/eval.py b/bot/cogs/eval.py index 2d52197e8..eb8bfb1cf 100644 --- a/bot/cogs/eval.py +++ b/bot/cogs/eval.py @@ -178,7 +178,7 @@ async def func(): # (None,) -> Any async def internal_group(self, ctx: Context) -> None: """Internal commands. Top secret!""" if not ctx.invoked_subcommand: - await ctx.send_help("internal") + await ctx.send_help(ctx.command) @internal_group.command(name='eval', aliases=('e',)) @with_role(Roles.admins, Roles.owners) diff --git a/bot/cogs/extensions.py b/bot/cogs/extensions.py index 4493046e1..365f198ff 100644 --- a/bot/cogs/extensions.py +++ b/bot/cogs/extensions.py @@ -65,7 +65,7 @@ class Extensions(commands.Cog): @group(name="extensions", aliases=("ext", "exts", "c", "cogs"), invoke_without_command=True) async def extensions_group(self, ctx: Context) -> None: """Load, unload, reload, and list loaded extensions.""" - await ctx.send_help("extensions") + await ctx.send_help(ctx.command) @extensions_group.command(name="load", aliases=("l",)) async def load_command(self, ctx: Context, *extensions: Extension) -> None: @@ -75,7 +75,7 @@ class Extensions(commands.Cog): If '\*' or '\*\*' is given as the name, all unloaded extensions will be loaded. """ # noqa: W605 if not extensions: - await ctx.send_help("extensions load") + await ctx.send_help(ctx.command) return if "*" in extensions or "**" in extensions: @@ -92,7 +92,7 @@ class Extensions(commands.Cog): If '\*' or '\*\*' is given as the name, all loaded extensions will be unloaded. """ # noqa: W605 if not extensions: - await ctx.send_help("extensions unload") + await ctx.send_help(ctx.command) return blacklisted = "\n".join(UNLOAD_BLACKLIST & set(extensions)) @@ -118,7 +118,7 @@ class Extensions(commands.Cog): If '\*\*' is given as the name, all extensions, including unloaded ones, will be reloaded. """ # noqa: W605 if not extensions: - await ctx.send_help("extensions reload") + await ctx.send_help(ctx.command) return if "**" in extensions: diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index 5cd59cc07..edfdfd9e2 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -43,7 +43,7 @@ class ModManagement(commands.Cog): @commands.group(name='infraction', aliases=('infr', 'infractions', 'inf'), invoke_without_command=True) async def infraction_group(self, ctx: Context) -> None: """Infraction manipulation commands.""" - await ctx.send_help("infraction") + await ctx.send_help(ctx.command) @infraction_group.command(name='edit') async def infraction_edit( diff --git a/bot/cogs/off_topic_names.py b/bot/cogs/off_topic_names.py index 829772f65..201579a0b 100644 --- a/bot/cogs/off_topic_names.py +++ b/bot/cogs/off_topic_names.py @@ -97,7 +97,7 @@ class OffTopicNames(Cog): @with_role(*MODERATION_ROLES) async def otname_group(self, ctx: Context) -> None: """Add or list items from the off-topic channel name rotation.""" - await ctx.send_help("otname") + await ctx.send_help(ctx.command) @otname_group.command(name='add', aliases=('a',)) @with_role(*MODERATION_ROLES) diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index 07a2497be..5f2aec7a5 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -245,7 +245,7 @@ class Reddit(Cog): @group(name="reddit", invoke_without_command=True) async def reddit_group(self, ctx: Context) -> None: """View the top posts from various subreddits.""" - await ctx.send_help("reddit") + await ctx.send_help(ctx.command) @reddit_group.command(name="top") async def top_command(self, ctx: Context, subreddit: Subreddit = "r/Python") -> None: diff --git a/bot/cogs/reminders.py b/bot/cogs/reminders.py index e2289c75d..c242d2920 100644 --- a/bot/cogs/reminders.py +++ b/bot/cogs/reminders.py @@ -281,7 +281,7 @@ class Reminders(Scheduler, Cog): @remind_group.group(name="edit", aliases=("change", "modify"), invoke_without_command=True) async def edit_reminder_group(self, ctx: Context) -> None: """Commands for modifying your current reminders.""" - await ctx.send_help("reminders edit") + await ctx.send_help(ctx.command) @edit_reminder_group.command(name="duration", aliases=("time",)) async def edit_reminder_duration(self, ctx: Context, id_: int, expiration: Duration) -> None: diff --git a/bot/cogs/site.py b/bot/cogs/site.py index c17761a2b..7fc2a9c34 100644 --- a/bot/cogs/site.py +++ b/bot/cogs/site.py @@ -21,7 +21,7 @@ class Site(Cog): @group(name="site", aliases=("s",), invoke_without_command=True) async def site_group(self, ctx: Context) -> None: """Commands for getting info about our website.""" - await ctx.send_help("site") + await ctx.send_help(ctx.command) @site_group.command(name="home", aliases=("about",)) async def site_main(self, ctx: Context) -> None: diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 5de978758..c2782b9c8 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -289,7 +289,7 @@ class Snekbox(Cog): return if not code: # None or empty string - await ctx.send_help("eval") + await ctx.send_help(ctx.command) return log.info(f"Received code from {ctx.author} for evaluation:\n{code}") diff --git a/bot/cogs/utils.py b/bot/cogs/utils.py index 7350dc2ba..6b59d37c8 100644 --- a/bot/cogs/utils.py +++ b/bot/cogs/utils.py @@ -55,7 +55,7 @@ class Utils(Cog): if pep_number.isdigit(): pep_number = int(pep_number) else: - await ctx.send_help("pep") + await ctx.send_help(ctx.command) return # Handle PEP 0 directly because it's not in .rst or .txt so it can't be accessed like other PEPs. diff --git a/bot/cogs/watchchannels/bigbrother.py b/bot/cogs/watchchannels/bigbrother.py index 37f2d2b9d..e4fb173e0 100644 --- a/bot/cogs/watchchannels/bigbrother.py +++ b/bot/cogs/watchchannels/bigbrother.py @@ -30,7 +30,7 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): @with_role(*MODERATION_ROLES) async def bigbrother_group(self, ctx: Context) -> None: """Monitors users by relaying their messages to the Big Brother watch channel.""" - await ctx.send_help("bigbrother") + await ctx.send_help(ctx.command) @bigbrother_group.command(name='watched', aliases=('all', 'list')) @with_role(*MODERATION_ROLES) diff --git a/bot/cogs/watchchannels/talentpool.py b/bot/cogs/watchchannels/talentpool.py index b8473963d..9a85c68c2 100644 --- a/bot/cogs/watchchannels/talentpool.py +++ b/bot/cogs/watchchannels/talentpool.py @@ -34,7 +34,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): @with_role(*MODERATION_ROLES) async def nomination_group(self, ctx: Context) -> None: """Highlights the activity of helper nominees by relaying their messages to the talent pool channel.""" - await ctx.send_help("talentpool") + await ctx.send_help(ctx.command) @nomination_group.command(name='watched', aliases=('all', 'list')) @with_role(*MODERATION_ROLES) @@ -173,7 +173,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): @with_role(*MODERATION_ROLES) async def nomination_edit_group(self, ctx: Context) -> None: """Commands to edit nominations.""" - await ctx.send_help("talentpool edit") + await ctx.send_help(ctx.command) @nomination_edit_group.command(name='reason') @with_role(*MODERATION_ROLES) diff --git a/tests/bot/cogs/test_snekbox.py b/tests/bot/cogs/test_snekbox.py index 190d41d66..8490b02ca 100644 --- a/tests/bot/cogs/test_snekbox.py +++ b/tests/bot/cogs/test_snekbox.py @@ -208,9 +208,9 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): async def test_eval_command_call_help(self): """Test if the eval command call the help command if no code is provided.""" - ctx = MockContext() + ctx = MockContext(command="sentinel") await self.cog.eval_command.callback(self.cog, ctx=ctx, code='') - ctx.send_help.assert_called_once_with("eval") + ctx.send_help.assert_called_once_with("sentinel") async def test_send_eval(self): """Test the send_eval function.""" -- cgit v1.2.3 From 4f76259976491ff68247b0f66a6549b37bc090bd Mon Sep 17 00:00:00 2001 From: decorator-factory <42166884+decorator-factory@users.noreply.github.com> Date: Sun, 17 May 2020 14:16:45 +0300 Subject: Rename `string` to `greeting` --- bot/resources/tags/mutability.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/bot/resources/tags/mutability.md b/bot/resources/tags/mutability.md index 3447109ad..ef2f47403 100644 --- a/bot/resources/tags/mutability.md +++ b/bot/resources/tags/mutability.md @@ -4,11 +4,11 @@ Imagine that you want to make all letters in a string upper case. Conveniently, You might think that this would work: ```python ->>> string = "abcd" ->>> string.upper() -'ABCD' ->>> string -'abcd' +>>> greeting = "hello" +>>> greeting.upper() +'HELLO' +>>> greeting +'hello' ``` `string` didn't change. Why is that so? @@ -16,13 +16,13 @@ You might think that this would work: That's because strings in Python are _immutable_. You can't change them, you can only pass around existing strings or create new ones. ```python ->>> string = "abcd" ->>> string = string.upper() ->>> string -'ABCD' +>>> greeting = "hello" +>>> greeting = greeting.upper() +>>> greeting +'HELLO' ``` -`string.upper()` creates and returns a new string which is like the old one, but with all the letters turned to upper case. +`greeting.upper()` creates and returns a new string which is like the old one, but with all the letters turned to upper case. `int`, `float`, `complex`, `tuple`, `frozenset` are other examples of immutable data types in Python. -- cgit v1.2.3 From d390fb257011f9c5cdd7e6d35a1b194303aa9e5d Mon Sep 17 00:00:00 2001 From: decorator-factory <42166884+decorator-factory@users.noreply.github.com> Date: Sun, 17 May 2020 14:19:10 +0300 Subject: Fix incomplete variable renaming --- bot/resources/tags/mutability.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/resources/tags/mutability.md b/bot/resources/tags/mutability.md index ef2f47403..bde9b5e7e 100644 --- a/bot/resources/tags/mutability.md +++ b/bot/resources/tags/mutability.md @@ -11,7 +11,7 @@ You might think that this would work: 'hello' ``` -`string` didn't change. Why is that so? +`greeting` didn't change. Why is that so? That's because strings in Python are _immutable_. You can't change them, you can only pass around existing strings or create new ones. -- cgit v1.2.3 From adf50a6ddc6868dec108ad471c5f3f4033ccd69b Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 19:44:29 +0200 Subject: Changes discord-py to discord.py in Pipfile The `discord-py` package is no longer the official release, and so making this change silences some warnings about deprecation. --- Pipfile | 2 +- Pipfile.lock | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Pipfile b/Pipfile index 40ae52761..1d6cd7015 100644 --- a/Pipfile +++ b/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -discord-py = "~=1.3.2" +discord.py = "~=1.3.2" aiodns = "~=2.0" aiohttp = "~=3.5" sphinx = "~=2.2" diff --git a/Pipfile.lock b/Pipfile.lock index 414f4a053..25383b355 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8ec71e9c46d52bf3b8c72939519e993715c79b4bc9e6ad164c1cf88951dc48b4" + "sha256": "49c231092320b48c5a7618bf048a477f7e0ed33dcbfb71c6dc8f18ef819dd935" }, "pipfile-spec": 6, "requires": { @@ -166,11 +166,19 @@ "index": "pypi", "version": "==4.3.2" }, - "discord-py": { + "discord": { "hashes": [ - "sha256:406871b06d86c3dc49fba63238519f28628dac946fef8a0e22988ff58ec05580" + "sha256:9d4debb4a37845543bd4b92cb195bc53a302797333e768e70344222857ff1559", + "sha256:ff6653655e342e7721dfb3f10421345fd852c2a33f2cca912b1c39b3778a9429" ], "index": "pypi", + "version": "==1.0.1" + }, + "discord.py": { + "hashes": [ + "sha256:406871b06d86c3dc49fba63238519f28628dac946fef8a0e22988ff58ec05580", + "sha256:ad00e34c72d2faa8db2157b651d05f3c415d7d05078e7e41dc9e8dc240051beb" + ], "version": "==1.3.3" }, "docutils": { @@ -480,10 +488,10 @@ }, "soupsieve": { "hashes": [ - "sha256:e914534802d7ffd233242b785229d5ba0766a7f487385e3f714446a07bf540ae", - "sha256:fcd71e08c0aee99aca1b73f45478549ee7e7fc006d51b37bec9e9def7dc22b69" + "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", + "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232" ], - "version": "==2.0" + "version": "==2.0.1" }, "sphinx": { "hashes": [ -- cgit v1.2.3 From ecf7f24f05b9baa8705f3b2c2d044a42292fbc07 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 19:50:12 +0200 Subject: Add the REDIS_PASSWORD environment variable In production, we will need this password to make a connection to Redis. --- bot/constants.py | 11 +++++++++-- bot/utils/redis_dict.py | 5 +++-- config-default.yml | 7 +++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/bot/constants.py b/bot/constants.py index 01e8ac3a3..5d854dd7a 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -199,8 +199,15 @@ class Bot(metaclass=YAMLGetter): prefix: str token: str sentry_dsn: str - redis_host: str - redis_port: int + + +class Redis(metaclass=YAMLGetter): + section = "bot" + subsection = "redis" + + host: str + port: int + password: str class Filter(metaclass=YAMLGetter): diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py index 47905314a..4a5e34249 100644 --- a/bot/utils/redis_dict.py +++ b/bot/utils/redis_dict.py @@ -27,8 +27,9 @@ class RedisDict(MutableMapping): _namespaces = [] _redis = redis_py.Redis( - host=constants.Bot.redis_host, - port=constants.Bot.redis_port, + host=constants.Redis.host, + port=constants.Redis.port, + password=constants.Redis.password, ) # Can be overridden for testing def __init__(self, namespace: Optional[str] = None) -> None: diff --git a/config-default.yml b/config-default.yml index 722afa41b..3b58d9099 100644 --- a/config-default.yml +++ b/config-default.yml @@ -2,8 +2,11 @@ bot: prefix: "!" token: !ENV "BOT_TOKEN" sentry_dsn: !ENV "BOT_SENTRY_DSN" - redis_host: "redis" - redis_port: 6379 + + redis: + host: "redis" + port: 6379 + password: !ENV "REDIS_PASSWORD" stats: statsd_host: "graphite" -- cgit v1.2.3 From bd4b439bb7f4abb6b22ad6e0d33bbe9203317475 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Tue, 19 May 2020 01:30:33 +0100 Subject: [bug] Adjustment to changes in #941, return message sent by webhook so publish can take place --- bot/cogs/reddit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index a300cfe0f..3b77538a0 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -218,7 +218,7 @@ class Reddit(Cog): for subreddit in RedditConfig.subreddits: top_posts = await self.get_top_posts(subreddit=subreddit, time="day") - message = await self.webhook.send(username=f"{subreddit} Top Daily Posts", embed=top_posts) + message = await self.webhook.send(username=f"{subreddit} Top Daily Posts", embed=top_posts, wait=True) if message.channel.is_news(): await message.publish() -- cgit v1.2.3 From 1d36ec7001c35d96ff24f7493d26997c02891e93 Mon Sep 17 00:00:00 2001 From: "S. Co1" Date: Mon, 18 May 2020 21:17:53 -0400 Subject: Add Steam gift card scam to domain blacklist --- config-default.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config-default.yml b/config-default.yml index 83ea59016..c0b5b062f 100644 --- a/config-default.yml +++ b/config-default.yml @@ -316,6 +316,8 @@ filter: - poweredbydialup.online - poweredbysecurity.org - poweredbysecurity.online + - ssteam.site + - steamwalletgift.com word_watchlist: - goo+ks* -- cgit v1.2.3 From 75385da574dfdd622ab4b0c7d5771ebd3218542d Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 19 May 2020 07:58:40 +0300 Subject: Stats: Fixed stat names Co-authored-by: Joseph Banks --- bot/cogs/snekbox.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 1d240d8d8..efff6d815 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -207,9 +207,9 @@ class Snekbox(Cog): # Collect stats of eval fails + successes if icon == ":x:": - self.bot.stats.incr("evals.fail") + self.bot.stats.incr("snekbox.python.fail") elif icon in (":warning:", ":white_check_mark:"): - self.bot.stats.incr("evals.success") + self.bot.stats.incr("snekbox.python.success") response = await ctx.send(msg) self.bot.loop.create_task( @@ -299,16 +299,16 @@ class Snekbox(Cog): return if Roles.helpers in (role.id for role in ctx.author.roles): - self.bot.stats.incr("evals.roles.helpers") + self.bot.stats.incr("snekbox_usages.roles.helpers") else: - self.bot.stats.incr("evals.roles.developers") + self.bot.stats.incr("snekbox_usages.roles.developers") if ctx.channel.category_id == Categories.help_in_use: - self.bot.stats.incr("evals.channels.help") + self.bot.stats.incr("snekbox_usages.channels.help") elif ctx.channel.id == Channels.bot_commands: - self.bot.stats.incr("evals.channels.bot_commands") + self.bot.stats.incr("snekbox_usages.channels.bot_commands") else: - self.bot.stats.incr("evals.channels.topical") + self.bot.stats.incr("snekbox_usages.channels.topical") log.info(f"Received code from {ctx.author} for evaluation:\n{code}") -- cgit v1.2.3 From 3081aad0cb360436c451b9a4515d494711adaf81 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 19 May 2020 07:59:21 +0300 Subject: Stats: Fix docstrings Co-authored-by: Joseph Banks --- bot/cogs/stats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py index acee1f5a9..9baf222e2 100644 --- a/bot/cogs/stats.py +++ b/bot/cogs/stats.py @@ -106,14 +106,14 @@ class Stats(Cog): @loop(hours=1) async def update_guild_boost(self) -> None: - """Update every hour guild boosts amount + level.""" + """Post the server boost level and tier every hour.""" await self.bot.wait_until_guild_available() g = self.bot.get_guild(Guild.id) self.bot.stats.gauge("boost.amount", g.premium_subscription_count) self.bot.stats.gauge("boost.tier", g.premium_tier) def cog_unload(self) -> None: - """Stop guild boost stat collecting task on Cog unload.""" + """Stop the boost statistic task on unload of the Cog.""" self.update_guild_boost.stop() -- cgit v1.2.3 From 21916ad9c19a326eb8406ea751e5fd9f80e9d912 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:42:11 +0300 Subject: ModLog Tests: Fix truncation tests docstring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Leon Sandøy --- tests/bot/cogs/moderation/test_modlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/moderation/test_modlog.py b/tests/bot/cogs/moderation/test_modlog.py index d60836474..b5ad21a09 100644 --- a/tests/bot/cogs/moderation/test_modlog.py +++ b/tests/bot/cogs/moderation/test_modlog.py @@ -15,7 +15,7 @@ class ModLogTests(unittest.IsolatedAsyncioTestCase): self.channel = MockTextChannel() async def test_log_entry_description_truncation(self): - """Should truncate embed description for ModLog entry.""" + """Test that embed description for ModLog entry is truncated.""" self.bot.get_channel.return_value = self.channel await self.cog.send_log_message( icon_url="foo", -- cgit v1.2.3 From 1432e5ba36fc09c7233e5be4745f540c2c4af792 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:47:59 +0300 Subject: Infraction Tests: Small fixes - Remove unnecessary space from placeholder - Rename `has_active_infraction` to `get_active_infraction` --- tests/bot/cogs/moderation/test_infractions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 51a8cc645..2b1ff5728 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -17,11 +17,11 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): self.guild = MockGuild(id=4567) self.ctx = MockContext(bot=self.bot, author=self.user, guild=self.guild) - @patch("bot.cogs.moderation.utils.has_active_infraction") + @patch("bot.cogs.moderation.utils.get_active_infraction") @patch("bot.cogs.moderation.utils.post_infraction") - async def test_apply_ban_reason_truncation(self, post_infraction_mock, has_active_mock): + async def test_apply_ban_reason_truncation(self, post_infraction_mock, get_active_mock): """Should truncate reason for `ctx.guild.ban`.""" - has_active_mock.return_value = False + get_active_mock.return_value = 'foo' post_infraction_mock.return_value = {"foo": "bar"} self.cog.apply_infraction = AsyncMock() @@ -32,7 +32,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): ban = self.cog.apply_infraction.call_args[0][3] self.assertEqual( ban.cr_frame.f_locals["kwargs"]["reason"], - textwrap.shorten("foo bar" * 3000, 512, placeholder=" ...") + textwrap.shorten("foo bar" * 3000, 512, placeholder="...") ) # Await ban to avoid warning await ban -- cgit v1.2.3 From 787c106fb4a55eacc7af04afb9bcd8f206099e81 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:50:46 +0300 Subject: Infractions: Remove space from placeholder --- bot/cogs/moderation/infractions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 01266d346..5bfaad796 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -259,7 +259,7 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - truncated_reason = textwrap.shorten(reason, width=512, placeholder=" ...") + truncated_reason = textwrap.shorten(reason, width=512, placeholder="...") action = ctx.guild.ban(user, reason=truncated_reason, delete_message_days=0) await self.apply_infraction(ctx, infraction, user, action) -- cgit v1.2.3 From caac9b92d7c3f73ca5428597606105730e56cefc Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:53:58 +0300 Subject: ModLog: Fix embed description truncation --- bot/cogs/moderation/modlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index c6497b38d..9d28030d9 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -100,7 +100,7 @@ class ModLog(Cog, name="ModLog"): """Generate log embed and send to logging channel.""" # Truncate string directly here to avoid removing newlines embed = discord.Embed( - description=text[:2046] + "..." if len(text) > 2048 else text + description=text[:2045] + "..." if len(text) > 2048 else text ) if title and icon_url: -- cgit v1.2.3 From 5989bcfefa244eb05f37b76d1e1df2f45e5782fa Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:54:49 +0300 Subject: ModLog Tests: Fix embed description truncate test --- tests/bot/cogs/moderation/test_modlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/moderation/test_modlog.py b/tests/bot/cogs/moderation/test_modlog.py index b5ad21a09..f2809f40a 100644 --- a/tests/bot/cogs/moderation/test_modlog.py +++ b/tests/bot/cogs/moderation/test_modlog.py @@ -25,5 +25,5 @@ class ModLogTests(unittest.IsolatedAsyncioTestCase): ) embed = self.channel.send.call_args[1]["embed"] self.assertEqual( - embed.description, ("foo bar" * 3000)[:2046] + "..." + embed.description, ("foo bar" * 3000)[:2045] + "..." ) -- cgit v1.2.3 From 874cb001df91ea8223385dd2b32ab4e3c280e183 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:57:07 +0300 Subject: Infr. Tests: Add more content to await comment --- tests/bot/cogs/moderation/test_infractions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 2b1ff5728..f8f340c2e 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -34,7 +34,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): ban.cr_frame.f_locals["kwargs"]["reason"], textwrap.shorten("foo bar" * 3000, 512, placeholder="...") ) - # Await ban to avoid warning + # Await ban to avoid not awaited coroutine warning await ban @patch("bot.cogs.moderation.utils.post_infraction") @@ -51,5 +51,5 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): kick.cr_frame.f_locals["kwargs"]["reason"], textwrap.shorten("foo bar" * 3000, 512, placeholder="...") ) - # Await kick to avoid warning + # Await kick to avoid not awaited coroutine warning await kick -- cgit v1.2.3 From e9bd09d90c5acf61caa955533f406851e1a65aec Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:59:11 +0300 Subject: Infr. Tests: Replace `str` with `dict` To allow `.get`, I had to replace `str` return value with `dict` --- tests/bot/cogs/moderation/test_infractions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index f8f340c2e..139209749 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -21,7 +21,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): @patch("bot.cogs.moderation.utils.post_infraction") async def test_apply_ban_reason_truncation(self, post_infraction_mock, get_active_mock): """Should truncate reason for `ctx.guild.ban`.""" - get_active_mock.return_value = 'foo' + get_active_mock.return_value = {"foo": "bar"} post_infraction_mock.return_value = {"foo": "bar"} self.cog.apply_infraction = AsyncMock() -- cgit v1.2.3 From d9730e41b3144862fdd9c221d160a40144a7c881 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 11:02:37 +0300 Subject: Infr. Test: Replace `get_active_mock` return value Replace `{"foo": "bar"}` with `{"id": 1}` --- tests/bot/cogs/moderation/test_infractions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 139209749..925439bf3 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -21,7 +21,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): @patch("bot.cogs.moderation.utils.post_infraction") async def test_apply_ban_reason_truncation(self, post_infraction_mock, get_active_mock): """Should truncate reason for `ctx.guild.ban`.""" - get_active_mock.return_value = {"foo": "bar"} + get_active_mock.return_value = {"id": 1} post_infraction_mock.return_value = {"foo": "bar"} self.cog.apply_infraction = AsyncMock() -- cgit v1.2.3 From a1b6d147befd4043acdddc00667d3bda94cc76ad Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 11:15:09 +0300 Subject: Infr Tests: Make `get_active_infraction` return `None` --- tests/bot/cogs/moderation/test_infractions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 925439bf3..5548d9f68 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -21,7 +21,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): @patch("bot.cogs.moderation.utils.post_infraction") async def test_apply_ban_reason_truncation(self, post_infraction_mock, get_active_mock): """Should truncate reason for `ctx.guild.ban`.""" - get_active_mock.return_value = {"id": 1} + get_active_mock.return_value = None post_infraction_mock.return_value = {"foo": "bar"} self.cog.apply_infraction = AsyncMock() -- cgit v1.2.3 From 6c5979b88961cb1df3d669e07ae108d12e698119 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 21 May 2020 11:47:39 +0300 Subject: Config: Added new `HelpChannels` config `deleted_idle_minutes` This show how much minutes should this wait before making channel dormant when no messages in channel (original message deleted). --- bot/constants.py | 1 + config-default.yml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/bot/constants.py b/bot/constants.py index fd280e9de..3003c9d36 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -541,6 +541,7 @@ class HelpChannels(metaclass=YAMLGetter): claim_minutes: int cmd_whitelist: List[int] idle_minutes: int + deleted_idle_minutes: int max_available: int max_total_channels: int name_prefix: str diff --git a/config-default.yml b/config-default.yml index 83ea59016..2e8a777ba 100644 --- a/config-default.yml +++ b/config-default.yml @@ -529,6 +529,10 @@ help_channels: # Allowed duration of inactivity before making a channel dormant idle_minutes: 30 + # Allowed duration of inactivity when question message deleted + # and no one other sent before message making channel dormant. + deleted_idle_minutes: 5 + # Maximum number of channels to put in the available category max_available: 2 -- cgit v1.2.3 From 0c84302f7e3475c13924fda33c52e98566114082 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 21 May 2020 12:27:25 +0300 Subject: Help: Implemented faster close when claimant delete msg no more messages - Created function `is_empty` that check is there any message in channel after bot own available message. - `on_message_delete` that reschedule task when message is on correct channel and is empty. - In `move_idle_channel` function, implemented choosing right cooldown, based on is channel empty or not. --- bot/cogs/help_channels.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 1bd1f9d68..4415ce550 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -461,7 +461,11 @@ class HelpChannels(Scheduler, commands.Cog): """ log.trace(f"Handling in-use channel #{channel} ({channel.id}).") - idle_seconds = constants.HelpChannels.idle_minutes * 60 + if not self.is_empty(channel): + idle_seconds = constants.HelpChannels.idle_minutes * 60 + else: + idle_seconds = constants.HelpChannels.deleted_idle_minutes * 60 + time_elapsed = await self.get_idle_time(channel) if time_elapsed is None or time_elapsed >= idle_seconds: @@ -713,6 +717,32 @@ class HelpChannels(Scheduler, commands.Cog): # be put in the queue. await self.move_to_available() + @commands.Cog.listener() + async def on_message_delete(self, msg: discord.Message) -> None: + """Reschedule dormant when help channel is empty.""" + if not self.is_in_category(msg.channel, constants.Categories.help_in_use) or not self.is_empty(msg.channel): + return + + log.info(f"Claimant of #{msg.channel} ({msg.author}) deleted message, channel is empty now. Rescheduling task.") + + # Cancel existing dormant task before scheduling new. + self.cancel_task(msg.channel.id) + + task = TaskData(constants.HelpChannels.deleted_idle_minutes * 60, self.move_idle_channel(msg.channel)) + self.schedule_task(msg.channel.id, task) + + async def is_empty(self, channel: discord.TextChannel) -> bool: + """Check is last message bot sent available message.""" + msg = await self.get_last_message(channel) + if not msg or not msg.author.bot or not msg.embeds: + return False + + embed = msg.embeds[0] + if embed.description == AVAILABLE_MSG: + return True + else: + return False + async def reset_send_permissions(self) -> None: """Reset send permissions in the Available category for claimants.""" log.trace("Resetting send permissions in the Available category.") -- cgit v1.2.3 From cc3591df0f14041be683bb6716d1e427c52aa2d7 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Fri, 22 May 2020 02:34:04 +0200 Subject: Add the REDIS_PASSWORD environment variable In production, we will need this password to make a connection to Redis. --- bot/utils/redis_cache.py | 115 +++++++++++++++++++++++++++++++++++++++ bot/utils/redis_dict.py | 137 ----------------------------------------------- 2 files changed, 115 insertions(+), 137 deletions(-) create mode 100644 bot/utils/redis_cache.py delete mode 100644 bot/utils/redis_dict.py diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py new file mode 100644 index 000000000..d0a7eba4a --- /dev/null +++ b/bot/utils/redis_cache.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +from enum import Enum +from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Union + +from bot.bot import Bot + +ValidRedisKey = Union[str, int, float] +JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] + + +class RedisCache: + """ + A simplified interface for a Redis connection. + + This class must be created as a class attribute in a class. This is because it + uses __set_name__ to create a namespace like MyCog.my_class_attribute which is + used as a hash name when we store stuff in Redis, to prevent collisions. + + The class this object is instantiated in must also contains an attribute with an + instance of Bot. This is because Bot contains our redis_pool, which is how this + class communicates with the Redis server. + + We implement several convenient methods that are fairly similar to have a dict + behaves, and should be familiar to Python users. The biggest difference is that + all the public methods in this class are coroutines. + """ + + _namespaces = [] + + def __init__(self) -> None: + """Raise a NotImplementedError if `__set_name__` hasn't been run.""" + if not self._namespace: + raise NotImplementedError("RedisCache must be a class attribute.") + + def _set_namespace(self, namespace: str) -> None: + """Try to set the namespace, but do not permit collisions.""" + while namespace in self._namespaces: + namespace += "_" + + self._namespaces.append(namespace) + self._namespace = namespace + + def __set_name__(self, owner: object, attribute_name: str) -> None: + """ + Set the namespace to Class.attribute_name. + + Called automatically when this class is constructed inside a class as an attribute. + """ + if not self._has_custom_namespace: + self._set_namespace(f"{owner.__name__}.{attribute_name}") + + def __get__(self, instance: RedisCache, owner: Any) -> RedisCache: + """Fetch the Bot instance, we need it for the redis pool.""" + if self.bot: + return self + + if instance is None: + raise NotImplementedError("You must create an instance of RedisCache to use it.") + + for attribute in vars(instance).values(): + if isinstance(attribute, Bot): + self.bot = attribute + self._redis = self.bot.redis_pool + return self + else: + raise RuntimeError("Cannot initialize a RedisCache without a `Bot` instance.") + + def __repr__(self) -> str: + """Return a beautiful representation of this object instance.""" + return f"RedisCache(namespace={self._namespace!r})" + + async def set(self, key: ValidRedisKey, value: JSONSerializableType) -> None: + """Store an item in the Redis cache.""" + # await self._redis.hset(self._namespace, key, value) + + async def get(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + """Get an item from the Redis cache.""" + # value = await self._redis.hget(self._namespace, key) + + async def delete(self, key: ValidRedisKey) -> None: + """Delete an item from the Redis cache.""" + # await self._redis.hdel(self._namespace, key) + + async def contains(self, key: ValidRedisKey) -> bool: + """Check if a key exists in the Redis cache.""" + # return await self._redis.hexists(self._namespace, key) + + async def items(self) -> AsyncIterator: + """Iterate all the items in the Redis cache.""" + # data = await redis.hgetall(self.get_with_namespace(key)) + # for item in data: + # yield item + + async def length(self) -> int: + """Return the number of items in the Redis cache.""" + # return await self._redis.hlen(self._namespace) + + async def to_dict(self) -> Dict: + """Convert to dict and return.""" + # return dict(self.items()) + + async def clear(self) -> None: + """Deletes the entire hash from the Redis cache.""" + # await self._redis.delete(self._namespace) + + async def pop(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + """Get the item, remove it from the cache, and provide a default if not found.""" + value = await self.get(key, default) + await self.delete(key) + return value + + async def update(self) -> None: + """Update the Redis cache with multiple values.""" + # https://aioredis.readthedocs.io/en/v1.3.0/mixins.html#aioredis.commands.HashCommandsMixin.hmset_dict diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py deleted file mode 100644 index 4a5e34249..000000000 --- a/bot/utils/redis_dict.py +++ /dev/null @@ -1,137 +0,0 @@ -from __future__ import annotations - -import json -from collections.abc import MutableMapping -from enum import Enum -from typing import Dict, List, Optional, Tuple, Union - -import redis as redis_py - -from bot import constants - -ValidRedisKey = Union[str, int, float] -JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] - - -class RedisDict(MutableMapping): - """ - A dictionary interface for a Redis database. - - Objects created by this class should mostly behave like a normal dictionary, - but will store all the data in our Redis database for persistence between restarts. - - Redis is limited to simple types, so to allow you to store collections like lists - and dictionaries, we JSON deserialize every value. That means that it will not be possible - to store complex objects, only stuff like strings, numbers, and collections of strings and numbers. - """ - - _namespaces = [] - _redis = redis_py.Redis( - host=constants.Redis.host, - port=constants.Redis.port, - password=constants.Redis.password, - ) # Can be overridden for testing - - def __init__(self, namespace: Optional[str] = None) -> None: - """Initialize the RedisDict with the right namespace.""" - super().__init__() - self._has_custom_namespace = namespace is not None - - if self._has_custom_namespace: - self._set_namespace(namespace) - else: - self.namespace = "global" - - def _set_namespace(self, namespace: str) -> None: - """Try to set the namespace, but do not permit collisions.""" - while namespace in self._namespaces: - namespace = namespace + "_" - - self._namespaces.append(namespace) - self._namespace = namespace - - def __set_name__(self, owner: object, attribute_name: str) -> None: - """ - Set the namespace to Class.attribute_name. - - Called automatically when this class is constructed inside a class as an attribute, as long as - no custom namespace is provided to the constructor. - """ - if not self._has_custom_namespace: - self._set_namespace(f"{owner.__name__}.{attribute_name}") - - def __repr__(self) -> str: - """Return a beautiful representation of this object instance.""" - return f"RedisDict(namespace={self._namespace!r})" - - def __eq__(self, other: RedisDict) -> bool: - """Check equality between two RedisDicts.""" - return self.items() == other.items() and self._namespace == other._namespace - - def __ne__(self, other: RedisDict) -> bool: - """Check inequality between two RedisDicts.""" - return self.items() != other.items() or self._namespace != other._namespace - - def __setitem__(self, key: ValidRedisKey, value: JSONSerializableType): - """Store an item in the Redis cache.""" - # JSON serialize the value before storing it. - json_value = json.dumps(value) - self._redis.hset(self._namespace, key, json_value) - - def __getitem__(self, key: ValidRedisKey): - """Get an item from the Redis cache.""" - value = self._redis.hget(self._namespace, key) - - if value: - return json.loads(value) - - def __delitem__(self, key: ValidRedisKey): - """Delete an item from the Redis cache.""" - self._redis.hdel(self._namespace, key) - - def __contains__(self, key: ValidRedisKey): - """Check if a key exists in the Redis cache.""" - return self._redis.hexists(self._namespace, key) - - def __iter__(self): - """Iterate all the items in the Redis cache.""" - keys = self._redis.hkeys(self._namespace) - return iter([key.decode('utf-8') for key in keys]) - - def __len__(self): - """Return the number of items in the Redis cache.""" - return self._redis.hlen(self._namespace) - - def copy(self) -> Dict: - """Convert to dict and return.""" - return dict(self.items()) - - def clear(self) -> None: - """Deletes the entire hash from the Redis cache.""" - self._redis.delete(self._namespace) - - def get(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: - """Get the item, but provide a default if not found.""" - if key in self: - return self[key] - else: - return default - - def pop(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: - """Get the item, remove it from the cache, and provide a default if not found.""" - value = self.get(key, default) - del self[key] - return value - - def popitem(self) -> JSONSerializableType: - """Get the last item added to the cache.""" - key = list(self.keys())[-1] - return self.pop(key) - - def setdefault(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: - """Try to get the item. If the item does not exist, set it to `default` and return that.""" - value = self.get(key) - - if value is None: - self[key] = default - return default -- cgit v1.2.3 From 23c2e7a42c13e03a1765e49f5dac3cfd4fed65b7 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Fri, 22 May 2020 02:34:51 +0200 Subject: Replace redis-py with aioredis. --- Pipfile | 2 +- Pipfile.lock | 162 +++++++++++++++++++++++++++++++++++++---------------------- 2 files changed, 104 insertions(+), 60 deletions(-) diff --git a/Pipfile b/Pipfile index 1d6cd7015..cd2f2ad7a 100644 --- a/Pipfile +++ b/Pipfile @@ -23,7 +23,7 @@ colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} statsd = "~=3.3" feedparser = "~=5.2" beautifulsoup4 = "~=4.9" -redis = "~=3.5" +aioredis = "~=1.3.1" [dev-packages] coverage = "~=5.0" diff --git a/Pipfile.lock b/Pipfile.lock index 25383b355..1941f6887 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "49c231092320b48c5a7618bf048a477f7e0ed33dcbfb71c6dc8f18ef819dd935" + "sha256": "c0b3e4d3e2c9ddb6ba28d2c09d521fe90ad4ea3df5c7ea7cd3a8b679fb3f85f9" }, "pipfile-spec": 6, "requires": { @@ -18,11 +18,11 @@ "default": { "aio-pika": { "hashes": [ - "sha256:9e4614636296e0040055bd6b304e97a38cc9796669ef391fc9b36649831d43ee", - "sha256:c9d242b3c7142d64b185feb6c5cce4154962610e89ec2e9b52bd69ef01f89b2f" + "sha256:c4cbbeb85b3c7bf81bc127371846cd949e6231717ce1e6ac7ee1dd5ede21f866", + "sha256:ec7fef24f588d90314873463ab4f2c3debce0bd8830e49e3786586be96bc2e8e" ], "index": "pypi", - "version": "==6.6.0" + "version": "==6.6.1" }, "aiodns": { "hashes": [ @@ -50,6 +50,14 @@ "index": "pypi", "version": "==3.6.2" }, + "aioredis": { + "hashes": [ + "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a", + "sha256:b61808d7e97b7cd5a92ed574937a079c9387fdadd22bfbfa7ad2fd319ecc26e3" + ], + "index": "pypi", + "version": "==1.3.1" + }, "aiormq": { "hashes": [ "sha256:41a9d4eb17db805f30ed172f3f609fe0c2b16657fb15b1b67df19d251dd93c0d", @@ -87,12 +95,12 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:594ca51a10d2b3443cbac41214e12dbb2a1cd57e1a7344659849e2e20ba6a8d8", - "sha256:a4bbe77fd30670455c5296242967a123ec28c37e9702a8a81bd2f20a4baf0368", - "sha256:d4e96ac9b0c3a6d3f0caae2e4124e6055c5dcafde8e2f831ff194c104f0775a0" + "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7", + "sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8", + "sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c" ], "index": "pypi", - "version": "==4.9.0" + "version": "==4.9.1" }, "certifi": { "hashes": [ @@ -205,6 +213,51 @@ "index": "pypi", "version": "==0.18.0" }, + "hiredis": { + "hashes": [ + "sha256:01b577f84c20ecc9c07fc4c184231b08e3c3942de096fa99978e053de231c423", + "sha256:01ff0900134166961c9e339df77c33b72f7edc5cb41739f0babcd9faa345926e", + "sha256:03ed34a13316d0c34213c4fd46e0fa3a5299073f4d4f08e93fed8c2108b399b3", + "sha256:040436e91df5143aff9e0debb49530d0b17a6bd52200ce568621c31ef581b10d", + "sha256:091eb38fbf968d1c5b703e412bbbd25f43a7967d8400842cee33a5a07b33c27b", + "sha256:102f9b9dc6ed57feb3a7c9bdf7e71cb7c278fe8df1edfcfe896bc3e0c2be9447", + "sha256:2b4b392c7e3082860c8371fab3ae762139090f9115819e12d9f56060f9ede05d", + "sha256:2c9cc0b986397b833073f466e6b9e9c70d1d4dc2c2c1b3e9cae3a23102ff296c", + "sha256:2fa65a9df683bca72073cd77709ddeb289ea2b114d3775d225fbbcc5faf808c5", + "sha256:38437a681f17c975fd22349e72c29bc643f8e7eb2d6dc5df419eac59afa4d7ce", + "sha256:3b3428fa3cf1ee178807b52c9bee8950ab94cd4eaa9bfae8c1bbae3c49501d34", + "sha256:3dd8c2fae7f5494978facb0e93297dd627b1a3f536f3b070cf0a7d9157a07dcb", + "sha256:4414a96c212e732723b5c3d7c04d386ebbb2ec359e1de646322cbc3f875cbd0d", + "sha256:48c627581ad4ef60adbac980981407939acf13a0e18f093502c7b542223c4f19", + "sha256:4a60e71625a2d78d8ab84dfb2fa2cfd9458c964b6e6c04fea76d9ade153fb371", + "sha256:585ace09f434e43d8a8dbeb366865b1a044d7c06319b3c7372a0a00e63b860f4", + "sha256:74b364b3f06c9cf0a53f7df611045bc9437ed972a283fa1f0b12537236d23ddc", + "sha256:75c65c3850e89e9daa68d1b9bedd5806f177d60aa5a7b0953b4829481cfc1f72", + "sha256:7f052de8bf744730a9120dbdc67bfeb7605a01f69fb8e7ba5c475af33c24e145", + "sha256:8113a7d5e87ecf57cd4ae263cc9e429adb9a3e59f5a7768da5d3312a8d0a051a", + "sha256:84857ce239eb8ed191ac78e77ff65d52902f00f30f4ee83bf80eb71da73b70e6", + "sha256:8644a48ddc4a40b3e3a6b9443f396c2ee353afb2d45656c4fc68d04a82e8e3f7", + "sha256:936aa565e673536e8a211e43ec43197406f24cd1f290138bd143765079c8ba00", + "sha256:9afeb88c67bbc663b9f27385c496da056d06ad87f55df6e393e1516cfecb0461", + "sha256:9d62cc7880110e4f83b0a51d218f465d3095e2751fbddd34e553dbd106a929ff", + "sha256:a1fadd062fc8d647ff39220c57ea2b48c99bb73f18223828ec97f88fc27e7898", + "sha256:a7754a783b1e5d6f627c19d099b178059c62f782ab62b4d8ba165b9fbc2ee34c", + "sha256:aa59dd63bb3f736de4fc2d080114429d5d369dfb3265f771778e8349d67a97a4", + "sha256:ae2ee0992f8de249715435942137843a93db204dd7db1e7cc9bdc5a8436443e8", + "sha256:b36842d7cf32929d568f37ec5b3173b72b2ec6572dec4d6be6ce774762215aee", + "sha256:bcbf9379c553b5facc6c04c1e5569b44b38ff16bcbf354676287698d61ee0c92", + "sha256:cbccbda6f1c62ab460449d9c85fdf24d0d32a6bf45176581151e53cc26a5d910", + "sha256:d0caf98dfb8af395d6732bd16561c0a2458851bea522e39f12f04802dbf6f502", + "sha256:d6456afeddba036def1a36d8a2758eca53202308d83db20ab5d0b66590919627", + "sha256:dbaef9a21a4f10bc281684ee4124f169e62bb533c2a92b55f8c06f64f9af7b8f", + "sha256:dce84916c09aaece006272b37234ae84a8ed13abb3a4d341a23933b8701abfb5", + "sha256:eb8c9c8b9869539d58d60ff4a28373a22514d40495911451343971cb4835b7a9", + "sha256:efc98b14ee3a8595e40b1425e8d42f5fd26f11a7b215a81ef9259068931754f4", + "sha256:fa2dc05b87d97acc1c6ae63f3e0f39eae5246565232484b08db6bf2dc1580678", + "sha256:fe7d6ce9f6a5fbe24f09d95ea93e9c7271abc4e1565da511e1449b107b4d7848" + ], + "version": "==1.0.1" + }, "humanfriendly": { "hashes": [ "sha256:bf52ec91244819c780341a3438d5d7b09f431d3f113a475147ac9b7b167a3d12", @@ -235,36 +288,36 @@ }, "lxml": { "hashes": [ - "sha256:06d4e0bbb1d62e38ae6118406d7cdb4693a3fa34ee3762238bcb96c9e36a93cd", - "sha256:0701f7965903a1c3f6f09328c1278ac0eee8f56f244e66af79cb224b7ef3801c", - "sha256:1f2c4ec372bf1c4a2c7e4bb20845e8bcf8050365189d86806bad1e3ae473d081", - "sha256:4235bc124fdcf611d02047d7034164897ade13046bda967768836629bc62784f", - "sha256:5828c7f3e615f3975d48f40d4fe66e8a7b25f16b5e5705ffe1d22e43fb1f6261", - "sha256:585c0869f75577ac7a8ff38d08f7aac9033da2c41c11352ebf86a04652758b7a", - "sha256:5d467ce9c5d35b3bcc7172c06320dddb275fea6ac2037f72f0a4d7472035cea9", - "sha256:63dbc21efd7e822c11d5ddbedbbb08cd11a41e0032e382a0fd59b0b08e405a3a", - "sha256:7bc1b221e7867f2e7ff1933165c0cec7153dce93d0cdba6554b42a8beb687bdb", - "sha256:8620ce80f50d023d414183bf90cc2576c2837b88e00bea3f33ad2630133bbb60", - "sha256:8a0ebda56ebca1a83eb2d1ac266649b80af8dd4b4a3502b2c1e09ac2f88fe128", - "sha256:90ed0e36455a81b25b7034038e40880189169c308a3df360861ad74da7b68c1a", - "sha256:95e67224815ef86924fbc2b71a9dbd1f7262384bca4bc4793645794ac4200717", - "sha256:afdb34b715daf814d1abea0317b6d672476b498472f1e5aacbadc34ebbc26e89", - "sha256:b4b2c63cc7963aedd08a5f5a454c9f67251b1ac9e22fd9d72836206c42dc2a72", - "sha256:d068f55bda3c2c3fcaec24bd083d9e2eede32c583faf084d6e4b9daaea77dde8", - "sha256:d5b3c4b7edd2e770375a01139be11307f04341ec709cf724e0f26ebb1eef12c3", - "sha256:deadf4df349d1dcd7b2853a2c8796593cc346600726eff680ed8ed11812382a7", - "sha256:df533af6f88080419c5a604d0d63b2c33b1c0c4409aba7d0cb6de305147ea8c8", - "sha256:e4aa948eb15018a657702fee0b9db47e908491c64d36b4a90f59a64741516e77", - "sha256:e5d842c73e4ef6ed8c1bd77806bf84a7cb535f9c0cf9b2c74d02ebda310070e1", - "sha256:ebec08091a22c2be870890913bdadd86fcd8e9f0f22bcb398abd3af914690c15", - "sha256:edc15fcfd77395e24543be48871c251f38132bb834d9fdfdad756adb6ea37679", - "sha256:f2b74784ed7e0bc2d02bd53e48ad6ba523c9b36c194260b7a5045071abbb1012", - "sha256:fa071559f14bd1e92077b1b5f6c22cf09756c6de7139370249eb372854ce51e6", - "sha256:fd52e796fee7171c4361d441796b64df1acfceb51f29e545e812f16d023c4bbc", - "sha256:fe976a0f1ef09b3638778024ab9fb8cde3118f203364212c198f71341c0715ca" - ], - "index": "pypi", - "version": "==4.5.0" + "sha256:06748c7192eab0f48e3d35a7adae609a329c6257495d5e53878003660dc0fec6", + "sha256:0790ddca3f825dd914978c94c2545dbea5f56f008b050e835403714babe62a5f", + "sha256:1aa7a6197c1cdd65d974f3e4953764eee3d9c7b67e3966616b41fab7f8f516b7", + "sha256:22c6d34fdb0e65d5f782a4d1a1edb52e0a8365858dafb1c08cb1d16546cf0786", + "sha256:2754d4406438c83144f9ffd3628bbe2dcc6d62b20dbc5c1ec4bc4385e5d44b42", + "sha256:27ee0faf8077c7c1a589573b1450743011117f1aa1a91d5ae776bbc5ca6070f2", + "sha256:2b02c106709466a93ed424454ce4c970791c486d5fcdf52b0d822a7e29789626", + "sha256:2d1ddce96cf15f1254a68dba6935e6e0f1fe39247de631c115e84dd404a6f031", + "sha256:4f282737d187ae723b2633856085c31ae5d4d432968b7f3f478a48a54835f5c4", + "sha256:51bb4edeb36d24ec97eb3e6a6007be128b720114f9a875d6b370317d62ac80b9", + "sha256:7eee37c1b9815e6505847aa5e68f192e8a1b730c5c7ead39ff317fde9ce29448", + "sha256:7fd88cb91a470b383aafad554c3fe1ccf6dfb2456ff0e84b95335d582a799804", + "sha256:9144ce36ca0824b29ebc2e02ca186e54040ebb224292072250467190fb613b96", + "sha256:925baf6ff1ef2c45169f548cc85204433e061360bfa7d01e1be7ae38bef73194", + "sha256:a636346c6c0e1092ffc202d97ec1843a75937d8c98aaf6771348ad6422e44bb0", + "sha256:a87dbee7ad9dce3aaefada2081843caf08a44a8f52e03e0a4cc5819f8398f2f4", + "sha256:a9e3b8011388e7e373565daa5e92f6c9cb844790dc18e43073212bb3e76f7007", + "sha256:afb53edf1046599991fb4a7d03e601ab5f5422a5435c47ee6ba91ec3b61416a6", + "sha256:b26719890c79a1dae7d53acac5f089d66fd8cc68a81f4e4bd355e45470dc25e1", + "sha256:b7462cdab6fffcda853338e1741ce99706cdf880d921b5a769202ea7b94e8528", + "sha256:b77975465234ff49fdad871c08aa747aae06f5e5be62866595057c43f8d2f62c", + "sha256:c47a8a5d00060122ca5908909478abce7bbf62d812e3fc35c6c802df8fb01fe7", + "sha256:c79e5debbe092e3c93ca4aee44c9a7631bdd407b2871cb541b979fd350bbbc29", + "sha256:d8d40e0121ca1606aa9e78c28a3a7d88a05c06b3ca61630242cded87d8ce55fa", + "sha256:ee2be8b8f72a2772e72ab926a3bccebf47bb727bda41ae070dc91d1fb759b726", + "sha256:f95d28193c3863132b1f55c1056036bf580b5a488d908f7d22a04ace8935a3a9", + "sha256:fadd2a63a2bfd7fb604508e553d1cf68eca250b2fbdbd81213b5f6f2fbf23529" + ], + "index": "pypi", + "version": "==4.5.1" }, "markdownify": { "hashes": [ @@ -349,10 +402,10 @@ }, "packaging": { "hashes": [ - "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3", - "sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752" + "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", + "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" ], - "version": "==20.3" + "version": "==20.4" }, "pamqp": { "hashes": [ @@ -448,14 +501,6 @@ "index": "pypi", "version": "==5.3.1" }, - "redis": { - "hashes": [ - "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", - "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" - ], - "index": "pypi", - "version": "==3.5.2" - }, "requests": { "hashes": [ "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", @@ -474,10 +519,10 @@ }, "six": { "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "version": "==1.14.0" + "version": "==1.15.0" }, "snowballstemmer": { "hashes": [ @@ -837,15 +882,14 @@ "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" ], - "index": "pypi", "version": "==3.5.2" }, "six": { "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "version": "==1.14.0" + "version": "==1.15.0" }, "snowballstemmer": { "hashes": [ @@ -878,10 +922,10 @@ }, "virtualenv": { "hashes": [ - "sha256:b4c14d4d73a0c23db267095383c4276ef60e161f94fde0427f2f21a0132dde74", - "sha256:fd0e54dec8ac96c1c7c87daba85f0a59a7c37fe38748e154306ca21c73244637" + "sha256:a116629d4e7f4d03433b8afa27f43deba09d48bc48f5ecefa4f015a178efb6cf", + "sha256:a730548b27366c5e6cbdf6f97406d861cccece2e22275e8e1a757aeff5e00c70" ], - "version": "==20.0.20" + "version": "==20.0.21" } } } -- cgit v1.2.3 From 3f596d5245403b75759a9c73029768d9e4510303 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Fri, 22 May 2020 02:38:55 +0200 Subject: Opens a Redis connection in the Bot class. This global connection is the one we will be using in RedisCache to power all our commands. This also ensures that connection is closed when the bot starts its shutdown process. --- bot/bot.py | 17 +++++++++++++++++ bot/utils/__init__.py | 4 ++-- bot/utils/redis_cache.py | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/bot/bot.py b/bot/bot.py index a85a22aa9..f55eec5bb 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -5,6 +5,7 @@ import warnings from typing import Optional import aiohttp +import aioredis import discord from discord.ext import commands from sentry_sdk import push_scope @@ -28,11 +29,13 @@ class Bot(commands.Bot): super().__init__(*args, **kwargs) self.http_session: Optional[aiohttp.ClientSession] = None + self.redis_session: Optional[aioredis.Redis] = None self.api_client = api.APIClient(loop=self.loop) self._connector = None self._resolver = None self._guild_available = asyncio.Event() + self._redis_ready = asyncio.Event() statsd_url = constants.Stats.statsd_host @@ -42,8 +45,18 @@ class Bot(commands.Bot): # will effectively disable stats. statsd_url = "127.0.0.1" + asyncio.create_task(self._create_redis_session()) + self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") + async def _create_redis_session(self) -> None: + """Create the Redis connection pool, and then open the redis event gate.""" + self.redis_session = await aioredis.create_redis_pool( + address=(constants.Redis.host, constants.Redis.port), + password=constants.Redis.password, + ) + self._redis_ready.set() + def add_cog(self, cog: commands.Cog) -> None: """Adds a "cog" to the bot and logs the operation.""" super().add_cog(cog) @@ -78,6 +91,10 @@ class Bot(commands.Bot): if self.stats._transport: self.stats._transport.close() + if self.redis_session: + self.redis_session.close() + await self.redis_session.wait_closed() + async def login(self, *args, **kwargs) -> None: """Re-create the connector and set up sessions before logging into Discord.""" self._recreate() diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 5ce383bf2..c5a12d5e3 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -2,9 +2,9 @@ from abc import ABCMeta from discord.ext.commands import CogMeta -from bot.utils.redis_dict import RedisDict +from bot.utils.redis_cache import RedisCache -__all__ = ['RedisDict', 'CogABCMeta'] +__all__ = ['RedisCache', 'CogABCMeta'] class CogABCMeta(CogMeta, ABCMeta): diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index d0a7eba4a..467f16767 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -61,7 +61,7 @@ class RedisCache: for attribute in vars(instance).values(): if isinstance(attribute, Bot): self.bot = attribute - self._redis = self.bot.redis_pool + self._redis = self.bot.redis_session return self else: raise RuntimeError("Cannot initialize a RedisCache without a `Bot` instance.") -- cgit v1.2.3 From a46eff8d976cf65155f27ed75f49bd3e58155c84 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 22 May 2020 08:12:08 +0300 Subject: Help: Fix docstrings of `is_empty` and `on_message_delete` Co-authored-by: Mark --- bot/cogs/help_channels.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 4415ce550..ed1f7c55e 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -719,7 +719,11 @@ class HelpChannels(Scheduler, commands.Cog): @commands.Cog.listener() async def on_message_delete(self, msg: discord.Message) -> None: - """Reschedule dormant when help channel is empty.""" + """ + Reschedule an in-use channel to become dormant sooner if the channel is empty. + + The new time for the dormant task is configured with `HelpChannels.deleted_idle_minutes`. + """ if not self.is_in_category(msg.channel, constants.Categories.help_in_use) or not self.is_empty(msg.channel): return @@ -732,7 +736,7 @@ class HelpChannels(Scheduler, commands.Cog): self.schedule_task(msg.channel.id, task) async def is_empty(self, channel: discord.TextChannel) -> bool: - """Check is last message bot sent available message.""" + """Return True if the most recent message in `channel` is the bot's `AVAILABLE_MSG`.""" msg = await self.get_last_message(channel) if not msg or not msg.author.bot or not msg.embeds: return False -- cgit v1.2.3 From 841ce9ba155d2aea3011500f5129d6b3dd309b99 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 22 May 2020 08:29:41 +0300 Subject: Help: Create `embed_description_match` - Created function `embed_description_match`. - Implemented this to `is_empty` - Implemented this to `is_dormant_message` --- bot/cogs/help_channels.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index ed1f7c55e..554fdc55e 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -440,11 +440,18 @@ class HelpChannels(Scheduler, commands.Cog): def is_dormant_message(self, message: t.Optional[discord.Message]) -> bool: """Return True if the contents of the `message` match `DORMANT_MSG`.""" - if not message or not message.embeds: + if not message: + return False + + return self.embed_description_match(message, DORMANT_MSG) + + def embed_description_match(self, message: discord.Message, text: str) -> bool: + """Return `True` if `message` embed description match with `text`.""" + if not message.embeds: return False embed = message.embeds[0] - return message.author == self.bot.user and embed.description.strip() == DORMANT_MSG.strip() + return message.author == self.bot.user and embed.description.strip() == text.strip() @staticmethod def is_in_category(channel: discord.TextChannel, category_id: int) -> bool: @@ -722,7 +729,7 @@ class HelpChannels(Scheduler, commands.Cog): """ Reschedule an in-use channel to become dormant sooner if the channel is empty. - The new time for the dormant task is configured with `HelpChannels.deleted_idle_minutes`. + The new time for the dormant task is configured with `HelpChannels.deleted_idle_minutes`. """ if not self.is_in_category(msg.channel, constants.Categories.help_in_use) or not self.is_empty(msg.channel): return @@ -738,14 +745,10 @@ class HelpChannels(Scheduler, commands.Cog): async def is_empty(self, channel: discord.TextChannel) -> bool: """Return True if the most recent message in `channel` is the bot's `AVAILABLE_MSG`.""" msg = await self.get_last_message(channel) - if not msg or not msg.author.bot or not msg.embeds: + if not msg: return False - embed = msg.embeds[0] - if embed.description == AVAILABLE_MSG: - return True - else: - return False + return self.embed_description_match(msg, AVAILABLE_MSG) async def reset_send_permissions(self) -> None: """Reset send permissions in the Available category for claimants.""" -- cgit v1.2.3 From b3619949a17ba40a3b1f6364cf83464275717283 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 22 May 2020 08:35:03 +0300 Subject: Eval Stats: Replaced `elif` with `else` on icon check Co-authored-by: Mark --- bot/cogs/snekbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index efff6d815..e2e55e7ca 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -208,7 +208,7 @@ class Snekbox(Cog): # Collect stats of eval fails + successes if icon == ":x:": self.bot.stats.incr("snekbox.python.fail") - elif icon in (":warning:", ":white_check_mark:"): + else: self.bot.stats.incr("snekbox.python.success") response = await ctx.send(msg) -- cgit v1.2.3 From d3550ce1138e1b00e64ba355a09a08b480b077e6 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 11:11:43 -0700 Subject: HelpChannels: fix `is_empty` not being awaited --- bot/cogs/help_channels.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 554fdc55e..2aec22be4 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -468,7 +468,7 @@ class HelpChannels(Scheduler, commands.Cog): """ log.trace(f"Handling in-use channel #{channel} ({channel.id}).") - if not self.is_empty(channel): + if not await self.is_empty(channel): idle_seconds = constants.HelpChannels.idle_minutes * 60 else: idle_seconds = constants.HelpChannels.deleted_idle_minutes * 60 @@ -731,7 +731,10 @@ class HelpChannels(Scheduler, commands.Cog): The new time for the dormant task is configured with `HelpChannels.deleted_idle_minutes`. """ - if not self.is_in_category(msg.channel, constants.Categories.help_in_use) or not self.is_empty(msg.channel): + if not self.is_in_category(msg.channel, constants.Categories.help_in_use): + return + + if not await self.is_empty(msg.channel): return log.info(f"Claimant of #{msg.channel} ({msg.author}) deleted message, channel is empty now. Rescheduling task.") -- cgit v1.2.3 From 8deeeca83c6c2c3de3b856ea8d6f94f8b5db3526 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 11:16:21 -0700 Subject: HelpChannels: rename `embed_description_match` --- bot/cogs/help_channels.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 2aec22be4..b9b577256 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -443,15 +443,15 @@ class HelpChannels(Scheduler, commands.Cog): if not message: return False - return self.embed_description_match(message, DORMANT_MSG) + return self.match_bot_embed(message, DORMANT_MSG) - def embed_description_match(self, message: discord.Message, text: str) -> bool: - """Return `True` if `message` embed description match with `text`.""" + def match_bot_embed(self, message: discord.Message, description: str) -> bool: + """Return `True` if the bot's `message`'s embed description matches `description`.""" if not message.embeds: return False embed = message.embeds[0] - return message.author == self.bot.user and embed.description.strip() == text.strip() + return message.author == self.bot.user and embed.description.strip() == description.strip() @staticmethod def is_in_category(channel: discord.TextChannel, category_id: int) -> bool: @@ -751,7 +751,7 @@ class HelpChannels(Scheduler, commands.Cog): if not msg: return False - return self.embed_description_match(msg, AVAILABLE_MSG) + return self.match_bot_embed(msg, AVAILABLE_MSG) async def reset_send_permissions(self) -> None: """Reset send permissions in the Available category for claimants.""" -- cgit v1.2.3 From e8266b8e1029f31dea3ad6ecbe36b7df56b8acdc Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 11:20:46 -0700 Subject: HelpChannels: move message None check inside `match_bot_embed` It was being done repeatedly outside the function so let's move it in to reduce redundancy. --- bot/cogs/help_channels.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index b9b577256..07acff34d 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -440,14 +440,11 @@ class HelpChannels(Scheduler, commands.Cog): def is_dormant_message(self, message: t.Optional[discord.Message]) -> bool: """Return True if the contents of the `message` match `DORMANT_MSG`.""" - if not message: - return False - return self.match_bot_embed(message, DORMANT_MSG) - def match_bot_embed(self, message: discord.Message, description: str) -> bool: + def match_bot_embed(self, message: t.Optional[discord.Message], description: str) -> bool: """Return `True` if the bot's `message`'s embed description matches `description`.""" - if not message.embeds: + if not message or not message.embeds: return False embed = message.embeds[0] @@ -748,9 +745,6 @@ class HelpChannels(Scheduler, commands.Cog): async def is_empty(self, channel: discord.TextChannel) -> bool: """Return True if the most recent message in `channel` is the bot's `AVAILABLE_MSG`.""" msg = await self.get_last_message(channel) - if not msg: - return False - return self.match_bot_embed(msg, AVAILABLE_MSG) async def reset_send_permissions(self) -> None: -- cgit v1.2.3 From 278ae309be27058920424c4049272bd5171bc158 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 11:22:27 -0700 Subject: HelpChannels: remove `is_dormant_message` At this point, it's just a thin wrapper to call another function. It's redundant. --- bot/cogs/help_channels.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 07acff34d..f0e6746f0 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -438,10 +438,6 @@ class HelpChannels(Scheduler, commands.Cog): """Return True if `member` has the 'Help Cooldown' role.""" return any(constants.Roles.help_cooldown == role.id for role in member.roles) - def is_dormant_message(self, message: t.Optional[discord.Message]) -> bool: - """Return True if the contents of the `message` match `DORMANT_MSG`.""" - return self.match_bot_embed(message, DORMANT_MSG) - def match_bot_embed(self, message: t.Optional[discord.Message], description: str) -> bool: """Return `True` if the bot's `message`'s embed description matches `description`.""" if not message or not message.embeds: @@ -822,7 +818,7 @@ class HelpChannels(Scheduler, commands.Cog): embed = discord.Embed(description=AVAILABLE_MSG) msg = await self.get_last_message(channel) - if self.is_dormant_message(msg): + if self.match_bot_embed(msg, DORMANT_MSG): log.trace(f"Found dormant message {msg.id} in {channel_info}; editing it.") await msg.edit(embed=embed) else: -- cgit v1.2.3 From 57fe4bf893e94289b5b6f7158ff2d6b92b1e3fae Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Fri, 22 May 2020 22:56:25 +0200 Subject: Set up async testbed --- bot/bot.py | 3 +- bot/utils/redis_cache.py | 13 ++- tests/bot/utils/test_redis_cache.py | 128 ++++++++++++++++++++++++ tests/bot/utils/test_redis_dict.py | 189 ------------------------------------ 4 files changed, 135 insertions(+), 198 deletions(-) create mode 100644 tests/bot/utils/test_redis_cache.py delete mode 100644 tests/bot/utils/test_redis_dict.py diff --git a/bot/bot.py b/bot/bot.py index f55eec5bb..8a3805989 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -45,8 +45,7 @@ class Bot(commands.Bot): # will effectively disable stats. statsd_url = "127.0.0.1" - asyncio.create_task(self._create_redis_session()) - + self.loop.create_task(self._create_redis_session()) self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") async def _create_redis_session(self) -> None: diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 467f16767..483bbc2cd 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -30,8 +30,8 @@ class RedisCache: def __init__(self) -> None: """Raise a NotImplementedError if `__set_name__` hasn't been run.""" - if not self._namespace: - raise NotImplementedError("RedisCache must be a class attribute.") + self._namespace = None + self.bot = None def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" @@ -47,8 +47,7 @@ class RedisCache: Called automatically when this class is constructed inside a class as an attribute. """ - if not self._has_custom_namespace: - self._set_namespace(f"{owner.__name__}.{attribute_name}") + self._set_namespace(f"{owner.__name__}.{attribute_name}") def __get__(self, instance: RedisCache, owner: Any) -> RedisCache: """Fetch the Bot instance, we need it for the redis pool.""" @@ -106,9 +105,9 @@ class RedisCache: async def pop(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: """Get the item, remove it from the cache, and provide a default if not found.""" - value = await self.get(key, default) - await self.delete(key) - return value + # value = await self.get(key, default) + # await self.delete(key) + # return value async def update(self) -> None: """Update the Redis cache with multiple values.""" diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py new file mode 100644 index 000000000..f6344803f --- /dev/null +++ b/tests/bot/utils/test_redis_cache.py @@ -0,0 +1,128 @@ +import asyncio +import unittest +from unittest.mock import MagicMock + +import fakeredis.aioredis + +from bot.bot import Bot +from bot.utils import RedisCache + + +class RedisCacheTests(unittest.IsolatedAsyncioTestCase): + """Tests the RedisDict class from utils.redis_dict.py.""" + + redis = RedisCache() + + async def asyncSetUp(self): # noqa: N802 - this special method can't be all lowercase + """Sets up the objects that only have to be initialized once.""" + self.bot = MagicMock( + spec=Bot, + redis_session=await fakeredis.aioredis.create_redis_pool(), + _redis_ready=asyncio.Event(), + ) + self.bot._redis_ready.set() + + def test_class_attribute_namespace(self): + """Test that RedisDict creates a namespace automatically for class attributes.""" + self.assertEqual(self.redis._namespace, "RedisCacheTests.redis") + # Test that errors are raised when this isn't true. + + # def test_set_get_item(self): + # """Test that users can set and get items from the RedisDict.""" + # self.redis['favorite_fruit'] = 'melon' + # self.redis['favorite_number'] = 86 + # self.assertEqual(self.redis['favorite_fruit'], 'melon') + # self.assertEqual(self.redis['favorite_number'], 86) + # + # def test_set_item_types(self): + # """Test that setitem rejects keys and values that are not strings, ints or floats.""" + # fruits = ["lemon", "melon", "apple"] + # + # with self.assertRaises(DataError): + # self.redis[fruits] = "nice" + # + # def test_contains(self): + # """Test that we can reliably use the `in` operator with our RedisDict.""" + # self.redis['favorite_country'] = "Burkina Faso" + # + # self.assertIn('favorite_country', self.redis) + # self.assertNotIn('favorite_dentist', self.redis) + # + # def test_items(self): + # """Test that the RedisDict can be iterated.""" + # self.redis.clear() + # test_cases = ( + # ('favorite_turtle', 'Donatello'), + # ('second_favorite_turtle', 'Leonardo'), + # ('third_favorite_turtle', 'Raphael'), + # ) + # for key, value in test_cases: + # self.redis[key] = value + # + # # Test regular iteration + # for test_case, key in zip(test_cases, self.redis): + # value = test_case[1] + # self.assertEqual(self.redis[key], value) + # + # # Test .items iteration + # for key, value in self.redis.items(): + # self.assertEqual(self.redis[key], value) + # + # # Test .keys iteration + # for test_case, key in zip(test_cases, self.redis.keys()): + # value = test_case[1] + # self.assertEqual(self.redis[key], value) + # + # def test_length(self): + # """Test that we can get the correct len() from the RedisDict.""" + # self.redis.clear() + # self.redis['one'] = 1 + # self.redis['two'] = 2 + # self.redis['three'] = 3 + # self.assertEqual(len(self.redis), 3) + # + # self.redis['four'] = 4 + # self.assertEqual(len(self.redis), 4) + # + # def test_to_dict(self): + # """Test that the .copy method returns a workable dictionary copy.""" + # copy = self.redis.copy() + # local_copy = dict(self.redis.items()) + # self.assertIs(type(copy), dict) + # self.assertEqual(copy, local_copy) + # + # def test_clear(self): + # """Test that the .clear method removes the entire hash.""" + # self.redis.clear() + # self.redis['teddy'] = "with me" + # self.redis['in my dreams'] = "you have a weird hat" + # self.assertEqual(len(self.redis), 2) + # + # self.redis.clear() + # self.assertEqual(len(self.redis), 0) + # + # def test_pop(self): + # """Test that we can .pop an item from the RedisDict.""" + # self.redis.clear() + # self.redis['john'] = 'was afraid' + # + # self.assertEqual(self.redis.pop('john'), 'was afraid') + # self.assertEqual(self.redis.pop('pete', 'breakneck'), 'breakneck') + # self.assertEqual(len(self.redis), 0) + # + # def test_update(self): + # """Test that we can .update the RedisDict with multiple items.""" + # self.redis.clear() + # self.redis["reckfried"] = "lona" + # self.redis["bel air"] = "prince" + # self.redis.update({ + # "reckfried": "jona", + # "mega": "hungry, though", + # }) + # + # result = { + # "reckfried": "jona", + # "bel air": "prince", + # "mega": "hungry, though", + # } + # self.assertEqual(self.redis.copy(), result) diff --git a/tests/bot/utils/test_redis_dict.py b/tests/bot/utils/test_redis_dict.py deleted file mode 100644 index f422887ce..000000000 --- a/tests/bot/utils/test_redis_dict.py +++ /dev/null @@ -1,189 +0,0 @@ -import unittest - -import fakeredis -from redis import DataError - -from bot.utils import RedisDict - -redis_server = fakeredis.FakeServer() -RedisDict._redis = fakeredis.FakeStrictRedis(server=redis_server) - - -class RedisDictTests(unittest.TestCase): - """Tests the RedisDict class from utils.redis_dict.py.""" - - redis = RedisDict() - - def test_class_attribute_namespace(self): - """Test that RedisDict creates a namespace automatically for class attributes.""" - self.assertEqual(self.redis._namespace, "RedisDictTests.redis") - - def test_custom_namespace(self): - """Test that users can set a custom namespaces which never collide.""" - test_cases = ( - (RedisDict("firedog")._namespace, "firedog"), - (RedisDict("firedog")._namespace, "firedog_"), - (RedisDict("firedog")._namespace, "firedog__"), - ) - - for test_case, result in test_cases: - self.assertEqual(test_case, result) - - def test_custom_namespace_takes_precedence(self): - """Test that custom namespaces take precedence over class attribute ones.""" - class LemonJuice: - citrus = RedisDict("citrus") - watercat = RedisDict() - - test_class = LemonJuice() - self.assertEqual(test_class.citrus._namespace, "citrus") - self.assertEqual(test_class.watercat._namespace, "LemonJuice.watercat") - - def test_set_get_item(self): - """Test that users can set and get items from the RedisDict.""" - self.redis['favorite_fruit'] = 'melon' - self.redis['favorite_number'] = 86 - self.assertEqual(self.redis['favorite_fruit'], 'melon') - self.assertEqual(self.redis['favorite_number'], 86) - - def test_set_item_value_types(self): - """Test that setitem rejects values that are not JSON serializable.""" - with self.assertRaises(TypeError): - self.redis['favorite_thing'] = object - self.redis['favorite_stuff'] = RedisDict - - def test_set_item_key_types(self): - """Test that setitem rejects keys that are not strings, ints or floats.""" - fruits = ["lemon", "melon", "apple"] - - with self.assertRaises(DataError): - self.redis[fruits] = "nice" - - def test_get_method(self): - """Test that the .get method works like in a dict.""" - self.redis['favorite_movie'] = 'Code Jam Highlights' - - self.assertEqual(self.redis.get('favorite_movie'), 'Code Jam Highlights') - self.assertEqual(self.redis.get('favorite_youtuber', 'pydis'), 'pydis') - self.assertIsNone(self.redis.get('favorite_dog')) - - def test_membership(self): - """Test that we can reliably use the `in` operator with our RedisDict.""" - self.redis['favorite_country'] = "Burkina Faso" - - self.assertIn('favorite_country', self.redis) - self.assertNotIn('favorite_dentist', self.redis) - - def test_del_item(self): - """Test that users can delete items from the RedisDict.""" - self.redis['favorite_band'] = "Radiohead" - self.assertIn('favorite_band', self.redis) - - del self.redis['favorite_band'] - self.assertNotIn('favorite_band', self.redis) - - def test_iter(self): - """Test that the RedisDict can be iterated.""" - self.redis.clear() - test_cases = ( - ('favorite_turtle', 'Donatello'), - ('second_favorite_turtle', 'Leonardo'), - ('third_favorite_turtle', 'Raphael'), - ) - for key, value in test_cases: - self.redis[key] = value - - # Test regular iteration - for test_case, key in zip(test_cases, self.redis): - value = test_case[1] - self.assertEqual(self.redis[key], value) - - # Test .items iteration - for key, value in self.redis.items(): - self.assertEqual(self.redis[key], value) - - # Test .keys iteration - for test_case, key in zip(test_cases, self.redis.keys()): - value = test_case[1] - self.assertEqual(self.redis[key], value) - - def test_len(self): - """Test that we can get the correct len() from the RedisDict.""" - self.redis.clear() - self.redis['one'] = 1 - self.redis['two'] = 2 - self.redis['three'] = 3 - self.assertEqual(len(self.redis), 3) - - self.redis['four'] = 4 - self.assertEqual(len(self.redis), 4) - - def test_copy(self): - """Test that the .copy method returns a workable dictionary copy.""" - copy = self.redis.copy() - local_copy = dict(self.redis.items()) - self.assertIs(type(copy), dict) - self.assertEqual(copy, local_copy) - - def test_clear(self): - """Test that the .clear method removes the entire hash.""" - self.redis.clear() - self.redis['teddy'] = "with me" - self.redis['in my dreams'] = "you have a weird hat" - self.assertEqual(len(self.redis), 2) - - self.redis.clear() - self.assertEqual(len(self.redis), 0) - - def test_pop(self): - """Test that we can .pop an item from the RedisDict.""" - self.redis.clear() - self.redis['john'] = 'was afraid' - - self.assertEqual(self.redis.pop('john'), 'was afraid') - self.assertEqual(self.redis.pop('pete', 'breakneck'), 'breakneck') - self.assertEqual(len(self.redis), 0) - - def test_popitem(self): - """Test that we can .popitem an item from the RedisDict.""" - self.redis.clear() - self.redis['john'] = 'the revalator' - self.redis['teddy'] = 'big bear' - - self.assertEqual(len(self.redis), 2) - self.assertEqual(self.redis.popitem(), 'big bear') - self.assertEqual(len(self.redis), 1) - - def test_setdefault(self): - """Test that we can .setdefault an item from the RedisDict.""" - self.redis.clear() - self.redis.setdefault('john', 'is yellow and weak') - self.assertEqual(self.redis['john'], 'is yellow and weak') - - with self.assertRaises(TypeError): - self.redis.setdefault('geisha', object) - - def test_update(self): - """Test that we can .update the RedisDict with multiple items.""" - self.redis.clear() - self.redis["reckfried"] = "lona" - self.redis["bel air"] = "prince" - self.redis.update({ - "reckfried": "jona", - "mega": "hungry, though", - }) - - result = { - "reckfried": "jona", - "bel air": "prince", - "mega": "hungry, though", - } - self.assertEqual(self.redis.copy(), result) - - def test_equals(self): - """Test that RedisDicts can be compared with == and !=.""" - new_redis_dict = RedisDict("firedog_the_sequel") - new_new_redis_dict = new_redis_dict - - self.assertEqual(new_redis_dict, new_new_redis_dict) - self.assertNotEqual(new_redis_dict, self.redis) -- cgit v1.2.3 From fd6f3d30b4c67f9a81346bb142d4696948fa2812 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 15:40:50 -0700 Subject: Fix assertion for `create_task` in duck pond tests The assertion wasn't using the assertion method. Furthermore, it was testing a non-existent function `create_loop` rather than `create_task`. --- tests/bot/cogs/test_duck_pond.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_duck_pond.py b/tests/bot/cogs/test_duck_pond.py index 7e6bfc748..a8c0107c6 100644 --- a/tests/bot/cogs/test_duck_pond.py +++ b/tests/bot/cogs/test_duck_pond.py @@ -45,7 +45,7 @@ class DuckPondTests(base.LoggingTestsMixin, unittest.IsolatedAsyncioTestCase): self.assertEqual(cog.bot, bot) self.assertEqual(cog.webhook_id, constants.Webhooks.duck_pond) - bot.loop.create_loop.called_once_with(cog.fetch_webhook()) + bot.loop.create_task.assert_called_once_with(cog.fetch_webhook()) def test_fetch_webhook_succeeds_without_connectivity_issues(self): """The `fetch_webhook` method waits until `READY` event and sets the `webhook` attribute.""" -- cgit v1.2.3 From 45e6f8dba869a367b01d99a596bd3355802d1fbe Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 15:44:04 -0700 Subject: Improve aiohttp context manager mocking in snekbox tests I'm not sure how it even managed to work before. It was calling the `post` coroutine (without specifying a URL) and then changing `__aenter__`. Now, a separate mock is created for the context manager and the `post` simply returns that mocked context manager. --- tests/bot/cogs/test_snekbox.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/bot/cogs/test_snekbox.py b/tests/bot/cogs/test_snekbox.py index 1dec0ccaf..84b273a7d 100644 --- a/tests/bot/cogs/test_snekbox.py +++ b/tests/bot/cogs/test_snekbox.py @@ -21,7 +21,10 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): """Post the eval code to the URLs.snekbox_eval_api endpoint.""" resp = MagicMock() resp.json = AsyncMock(return_value="return") - self.bot.http_session.post().__aenter__.return_value = resp + + context_manager = MagicMock() + context_manager.__aenter__.return_value = resp + self.bot.http_session.post.return_value = context_manager self.assertEqual(await self.cog.post_eval("import random"), "return") self.bot.http_session.post.assert_called_with( @@ -41,7 +44,10 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): key = "MarkDiamond" resp = MagicMock() resp.json = AsyncMock(return_value={"key": key}) - self.bot.http_session.post().__aenter__.return_value = resp + + context_manager = MagicMock() + context_manager.__aenter__.return_value = resp + self.bot.http_session.post.return_value = context_manager self.assertEqual( await self.cog.upload_output("My awesome output"), @@ -57,7 +63,10 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): """Output upload gracefully fallback if the upload fail.""" resp = MagicMock() resp.json = AsyncMock(side_effect=Exception) - self.bot.http_session.post().__aenter__.return_value = resp + + context_manager = MagicMock() + context_manager.__aenter__.return_value = resp + self.bot.http_session.post.return_value = context_manager log = logging.getLogger("bot.cogs.snekbox") with self.assertLogs(logger=log, level='ERROR'): -- cgit v1.2.3 From 6aed2f6b69b79b5a7e5f327819d026e7a63a7dab Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 16:15:23 -0700 Subject: Fix unawaited coro warning when instantiating Bot for MockBot's spec The fix is to mock the loop and pass it to the Bot. It will then set it as `self.loop` rather than trying to get an event loop from asyncio. The `create_task` patch has been moved to this loop mock rather than being done in MockBot to ensure that it applies to anything calling it when instantiating the Bot. --- tests/helpers.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 2b79a6c2a..2efeff7db 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -4,6 +4,7 @@ import collections import itertools import logging import unittest.mock +from asyncio import AbstractEventLoop from typing import Iterable, Optional import discord @@ -264,10 +265,16 @@ class MockAPIClient(CustomMockMixin, unittest.mock.MagicMock): spec_set = APIClient -# Create a Bot instance to get a realistic MagicMock of `discord.ext.commands.Bot` -bot_instance = Bot(command_prefix=unittest.mock.MagicMock()) -bot_instance.http_session = None -bot_instance.api_client = None +def _get_mock_loop() -> unittest.mock.Mock: + """Return a mocked asyncio.AbstractEventLoop.""" + loop = unittest.mock.create_autospec(spec=AbstractEventLoop, spec_set=True) + + # Since calling `create_task` on our MockBot does not actually schedule the coroutine object + # as a task in the asyncio loop, this `side_effect` calls `close()` on the coroutine object + # to prevent "has not been awaited"-warnings. + loop.create_task.side_effect = lambda coroutine: coroutine.close() + + return loop class MockBot(CustomMockMixin, unittest.mock.MagicMock): @@ -277,17 +284,14 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): Instances of this class will follow the specifications of `discord.ext.commands.Bot` instances. For more information, see the `MockGuild` docstring. """ - spec_set = bot_instance + spec_set = Bot(command_prefix=unittest.mock.MagicMock(), loop=_get_mock_loop()) additional_spec_asyncs = ("wait_for",) def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - self.api_client = MockAPIClient() - # Since calling `create_task` on our MockBot does not actually schedule the coroutine object - # as a task in the asyncio loop, this `side_effect` calls `close()` on the coroutine object - # to prevent "has not been awaited"-warnings. - self.loop.create_task.side_effect = lambda coroutine: coroutine.close() + self.loop = _get_mock_loop() + self.api_client = MockAPIClient(loop=self.loop) # Create a TextChannel instance to get a realistic MagicMock of `discord.TextChannel` -- cgit v1.2.3 From 1ad7833d800918efca06e5d6b2fbafdb0d757009 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 16:23:12 -0700 Subject: Properly mock the redis pool in MockBot Because some of the redis pool/connection methods return futures rather than being coroutines, the redis pool had to be mocked using the CustomMockMixin so it could take advantage of `additional_spec_asyncs` to use AsyncMocks for these future-returning methods. --- tests/bot/utils/test_redis_cache.py | 12 +++--------- tests/helpers.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index f6344803f..991225481 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -1,11 +1,9 @@ -import asyncio import unittest -from unittest.mock import MagicMock import fakeredis.aioredis -from bot.bot import Bot from bot.utils import RedisCache +from tests import helpers class RedisCacheTests(unittest.IsolatedAsyncioTestCase): @@ -15,12 +13,8 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): # noqa: N802 - this special method can't be all lowercase """Sets up the objects that only have to be initialized once.""" - self.bot = MagicMock( - spec=Bot, - redis_session=await fakeredis.aioredis.create_redis_pool(), - _redis_ready=asyncio.Event(), - ) - self.bot._redis_ready.set() + self.bot = helpers.MockBot() + self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() def test_class_attribute_namespace(self): """Test that RedisDict creates a namespace automatically for class attributes.""" diff --git a/tests/helpers.py b/tests/helpers.py index 2efeff7db..33d4f787c 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -7,6 +7,7 @@ import unittest.mock from asyncio import AbstractEventLoop from typing import Iterable, Optional +import aioredis.abc import discord from discord.ext.commands import Context @@ -265,6 +266,17 @@ class MockAPIClient(CustomMockMixin, unittest.mock.MagicMock): spec_set = APIClient +class MockRedisPool(CustomMockMixin, unittest.mock.MagicMock): + """ + A MagicMock subclass to mock an aioredis connection pool. + + Instances of this class will follow the specifications of `aioredis.abc.AbcPool` instances. + For more information, see the `MockGuild` docstring. + """ + spec_set = aioredis.abc.AbcPool + additional_spec_asyncs = ("execute", "execute_pubsub") + + def _get_mock_loop() -> unittest.mock.Mock: """Return a mocked asyncio.AbstractEventLoop.""" loop = unittest.mock.create_autospec(spec=AbstractEventLoop, spec_set=True) @@ -293,6 +305,10 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): self.loop = _get_mock_loop() self.api_client = MockAPIClient(loop=self.loop) + # fakeredis can't be used cause it'd require awaiting a coroutine to create the pool, + # which cannot be done here in __init__. + self.redis_session = MockRedisPool() + # Create a TextChannel instance to get a realistic MagicMock of `discord.TextChannel` channel_data = { -- cgit v1.2.3 From d8f1634ab68b2cd480d57c8b9da8834866b5c9cc Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 16:25:10 -0700 Subject: Use autospecced mocks in MockBot for the stats and aiohttp session This will help catch anything that tries to get/set an attribute/method which doesn't exist. It'll also catch missing/too many parameters being passed to methods. --- tests/helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/helpers.py b/tests/helpers.py index 33d4f787c..d226be3f0 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -9,9 +9,11 @@ from typing import Iterable, Optional import aioredis.abc import discord +from aiohttp import ClientSession from discord.ext.commands import Context from bot.api import APIClient +from bot.async_stats import AsyncStatsClient from bot.bot import Bot @@ -304,6 +306,8 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): self.loop = _get_mock_loop() self.api_client = MockAPIClient(loop=self.loop) + self.http_session = unittest.mock.create_autospec(spec=ClientSession, spec_set=True) + self.stats = unittest.mock.create_autospec(spec=AsyncStatsClient, spec_set=True) # fakeredis can't be used cause it'd require awaiting a coroutine to create the pool, # which cannot be done here in __init__. -- cgit v1.2.3 From eb63fb02a49bf1979afd04a1350304edf00d3a56 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 02:06:27 +0200 Subject: Finish .set and .get, and add tests. The .set and .get will accept ints, floats, and strings. These will be converted into "typestrings", which is basically just a simple format that's been invented for this object. For example, an int looks like `b"i|2423"`. Note how it is still stored as a bytestring (like everything in Redis), but because of this prefix we are able to coerce it into the type we want on the way out of the db. --- bot/utils/redis_cache.py | 72 ++++++++++++++++++++++++++++++------- tests/bot/utils/test_redis_cache.py | 36 +++++++++++++------ tests/helpers.py | 2 +- 3 files changed, 85 insertions(+), 25 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 483bbc2cd..24f2f2e03 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -1,12 +1,10 @@ from __future__ import annotations -from enum import Enum -from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Union +from typing import Any, AsyncIterator, Dict, Optional, Union from bot.bot import Bot -ValidRedisKey = Union[str, int, float] -JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] +ValidRedisType = Union[str, int, float] class RedisCache: @@ -41,7 +39,39 @@ class RedisCache: self._namespaces.append(namespace) self._namespace = namespace - def __set_name__(self, owner: object, attribute_name: str) -> None: + @staticmethod + def _to_typestring(value: ValidRedisType) -> str: + """Turn a valid Redis type into a typestring.""" + if isinstance(value, float): + return f"f|{value}" + elif isinstance(value, int): + return f"i|{value}" + elif isinstance(value, str): + return f"s|{value}" + + @staticmethod + def _from_typestring(value: str) -> ValidRedisType: + """Turn a valid Redis type into a typestring.""" + if value.startswith("f|"): + return float(value[2:]) + if value.startswith("i|"): + return int(value[2:]) + if value.startswith("s|"): + return value[2:] + + async def _validate_cache(self) -> None: + """Validate that the RedisCache is ready to be used.""" + if self.bot is None: + raise RuntimeError("Critical error: RedisCache has no `Bot` instance.") + + if self._namespace is None: + raise RuntimeError( + "Critical error: RedisCache has no namespace. " + "Did you initialize this object as a class attribute?" + ) + await self.bot._redis_ready.wait() + + def __set_name__(self, owner: Any, attribute_name: str) -> None: """ Set the namespace to Class.attribute_name. @@ -54,8 +84,11 @@ class RedisCache: if self.bot: return self + if self._namespace is None: + raise RuntimeError("RedisCache must be a class attribute.") + if instance is None: - raise NotImplementedError("You must create an instance of RedisCache to use it.") + raise RuntimeError("You must create an instance of RedisCache to use it.") for attribute in vars(instance).values(): if isinstance(attribute, Bot): @@ -69,19 +102,32 @@ class RedisCache: """Return a beautiful representation of this object instance.""" return f"RedisCache(namespace={self._namespace!r})" - async def set(self, key: ValidRedisKey, value: JSONSerializableType) -> None: + async def set(self, key: ValidRedisType, value: ValidRedisType) -> None: """Store an item in the Redis cache.""" - # await self._redis.hset(self._namespace, key, value) + await self._validate_cache() + + # Convert to a typestring and then set it + key = self._to_typestring(key) + value = self._to_typestring(value) + await self._redis.hset(self._namespace, key, value) - async def get(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + async def get(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: """Get an item from the Redis cache.""" - # value = await self._redis.hget(self._namespace, key) + await self._validate_cache() + key = self._to_typestring(key) + value = await self._redis.hget(self._namespace, key) + + if value is None: + return default + else: + value = self._from_typestring(value.decode("utf-8")) + return value - async def delete(self, key: ValidRedisKey) -> None: + async def delete(self, key: ValidRedisType) -> None: """Delete an item from the Redis cache.""" # await self._redis.hdel(self._namespace, key) - async def contains(self, key: ValidRedisKey) -> bool: + async def contains(self, key: ValidRedisType) -> bool: """Check if a key exists in the Redis cache.""" # return await self._redis.hexists(self._namespace, key) @@ -103,7 +149,7 @@ class RedisCache: """Deletes the entire hash from the Redis cache.""" # await self._redis.delete(self._namespace) - async def pop(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + async def pop(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: """Get the item, remove it from the cache, and provide a default if not found.""" # value = await self.get(key, default) # await self.delete(key) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 991225481..ad38bfde0 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -7,27 +7,41 @@ from tests import helpers class RedisCacheTests(unittest.IsolatedAsyncioTestCase): - """Tests the RedisDict class from utils.redis_dict.py.""" + """Tests the RedisCache class from utils.redis_dict.py.""" redis = RedisCache() - async def asyncSetUp(self): # noqa: N802 - this special method can't be all lowercase + async def asyncSetUp(self): # noqa: N802 """Sets up the objects that only have to be initialized once.""" self.bot = helpers.MockBot() self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() - def test_class_attribute_namespace(self): + async def test_class_attribute_namespace(self): """Test that RedisDict creates a namespace automatically for class attributes.""" self.assertEqual(self.redis._namespace, "RedisCacheTests.redis") - # Test that errors are raised when this isn't true. - # def test_set_get_item(self): - # """Test that users can set and get items from the RedisDict.""" - # self.redis['favorite_fruit'] = 'melon' - # self.redis['favorite_number'] = 86 - # self.assertEqual(self.redis['favorite_fruit'], 'melon') - # self.assertEqual(self.redis['favorite_number'], 86) - # + # Test that errors are raised when not assigned as a class attribute + bad_cache = RedisCache() + + with self.assertRaises(RuntimeError): + await bad_cache.set("test", "me_up_deadman") + + async def test_set_get_item(self): + """Test that users can set and get items from the RedisDict.""" + test_cases = ( + ('favorite_fruit', 'melon'), + ('favorite_number', 86), + ('favorite_fraction', 86.54) + ) + + # Test that we can get and set different types. + for test in test_cases: + await self.redis.set(*test) + self.assertEqual(await self.redis.get(test[0]), test[1]) + + # Test that .get allows a default value + self.assertEqual(await self.redis.get('favorite_nothing', "bearclaw"), "bearclaw") + # def test_set_item_types(self): # """Test that setitem rejects keys and values that are not strings, ints or floats.""" # fruits = ["lemon", "melon", "apple"] diff --git a/tests/helpers.py b/tests/helpers.py index d226be3f0..2b176db79 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -299,7 +299,7 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): For more information, see the `MockGuild` docstring. """ spec_set = Bot(command_prefix=unittest.mock.MagicMock(), loop=_get_mock_loop()) - additional_spec_asyncs = ("wait_for",) + additional_spec_asyncs = ("wait_for", "_redis_ready") def __init__(self, **kwargs) -> None: super().__init__(**kwargs) -- cgit v1.2.3 From c8a9a7713c4394556faadb432d1ed3b7ba5c103a Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 11:35:19 +0200 Subject: Finish asyncifying RedisCache methods - All methods will now do a validation check - Complete interface spec added to class: - .update - .clear - .pop - .to_dict - .length - .contains - .delete - .get - .set --- bot/utils/redis_cache.py | 50 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 24f2f2e03..bd14fc239 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -50,8 +50,10 @@ class RedisCache: return f"s|{value}" @staticmethod - def _from_typestring(value: str) -> ValidRedisType: - """Turn a valid Redis type into a typestring.""" + def _from_typestring(value: Union[bytes, str]) -> ValidRedisType: + """Turn a typestring into a valid Redis type.""" + if isinstance(value, bytes): + value = value.decode('utf-8') if value.startswith("f|"): return float(value[2:]) if value.startswith("i|"): @@ -59,6 +61,14 @@ class RedisCache: if value.startswith("s|"): return value[2:] + def _dict_from_typestring(self, dictionary: Dict) -> Dict: + """Turns all contents of a dict into valid Redis types.""" + return {self._from_typestring(key): self._from_typestring(value) for key, value in dictionary.items()} + + def _dict_to_typestring(self, dictionary: Dict) -> Dict: + """Turns all contents of a dict into typestrings.""" + return {self._to_typestring(key): self._to_typestring(value) for key, value in dictionary.items()} + async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" if self.bot is None: @@ -120,41 +130,49 @@ class RedisCache: if value is None: return default else: - value = self._from_typestring(value.decode("utf-8")) + value = self._from_typestring(value) return value async def delete(self, key: ValidRedisType) -> None: """Delete an item from the Redis cache.""" - # await self._redis.hdel(self._namespace, key) + await self._validate_cache() + key = self._to_typestring(key) + return await self._redis.hdel(self._namespace, key) async def contains(self, key: ValidRedisType) -> bool: """Check if a key exists in the Redis cache.""" - # return await self._redis.hexists(self._namespace, key) + await self._validate_cache() + key = self._to_typestring(key) + return await self._redis.hexists(self._namespace, key) async def items(self) -> AsyncIterator: """Iterate all the items in the Redis cache.""" - # data = await redis.hgetall(self.get_with_namespace(key)) - # for item in data: - # yield item + await self._validate_cache() + data = await self._redis.hgetall(self._namespace) # Get all the keys + for key, value in self._dict_from_typestring(data).items(): + yield key, value async def length(self) -> int: """Return the number of items in the Redis cache.""" - # return await self._redis.hlen(self._namespace) + await self._validate_cache() + return await self._redis.hlen(self._namespace) async def to_dict(self) -> Dict: """Convert to dict and return.""" - # return dict(self.items()) + return {key: value async for key, value in self.items()} async def clear(self) -> None: """Deletes the entire hash from the Redis cache.""" - # await self._redis.delete(self._namespace) + await self._validate_cache() + await self._redis.delete(self._namespace) async def pop(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: """Get the item, remove it from the cache, and provide a default if not found.""" - # value = await self.get(key, default) - # await self.delete(key) - # return value + value = await self.get(key, default) + await self.delete(key) + return value - async def update(self) -> None: + async def update(self, items: Dict) -> None: """Update the Redis cache with multiple values.""" - # https://aioredis.readthedocs.io/en/v1.3.0/mixins.html#aioredis.commands.HashCommandsMixin.hmset_dict + await self._validate_cache() + await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) -- cgit v1.2.3 From 387bf5c6b6a21e25c4fc690fb992b6b3e4c165a6 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 11:36:12 +0200 Subject: Complete asyncified test suite for RedisCache This commit just alters existing code to work with the new interface, and with async. All tests are passing successfully. --- tests/bot/utils/test_redis_cache.py | 206 ++++++++++++++++++++---------------- 1 file changed, 112 insertions(+), 94 deletions(-) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index ad38bfde0..d257e91d9 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -16,16 +16,24 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): self.bot = helpers.MockBot() self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() - async def test_class_attribute_namespace(self): + def test_class_attribute_namespace(self): """Test that RedisDict creates a namespace automatically for class attributes.""" self.assertEqual(self.redis._namespace, "RedisCacheTests.redis") - # Test that errors are raised when not assigned as a class attribute + async def test_class_attribute_required(self): + """Test that errors are raised when not assigned as a class attribute.""" bad_cache = RedisCache() + self.assertIs(bad_cache._namespace, None) with self.assertRaises(RuntimeError): await bad_cache.set("test", "me_up_deadman") + def test_namespace_collision(self): + """Test that we prevent colliding namespaces.""" + bad_cache = RedisCache() + bad_cache._set_namespace("RedisCacheTests.redis") + self.assertEqual(bad_cache._namespace, "RedisCacheTests.redis_") + async def test_set_get_item(self): """Test that users can set and get items from the RedisDict.""" test_cases = ( @@ -42,95 +50,105 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): # Test that .get allows a default value self.assertEqual(await self.redis.get('favorite_nothing', "bearclaw"), "bearclaw") - # def test_set_item_types(self): - # """Test that setitem rejects keys and values that are not strings, ints or floats.""" - # fruits = ["lemon", "melon", "apple"] - # - # with self.assertRaises(DataError): - # self.redis[fruits] = "nice" - # - # def test_contains(self): - # """Test that we can reliably use the `in` operator with our RedisDict.""" - # self.redis['favorite_country'] = "Burkina Faso" - # - # self.assertIn('favorite_country', self.redis) - # self.assertNotIn('favorite_dentist', self.redis) - # - # def test_items(self): - # """Test that the RedisDict can be iterated.""" - # self.redis.clear() - # test_cases = ( - # ('favorite_turtle', 'Donatello'), - # ('second_favorite_turtle', 'Leonardo'), - # ('third_favorite_turtle', 'Raphael'), - # ) - # for key, value in test_cases: - # self.redis[key] = value - # - # # Test regular iteration - # for test_case, key in zip(test_cases, self.redis): - # value = test_case[1] - # self.assertEqual(self.redis[key], value) - # - # # Test .items iteration - # for key, value in self.redis.items(): - # self.assertEqual(self.redis[key], value) - # - # # Test .keys iteration - # for test_case, key in zip(test_cases, self.redis.keys()): - # value = test_case[1] - # self.assertEqual(self.redis[key], value) - # - # def test_length(self): - # """Test that we can get the correct len() from the RedisDict.""" - # self.redis.clear() - # self.redis['one'] = 1 - # self.redis['two'] = 2 - # self.redis['three'] = 3 - # self.assertEqual(len(self.redis), 3) - # - # self.redis['four'] = 4 - # self.assertEqual(len(self.redis), 4) - # - # def test_to_dict(self): - # """Test that the .copy method returns a workable dictionary copy.""" - # copy = self.redis.copy() - # local_copy = dict(self.redis.items()) - # self.assertIs(type(copy), dict) - # self.assertEqual(copy, local_copy) - # - # def test_clear(self): - # """Test that the .clear method removes the entire hash.""" - # self.redis.clear() - # self.redis['teddy'] = "with me" - # self.redis['in my dreams'] = "you have a weird hat" - # self.assertEqual(len(self.redis), 2) - # - # self.redis.clear() - # self.assertEqual(len(self.redis), 0) - # - # def test_pop(self): - # """Test that we can .pop an item from the RedisDict.""" - # self.redis.clear() - # self.redis['john'] = 'was afraid' - # - # self.assertEqual(self.redis.pop('john'), 'was afraid') - # self.assertEqual(self.redis.pop('pete', 'breakneck'), 'breakneck') - # self.assertEqual(len(self.redis), 0) - # - # def test_update(self): - # """Test that we can .update the RedisDict with multiple items.""" - # self.redis.clear() - # self.redis["reckfried"] = "lona" - # self.redis["bel air"] = "prince" - # self.redis.update({ - # "reckfried": "jona", - # "mega": "hungry, though", - # }) - # - # result = { - # "reckfried": "jona", - # "bel air": "prince", - # "mega": "hungry, though", - # } - # self.assertEqual(self.redis.copy(), result) + async def test_set_item_type(self): + """Test that .set rejects keys and values that are not strings, ints or floats.""" + fruits = ["lemon", "melon", "apple"] + + with self.assertRaises(TypeError): + await self.redis.set(fruits, "nice") + + async def test_delete_item(self): + """Test that .delete allows us to delete stuff from the RedisCache.""" + # Add an item and verify that it gets added + await self.redis.set("internet", "firetruck") + self.assertEqual(await self.redis.get("internet"), "firetruck") + + # Delete that item and verify that it gets deleted + await self.redis.delete("internet") + self.assertIs(await self.redis.get("internet"), None) + + async def test_contains(self): + """Test that we can check membership with .contains.""" + await self.redis.set('favorite_country', "Burkina Faso") + + self.assertIs(await self.redis.contains('favorite_country'), True) + self.assertIs(await self.redis.contains('favorite_dentist'), False) + + async def test_items(self): + """Test that the RedisDict can be iterated.""" + await self.redis.clear() + + # Set up our test cases in the Redis cache + test_cases = [ + ('favorite_turtle', 'Donatello'), + ('second_favorite_turtle', 'Leonardo'), + ('third_favorite_turtle', 'Raphael'), + ] + for key, value in test_cases: + await self.redis.set(key, value) + + # Consume the AsyncIterator into a regular list, easier to compare that way. + redis_items = [item async for item in self.redis.items()] + + # These sequences are probably in the same order now, but probably + # isn't good enough for tests. Let's not rely on .hgetall always + # returning things in sequence, and just sort both lists to be safe. + redis_items = sorted(redis_items) + test_cases = sorted(test_cases) + + # If these are equal now, everything works fine. + self.assertSequenceEqual(test_cases, redis_items) + + async def test_length(self): + """Test that we can get the correct .length from the RedisDict.""" + await self.redis.clear() + await self.redis.set('one', 1) + await self.redis.set('two', 2) + await self.redis.set('three', 3) + self.assertEqual(await self.redis.length(), 3) + + await self.redis.set('four', 4) + self.assertEqual(await self.redis.length(), 4) + + async def test_to_dict(self): + """Test that the .copy method returns a workable dictionary copy.""" + copy = await self.redis.to_dict() + local_copy = {key: value async for key, value in self.redis.items()} + self.assertIs(type(copy), dict) + self.assertDictEqual(copy, local_copy) + + async def test_clear(self): + """Test that the .clear method removes the entire hash.""" + await self.redis.clear() + await self.redis.set('teddy', 'with me') + await self.redis.set('in my dreams', 'you have a weird hat') + self.assertEqual(await self.redis.length(), 2) + + await self.redis.clear() + self.assertEqual(await self.redis.length(), 0) + + async def test_pop(self): + """Test that we can .pop an item from the RedisDict.""" + await self.redis.clear() + await self.redis.set('john', 'was afraid') + + self.assertEqual(await self.redis.pop('john'), 'was afraid') + self.assertEqual(await self.redis.pop('pete', 'breakneck'), 'breakneck') + self.assertEqual(await self.redis.length(), 0) + + async def test_update(self): + """Test that we can .update the RedisDict with multiple items.""" + await self.redis.clear() + await self.redis.set("reckfried", "lona") + await self.redis.set("bel air", "prince") + await self.redis.update({ + "reckfried": "jona", + "mega": "hungry, though", + }) + + result = { + "reckfried": "jona", + "bel air": "prince", + "mega": "hungry, though", + } + self.assertDictEqual(await self.redis.to_dict(), result) -- cgit v1.2.3 From 5bd8e13088822cfa4b189a30a7d745de61984dc7 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 11:45:01 +0200 Subject: Better docstring for RedisCache --- bot/utils/redis_cache.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index bd14fc239..26a100ef0 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -21,7 +21,37 @@ class RedisCache: We implement several convenient methods that are fairly similar to have a dict behaves, and should be familiar to Python users. The biggest difference is that - all the public methods in this class are coroutines. + all the public methods in this class are coroutines, and must be awaited. + + Because of limitations in Redis, this cache will only accept strings, integers and + floats both for keys and values. + + Simple example for how to use this: + + class SomeCog(Cog): + # To initialize a valid RedisCache, just add it as a class attribute here. + # Do not add it to the __init__ method or anywhere else, it MUST be a class + # attribute. Do not pass any parameters. + cache = RedisCache() + + async def my_method(self): + # Now we can store some stuff in the cache just by doing this. + # This data will persist through restarts! + await self.cache.set("key", "value") + + # To get the data, simply do this. + value = await self.cache.get("key") + + # Other methods work more or less like a dictionary. + # Checking if something is in the cache + await self.cache.contains("key") + + # iterating the cache + async for key, value in self.cache.items(): + print(value) + + # We can even iterate in a comprehension! + consumed = [value async for key, value in self.cache.items()] """ _namespaces = [] -- cgit v1.2.3 From db75440a273277111e7140b1c226630b865d154b Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 11:55:10 +0200 Subject: Better docstring for RedisCache.contains --- bot/utils/redis_cache.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 26a100ef0..2b60ae0c3 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -170,7 +170,11 @@ class RedisCache: return await self._redis.hdel(self._namespace, key) async def contains(self, key: ValidRedisType) -> bool: - """Check if a key exists in the Redis cache.""" + """ + Check if a key exists in the Redis cache. + + Return True if the key exists, otherwise False. + """ await self._validate_cache() key = self._to_typestring(key) return await self._redis.hexists(self._namespace, key) -- cgit v1.2.3 From fc1999ea80df2ebc904260ff0e6f56d9b36bc6c5 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 12:17:39 +0200 Subject: Unbreak the error_handler --- bot/cogs/error_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index e635bd46f..77d16c051 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -167,7 +167,7 @@ class ErrorHandler(Cog): self.bot.stats.incr("errors.missing_required_argument") elif isinstance(e, errors.TooManyArguments): await ctx.send("Too many arguments provided.") - await ctx.invoke(*help_command) + await prepared_help_command self.bot.stats.incr("errors.too_many_arguments") elif isinstance(e, errors.BadArgument): await ctx.send(f"Bad argument: {e}\n") -- cgit v1.2.3 From 49492ffd5e5c87d347048dc370085be12215ed7f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 14:12:13 +0200 Subject: Moving the Redis session creation to Bot._recreate --- bot/bot.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bot/bot.py b/bot/bot.py index 8a3805989..224f5f4e4 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -45,7 +45,6 @@ class Bot(commands.Bot): # will effectively disable stats. statsd_url = "127.0.0.1" - self.loop.create_task(self._create_redis_session()) self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") async def _create_redis_session(self) -> None: @@ -91,6 +90,7 @@ class Bot(commands.Bot): self.stats._transport.close() if self.redis_session: + self._redis_ready.clear() self.redis_session.close() await self.redis_session.wait_closed() @@ -101,7 +101,7 @@ class Bot(commands.Bot): await super().login(*args, **kwargs) def _recreate(self) -> None: - """Re-create the connector, aiohttp session, and the APIClient.""" + """Re-create the connector, aiohttp session, the APIClient and the Redis session.""" # Use asyncio for DNS resolution instead of threads so threads aren't spammed. # Doesn't seem to have any state with regards to being closed, so no need to worry? self._resolver = aiohttp.AsyncResolver() @@ -112,6 +112,9 @@ class Bot(commands.Bot): "The previous connector was not closed; it will remain open and be overwritten" ) + # Create the redis session + self.loop.create_task(self._create_redis_session()) + # Use AF_INET as its socket family to prevent HTTPS related problems both locally # and in production. self._connector = aiohttp.TCPConnector( -- cgit v1.2.3 From 489f9405a77eb88baa0d77a88fd04bf13cbbde1f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 14:21:39 +0200 Subject: CI needs REDIS_PASSWORD to pass tests --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d56675029..4500cb6e8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,6 +22,7 @@ jobs: REDDIT_CLIENT_ID: spam REDDIT_SECRET: ham WOLFRAM_API_KEY: baz + REDIS_PASSWORD: '' steps: - task: UsePythonVersion@0 -- cgit v1.2.3 From 1ea471865fa68a001e25980d50e71333752add6d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 14:27:11 +0200 Subject: Update exception message This was incorrectly suggesting the user needed to create an instance of RedisCache, when in fact it is the parent that needs to be instantiated. Co-authored-by: Sebastiaan Zeeff <33516116+SebastiaanZ@users.noreply.github.com> --- bot/utils/redis_cache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 2b60ae0c3..f9d9e571f 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -128,7 +128,10 @@ class RedisCache: raise RuntimeError("RedisCache must be a class attribute.") if instance is None: - raise RuntimeError("You must create an instance of RedisCache to use it.") + raise RuntimeError( + "You must access the RedisCache instance through the cog instance " + "before accessing it using the cog's class object." + ) for attribute in vars(instance).values(): if isinstance(attribute, Bot): -- cgit v1.2.3 From aa0bb028ed889d93376981213673053a540e137c Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 14:30:56 +0200 Subject: Fix typo in test_to_dict docstring --- tests/bot/utils/test_redis_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index d257e91d9..2ce57499a 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -111,7 +111,7 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(await self.redis.length(), 4) async def test_to_dict(self): - """Test that the .copy method returns a workable dictionary copy.""" + """Test that the .to_dict method returns a workable dictionary copy.""" copy = await self.redis.to_dict() local_copy = {key: value async for key, value in self.redis.items()} self.assertIs(type(copy), dict) -- cgit v1.2.3 From 2fb86258471626863c2214cabc2529e78c77729a Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 14:41:16 +0200 Subject: Don't rely on HDEL ignoring keys for .pop Previously we would try to .delete keys that did not exist if a default was provided when calling .pop. This is okay to do (because HDEL will just ignore any attempts to delete non-existing calls), but it does add an additional pointless API call to Redis, so I've added some validation as a small optimization. This also adds a few additional lines of documentation as requested by @SebastiaanZ in their review. --- bot/utils/redis_cache.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index f9d9e571f..6831be157 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -35,6 +35,14 @@ class RedisCache: cache = RedisCache() async def my_method(self): + + # Now we're ready to use the RedisCache. + # One thing to note here is that this will not work unless + # we access self.cache through an _instance_ of this class. + # + # For example, attempting to use SomeCog.cache will _not_ work, + # you _must_ instantiate the class first and use that instance. + # # Now we can store some stuff in the cache just by doing this. # This data will persist through restarts! await self.cache.set("key", "value") @@ -167,7 +175,13 @@ class RedisCache: return value async def delete(self, key: ValidRedisType) -> None: - """Delete an item from the Redis cache.""" + """ + Delete an item from the Redis cache. + + If we try to delete a key that does not exist, it will simply be ignored. + + See https://redis.io/commands/hdel for more info on how this works. + """ await self._validate_cache() key = self._to_typestring(key) return await self._redis.hdel(self._namespace, key) @@ -206,7 +220,12 @@ class RedisCache: async def pop(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: """Get the item, remove it from the cache, and provide a default if not found.""" value = await self.get(key, default) - await self.delete(key) + + # No need to try to delete something that doesn't exist, + # that's just a superfluous API call. + if value != default: + await self.delete(key) + return value async def update(self, items: Dict) -> None: -- cgit v1.2.3 From 5120717a47c07812d1631cf0905ff3062e139487 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 15:17:57 +0200 Subject: DRY approach to typestring prefix resolution Thanks to @kwzrd for this idea, basically we're making a constant with the typestring prefixes and iterating that in all our converters. These converter functions will also now raise TypeErrors if we try to convert something that isn't in this constants list. I've also added a new test that tests this functionality. --- bot/utils/redis_cache.py | 54 ++++++++++++++++++++++++++----------- tests/bot/utils/test_redis_cache.py | 21 +++++++++++++++ 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 6831be157..1ec1b9fea 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -4,6 +4,11 @@ from typing import Any, AsyncIterator, Dict, Optional, Union from bot.bot import Bot +TYPESTRING_PREFIXES = ( + ("f|", float), + ("i|", int), + ("s|", str), +) ValidRedisType = Union[str, int, float] @@ -78,26 +83,45 @@ class RedisCache: self._namespace = namespace @staticmethod - def _to_typestring(value: ValidRedisType) -> str: - """Turn a valid Redis type into a typestring.""" - if isinstance(value, float): - return f"f|{value}" - elif isinstance(value, int): - return f"i|{value}" - elif isinstance(value, str): - return f"s|{value}" + def _valid_typestring_types() -> str: + """ + Creates a nice, readable list of valid types for typestrings, useful for error messages. + + This will be dynamically updated if we change the TYPESTRING_PREFIXES constant up top. + """ + valid_types = ", ".join([str(_type).split("'")[1] for _, _type in TYPESTRING_PREFIXES]) + valid_types = ", and ".join(valid_types.rsplit(", ", 1)) + return valid_types @staticmethod - def _from_typestring(value: Union[bytes, str]) -> ValidRedisType: + def _valid_typestring_prefixes() -> str: + """ + Creates a nice, readable list of valid prefixes for typestrings, useful for error messages. + + This will be dynamically updated if we change the TYPESTRING_PREFIXES constant up top. + """ + valid_prefixes = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_PREFIXES]) + valid_prefixes = ", and ".join(valid_prefixes.rsplit(", ", 1)) + return valid_prefixes + + def _to_typestring(self, value: ValidRedisType) -> str: + """Turn a valid Redis type into a typestring.""" + for prefix, _type in TYPESTRING_PREFIXES: + if isinstance(value, _type): + return f"{prefix}{value}" + raise TypeError(f"RedisCache._from_typestring only supports the types {self._valid_typestring_types()}.") + + def _from_typestring(self, value: Union[bytes, str]) -> ValidRedisType: """Turn a typestring into a valid Redis type.""" + # Stuff that comes out of Redis will be bytestrings, so let's decode those. if isinstance(value, bytes): value = value.decode('utf-8') - if value.startswith("f|"): - return float(value[2:]) - if value.startswith("i|"): - return int(value[2:]) - if value.startswith("s|"): - return value[2:] + + # Now we convert our unicode string back into the type it originally was. + for prefix, _type in TYPESTRING_PREFIXES: + if value.startswith(prefix): + return _type(value[2:]) + raise TypeError(f"RedisCache._to_typestring only supports the prefixes {self._valid_typestring_prefixes()}.") def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 2ce57499a..150195726 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -152,3 +152,24 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): "mega": "hungry, though", } self.assertDictEqual(await self.redis.to_dict(), result) + + def test_typestring_conversion(self): + """Test the typestring-related helper functions.""" + conversion_tests = ( + (12, "i|12"), + (12.4, "f|12.4"), + ("cowabunga", "s|cowabunga"), + ) + + # Test conversion to typestring + for _input, expected in conversion_tests: + self.assertEqual(self.redis._to_typestring(_input), expected) + + # Test conversion from typestrings + for _input, expected in conversion_tests: + self.assertEqual(self.redis._from_typestring(expected), _input) + + # Test that exceptions are raised on invalid input + with self.assertRaises(TypeError): + self.redis._to_typestring(["internet"]) + self.redis._from_typestring("o|firedog") -- cgit v1.2.3 From a52a13020f3468c671cb549052a9c8e303ae9d8c Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 23 May 2020 10:27:32 -0700 Subject: Remove redis session mock from MockBot It's not feasible to mock it because all the commands return futures rather than being coroutines, so they cannot automatically be turned into AsyncMocks. Furthermore, no code should ever use the redis session directly besides RedisCache. Since the tests for RedisCache already use fakeredis, there's no use in trying to mock redis in MockBot. --- tests/helpers.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 2b176db79..5ad826156 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -7,7 +7,6 @@ import unittest.mock from asyncio import AbstractEventLoop from typing import Iterable, Optional -import aioredis.abc import discord from aiohttp import ClientSession from discord.ext.commands import Context @@ -268,17 +267,6 @@ class MockAPIClient(CustomMockMixin, unittest.mock.MagicMock): spec_set = APIClient -class MockRedisPool(CustomMockMixin, unittest.mock.MagicMock): - """ - A MagicMock subclass to mock an aioredis connection pool. - - Instances of this class will follow the specifications of `aioredis.abc.AbcPool` instances. - For more information, see the `MockGuild` docstring. - """ - spec_set = aioredis.abc.AbcPool - additional_spec_asyncs = ("execute", "execute_pubsub") - - def _get_mock_loop() -> unittest.mock.Mock: """Return a mocked asyncio.AbstractEventLoop.""" loop = unittest.mock.create_autospec(spec=AbstractEventLoop, spec_set=True) @@ -309,10 +297,6 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): self.http_session = unittest.mock.create_autospec(spec=ClientSession, spec_set=True) self.stats = unittest.mock.create_autospec(spec=AsyncStatsClient, spec_set=True) - # fakeredis can't be used cause it'd require awaiting a coroutine to create the pool, - # which cannot be done here in __init__. - self.redis_session = MockRedisPool() - # Create a TextChannel instance to get a realistic MagicMock of `discord.TextChannel` channel_data = { -- cgit v1.2.3 From 923d03a8040251ae7766b9655a3d0ff9f8413c8b Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 23 May 2020 11:14:13 -0700 Subject: Show a warning when redis pool isn't closed --- bot/bot.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bot/bot.py b/bot/bot.py index 224f5f4e4..bf7f9c9df 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -112,6 +112,11 @@ class Bot(commands.Bot): "The previous connector was not closed; it will remain open and be overwritten" ) + if self.redis_session and not self.redis_session.closed: + log.warning( + "The previous redis pool was not closed; it will remain open and be overwritten" + ) + # Create the redis session self.loop.create_task(self._create_redis_session()) -- cgit v1.2.3 From 3f8dce7502e3afb2d119979cfc455efcde7ad9db Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 10:18:39 +0200 Subject: use __name__ for type list Instead of relying on __str__ representation, we'll use the __name__ dunder. Co-authored-by: Mark --- bot/utils/redis_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 1ec1b9fea..fadbca673 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -89,7 +89,7 @@ class RedisCache: This will be dynamically updated if we change the TYPESTRING_PREFIXES constant up top. """ - valid_types = ", ".join([str(_type).split("'")[1] for _, _type in TYPESTRING_PREFIXES]) + valid_types = ", ".join(str(_type.__name__) for _, _type in TYPESTRING_PREFIXES) valid_types = ", and ".join(valid_types.rsplit(", ", 1)) return valid_types -- cgit v1.2.3 From ed12a2fa303b7faebeb773dac096bd2b0b8ec23d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 10:25:17 +0200 Subject: len(prefix) instead of hardcoding 2 Co-authored-by: Mark --- bot/utils/redis_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index fadbca673..13e88e8e1 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -120,7 +120,7 @@ class RedisCache: # Now we convert our unicode string back into the type it originally was. for prefix, _type in TYPESTRING_PREFIXES: if value.startswith(prefix): - return _type(value[2:]) + return _type(value[len(prefix):]) raise TypeError(f"RedisCache._to_typestring only supports the prefixes {self._valid_typestring_prefixes()}.") def _dict_from_typestring(self, dictionary: Dict) -> Dict: -- cgit v1.2.3 From 98d8bb3e841ba52fe036b36492029a9fdeb36518 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 10:53:46 +0200 Subject: Refactor the nice prefix/type strings to constants It's leaner to just move that code out of the class and up to the module level as constants. This commit also renames ValidRedisType to RedisType. --- bot/utils/redis_cache.py | 50 ++++++++++++++++++------------------------------ 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 13e88e8e1..d5563c079 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -4,12 +4,20 @@ from typing import Any, AsyncIterator, Dict, Optional, Union from bot.bot import Bot +RedisType = Union[str, int, float] TYPESTRING_PREFIXES = ( ("f|", float), ("i|", int), ("s|", str), ) -ValidRedisType = Union[str, int, float] + +# Makes a nice list like "float, int, and str" +NICE_TYPE_LIST = ", ".join(str(_type.__name__) for _, _type in TYPESTRING_PREFIXES) +NICE_TYPE_LIST = ", and ".join(NICE_TYPE_LIST.rsplit(", ", 1)) + +# Makes a list like "'f|', 'i|', and 's|'" +NICE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_PREFIXES]) +NICE_PREFIX_LIST = ", and ".join(NICE_PREFIX_LIST.rsplit(", ", 1)) class RedisCache: @@ -83,35 +91,15 @@ class RedisCache: self._namespace = namespace @staticmethod - def _valid_typestring_types() -> str: - """ - Creates a nice, readable list of valid types for typestrings, useful for error messages. - - This will be dynamically updated if we change the TYPESTRING_PREFIXES constant up top. - """ - valid_types = ", ".join(str(_type.__name__) for _, _type in TYPESTRING_PREFIXES) - valid_types = ", and ".join(valid_types.rsplit(", ", 1)) - return valid_types - - @staticmethod - def _valid_typestring_prefixes() -> str: - """ - Creates a nice, readable list of valid prefixes for typestrings, useful for error messages. - - This will be dynamically updated if we change the TYPESTRING_PREFIXES constant up top. - """ - valid_prefixes = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_PREFIXES]) - valid_prefixes = ", and ".join(valid_prefixes.rsplit(", ", 1)) - return valid_prefixes - - def _to_typestring(self, value: ValidRedisType) -> str: + def _to_typestring(value: RedisType) -> str: """Turn a valid Redis type into a typestring.""" for prefix, _type in TYPESTRING_PREFIXES: if isinstance(value, _type): return f"{prefix}{value}" - raise TypeError(f"RedisCache._from_typestring only supports the types {self._valid_typestring_types()}.") + raise TypeError(f"RedisCache._from_typestring only supports the types {NICE_TYPE_LIST}.") - def _from_typestring(self, value: Union[bytes, str]) -> ValidRedisType: + @staticmethod + def _from_typestring(value: Union[bytes, str]) -> RedisType: """Turn a typestring into a valid Redis type.""" # Stuff that comes out of Redis will be bytestrings, so let's decode those. if isinstance(value, bytes): @@ -121,7 +109,7 @@ class RedisCache: for prefix, _type in TYPESTRING_PREFIXES: if value.startswith(prefix): return _type(value[len(prefix):]) - raise TypeError(f"RedisCache._to_typestring only supports the prefixes {self._valid_typestring_prefixes()}.") + raise TypeError(f"RedisCache._to_typestring only supports the prefixes {NICE_PREFIX_LIST}.") def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" @@ -177,7 +165,7 @@ class RedisCache: """Return a beautiful representation of this object instance.""" return f"RedisCache(namespace={self._namespace!r})" - async def set(self, key: ValidRedisType, value: ValidRedisType) -> None: + async def set(self, key: RedisType, value: RedisType) -> None: """Store an item in the Redis cache.""" await self._validate_cache() @@ -186,7 +174,7 @@ class RedisCache: value = self._to_typestring(value) await self._redis.hset(self._namespace, key, value) - async def get(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: + async def get(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: """Get an item from the Redis cache.""" await self._validate_cache() key = self._to_typestring(key) @@ -198,7 +186,7 @@ class RedisCache: value = self._from_typestring(value) return value - async def delete(self, key: ValidRedisType) -> None: + async def delete(self, key: RedisType) -> None: """ Delete an item from the Redis cache. @@ -210,7 +198,7 @@ class RedisCache: key = self._to_typestring(key) return await self._redis.hdel(self._namespace, key) - async def contains(self, key: ValidRedisType) -> bool: + async def contains(self, key: RedisType) -> bool: """ Check if a key exists in the Redis cache. @@ -241,7 +229,7 @@ class RedisCache: await self._validate_cache() await self._redis.delete(self._namespace) - async def pop(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: + async def pop(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: """Get the item, remove it from the cache, and provide a default if not found.""" value = await self.get(key, default) -- cgit v1.2.3 From 1d05a4d409cd0652cec36128114739ede2f529cf Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 10:54:34 +0200 Subject: Improves various docstrings and comments. Thanks to @MarkKoz for suggesting most of these in their code review. --- bot/utils/redis_cache.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index d5563c079..e4dce7526 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -24,14 +24,6 @@ class RedisCache: """ A simplified interface for a Redis connection. - This class must be created as a class attribute in a class. This is because it - uses __set_name__ to create a namespace like MyCog.my_class_attribute which is - used as a hash name when we store stuff in Redis, to prevent collisions. - - The class this object is instantiated in must also contains an attribute with an - instance of Bot. This is because Bot contains our redis_pool, which is how this - class communicates with the Redis server. - We implement several convenient methods that are fairly similar to have a dict behaves, and should be familiar to Python users. The biggest difference is that all the public methods in this class are coroutines, and must be awaited. @@ -39,6 +31,10 @@ class RedisCache: Because of limitations in Redis, this cache will only accept strings, integers and floats both for keys and values. + Please note that this class MUST be created as a class attribute, and that that class + must also contain an attribute with an instance of our Bot. See `__get__` and `__set_name__` + for more information about how this works. + Simple example for how to use this: class SomeCog(Cog): @@ -78,12 +74,18 @@ class RedisCache: _namespaces = [] def __init__(self) -> None: - """Raise a NotImplementedError if `__set_name__` hasn't been run.""" + """Initialize the RedisCache.""" self._namespace = None self.bot = None def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" + # We need a unique namespace, to prevent collisions. This loop + # will try appending underscores to the end of the namespace until + # it finds one that is unique. + # + # For example, if `john` and `john_` are both taken, the namespace will + # be `john__` at the end of this loop. while namespace in self._namespaces: namespace += "_" @@ -136,11 +138,26 @@ class RedisCache: Set the namespace to Class.attribute_name. Called automatically when this class is constructed inside a class as an attribute. + + This class MUST be created as a class attribute in a class, otherwise it will raise + exceptions whenever a method is used. This is because it uses this method to create + a namespace like `MyCog.my_class_attribute` which is used as a hash name when we store + stuff in Redis, to prevent collisions. """ self._set_namespace(f"{owner.__name__}.{attribute_name}") def __get__(self, instance: RedisCache, owner: Any) -> RedisCache: - """Fetch the Bot instance, we need it for the redis pool.""" + """ + This is called if the RedisCache is a class attribute, and is accessed. + + The class this object is instantiated in must contain an attribute with an + instance of Bot. This is because Bot contains our redis_session, which is + the mechanism by which we will communicate with the Redis server. + + Any attempt to use RedisCache in a class that does not have a Bot instance + will fail. It is mostly intended to be used inside of a Cog, although theoretically + it should work in any class that has a Bot instance. + """ if self.bot: return self -- cgit v1.2.3 From f05fefb4a51c6653f7f93805489838259782c376 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 11:26:04 +0200 Subject: Better RuntimeErrors. We provide suggestions for how to solve these problems now. --- bot/utils/redis_cache.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index e4dce7526..558ab33a7 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -124,7 +124,13 @@ class RedisCache: async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" if self.bot is None: - raise RuntimeError("Critical error: RedisCache has no `Bot` instance.") + raise RuntimeError( + "Critical error: RedisCache has no `Bot` instance. " + "This happens when the class RedisCache was created in doesn't " + "have a Bot instance. Please make sure that you're instantiating " + "the RedisCache inside a class that has a Bot instance " + "class attribute." + ) if self._namespace is None: raise RuntimeError( @@ -176,7 +182,13 @@ class RedisCache: self._redis = self.bot.redis_session return self else: - raise RuntimeError("Cannot initialize a RedisCache without a `Bot` instance.") + raise RuntimeError( + "Critical error: RedisCache has no `Bot` instance. " + "This happens when the class RedisCache was created in doesn't " + "have a Bot instance. Please make sure that you're instantiating " + "the RedisCache inside a class that has a Bot instance " + "class attribute." + ) def __repr__(self) -> str: """Return a beautiful representation of this object instance.""" -- cgit v1.2.3 From b2009d5304beba4829b7727ca154bb6a0d1cd50a Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 11:42:33 +0200 Subject: Make .items return ItemsView instead of AsyncIter There really was no compelling reason why this method should return an AsyncIterator or than that `async for items in cache.items()` has nice readability, but there were a few concerns. One is a concern about race conditions raised by @SebastiaanZ, and @MarkKoz raised a concern that it was misleading to have an AsyncIterator that only "pretended" to be lazy. To address these concerns, I've refactored it to return a regular ItemsView instead. I also improved the docstring, and fixed the relevant tests. --- bot/utils/redis_cache.py | 28 +++++++++++++++++++++------- tests/bot/utils/test_redis_cache.py | 4 ++-- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 558ab33a7..fb9a534bd 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, AsyncIterator, Dict, Optional, Union +from typing import Any, Dict, ItemsView, Optional, Union from bot.bot import Bot @@ -237,12 +237,26 @@ class RedisCache: key = self._to_typestring(key) return await self._redis.hexists(self._namespace, key) - async def items(self) -> AsyncIterator: - """Iterate all the items in the Redis cache.""" + async def items(self) -> ItemsView: + """ + Fetch all the key/value pairs in the cache. + + Returns a normal ItemsView, like you would get from dict.items(). + + Keep in mind that these items are just a _copy_ of the data in the + RedisCache - any changes you make to them will not be reflected + into the RedisCache itself. If you want to change these, you need + to make a .set call. + + Example: + items = await my_cache.items() + for key, value in items: + # Iterate like a normal dictionary + """ await self._validate_cache() - data = await self._redis.hgetall(self._namespace) # Get all the keys - for key, value in self._dict_from_typestring(data).items(): - yield key, value + return self._dict_from_typestring( + await self._redis.hgetall(self._namespace) + ).items() async def length(self) -> int: """Return the number of items in the Redis cache.""" @@ -251,7 +265,7 @@ class RedisCache: async def to_dict(self) -> Dict: """Convert to dict and return.""" - return {key: value async for key, value in self.items()} + return {key: value for key, value in await self.items()} async def clear(self) -> None: """Deletes the entire hash from the Redis cache.""" diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 150195726..6e12002ed 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -88,7 +88,7 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): await self.redis.set(key, value) # Consume the AsyncIterator into a regular list, easier to compare that way. - redis_items = [item async for item in self.redis.items()] + redis_items = [item for item in await self.redis.items()] # These sequences are probably in the same order now, but probably # isn't good enough for tests. Let's not rely on .hgetall always @@ -113,7 +113,7 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_to_dict(self): """Test that the .to_dict method returns a workable dictionary copy.""" copy = await self.redis.to_dict() - local_copy = {key: value async for key, value in self.redis.items()} + local_copy = {key: value for key, value in await self.redis.items()} self.assertIs(type(copy), dict) self.assertDictEqual(copy, local_copy) -- cgit v1.2.3 From 01bedcadf762262eef0a2b406faf66cdc16a5c85 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 13:04:41 +0200 Subject: Add .increment and .decrement methods. Sometimes, we just want to store a counter in the cache. In this case, it is convenient to have a single method that will allow us to increment or decrement this counter. These methods allow you to decrement or increment floats and integers by an specified amount. By default, it'll increment or decrement by 1. Since this involves several API requests, we create an asyncio.Lock so that we don't end up with race conditions. --- bot/utils/redis_cache.py | 35 +++++++++++++++++++++++++++++++++++ tests/bot/utils/test_redis_cache.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index fb9a534bd..290fae1a0 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio from typing import Any, Dict, ItemsView, Optional, Union from bot.bot import Bot @@ -77,6 +78,7 @@ class RedisCache: """Initialize the RedisCache.""" self._namespace = None self.bot = None + self.increment_lock = asyncio.Lock() def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" @@ -287,3 +289,36 @@ class RedisCache: """Update the Redis cache with multiple values.""" await self._validate_cache() await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) + + async def increment(self, key: RedisType, amount: Optional[int, float] = 1) -> None: + """ + Increment the value by `amount`. + + This works for both floats and ints, but will raise a TypeError + if you try to do it for any other type of value. + + This also supports negative amounts, although it would provide better + readability to use .decrement() for that. + """ + # Since this has several API calls, we need a lock to prevent race conditions + async with self.increment_lock: + value = await self.get(key) + + # Can't increment a non-existing value + if value is None: + raise RuntimeError("The provided key does not exist!") + + # If it does exist, and it's an int or a float, increment and set it. + if isinstance(value, int) or isinstance(value, float): + value += amount + await self.set(key, value) + else: + raise TypeError("You may only increment or decrement values that are integers or floats.") + + async def decrement(self, key: RedisType, amount: Optional[int, float] = 1) -> None: + """ + Decrement the value by `amount`. + + Basically just does the opposite of .increment. + """ + await self.increment(key, -amount) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 6e12002ed..dbbaef018 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -173,3 +173,37 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): with self.assertRaises(TypeError): self.redis._to_typestring(["internet"]) self.redis._from_typestring("o|firedog") + + async def test_increment_decrement(self): + """Test .increment and .decrement methods.""" + await self.redis.set("entropic", 5) + await self.redis.set("disentropic", 12.5) + + # Test default increment + await self.redis.increment("entropic") + self.assertEqual(await self.redis.get("entropic"), 6) + + # Test default decrement + await self.redis.decrement("entropic") + self.assertEqual(await self.redis.get("entropic"), 5) + + # Test float increment with float + await self.redis.increment("disentropic", 2.0) + self.assertEqual(await self.redis.get("disentropic"), 14.5) + + # Test float increment with int + await self.redis.increment("disentropic", 2) + self.assertEqual(await self.redis.get("disentropic"), 16.5) + + # Test negative increments, because why not. + await self.redis.increment("entropic", -5) + self.assertEqual(await self.redis.get("entropic"), 0) + + # Negative decrements? Sure. + await self.redis.decrement("entropic", -5) + self.assertEqual(await self.redis.get("entropic"), 5) + + # What about if we use a negative float to decrement an int? + # This should convert the type into a float. + await self.redis.decrement("entropic", -2.5) + self.assertEqual(await self.redis.get("entropic"), 7.5) -- cgit v1.2.3 From f80ce10aee4a46ab4fc4a2d249fe182fb812a826 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 13:13:03 +0200 Subject: Rename Bot._redis_ready to Bot.redis_ready It's a public attribute, we're accessing it from RedisCache. --- bot/bot.py | 6 +++--- bot/utils/redis_cache.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/bot.py b/bot/bot.py index bf7f9c9df..0d423201b 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -30,12 +30,12 @@ class Bot(commands.Bot): self.http_session: Optional[aiohttp.ClientSession] = None self.redis_session: Optional[aioredis.Redis] = None + self.redis_ready = asyncio.Event() self.api_client = api.APIClient(loop=self.loop) self._connector = None self._resolver = None self._guild_available = asyncio.Event() - self._redis_ready = asyncio.Event() statsd_url = constants.Stats.statsd_host @@ -53,7 +53,7 @@ class Bot(commands.Bot): address=(constants.Redis.host, constants.Redis.port), password=constants.Redis.password, ) - self._redis_ready.set() + self.redis_ready.set() def add_cog(self, cog: commands.Cog) -> None: """Adds a "cog" to the bot and logs the operation.""" @@ -90,7 +90,7 @@ class Bot(commands.Bot): self.stats._transport.close() if self.redis_session: - self._redis_ready.clear() + self.redis_ready.clear() self.redis_session.close() await self.redis_session.wait_closed() diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 290fae1a0..bd885c22c 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -139,7 +139,7 @@ class RedisCache: "Critical error: RedisCache has no namespace. " "Did you initialize this object as a class attribute?" ) - await self.bot._redis_ready.wait() + await self.bot.redis_ready.wait() def __set_name__(self, owner: Any, attribute_name: str) -> None: """ -- cgit v1.2.3 From 361b740f27a579a085e93ebfdd06df10f386cca1 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 13:44:18 +0200 Subject: Add logging to the RedisCache. Mostly trace and exception logging. --- bot/utils/redis_cache.py | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index bd885c22c..5fc34d464 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -1,10 +1,13 @@ from __future__ import annotations import asyncio +import logging from typing import Any, Dict, ItemsView, Optional, Union from bot.bot import Bot +log = logging.getLogger(__name__) + RedisType = Union[str, int, float] TYPESTRING_PREFIXES = ( ("f|", float), @@ -91,6 +94,7 @@ class RedisCache: while namespace in self._namespaces: namespace += "_" + log.trace(f"RedisCache setting namespace to {self._namespace}") self._namespaces.append(namespace) self._namespace = namespace @@ -126,6 +130,7 @@ class RedisCache: async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" if self.bot is None: + log.exception("Attempt to use RedisCache with no `Bot` instance.") raise RuntimeError( "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " @@ -135,6 +140,7 @@ class RedisCache: ) if self._namespace is None: + log.exception("Attempt to use RedisCache with no namespace.") raise RuntimeError( "Critical error: RedisCache has no namespace. " "Did you initialize this object as a class attribute?" @@ -170,9 +176,14 @@ class RedisCache: return self if self._namespace is None: + log.exception("RedisCache must be a class attribute.") raise RuntimeError("RedisCache must be a class attribute.") if instance is None: + log.exception( + "Attempt to access RedisCache instance through the cog's class object " + "before accessing it through the cog instance." + ) raise RuntimeError( "You must access the RedisCache instance through the cog instance " "before accessing it using the cog's class object." @@ -184,6 +195,7 @@ class RedisCache: self._redis = self.bot.redis_session return self else: + log.exception("Attempt to use RedisCache with no `Bot` instance.") raise RuntimeError( "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " @@ -203,18 +215,24 @@ class RedisCache: # Convert to a typestring and then set it key = self._to_typestring(key) value = self._to_typestring(value) + + log.trace(f"Setting {key} to {value}.") await self._redis.hset(self._namespace, key, value) async def get(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: """Get an item from the Redis cache.""" await self._validate_cache() key = self._to_typestring(key) + + log.trace(f"Attempting to retrieve {key}.") value = await self._redis.hget(self._namespace, key) if value is None: + log.trace(f"Value not found, returning default value {default}") return default else: value = self._from_typestring(value) + log.trace(f"Value found, returning value {value}") return value async def delete(self, key: RedisType) -> None: @@ -227,6 +245,8 @@ class RedisCache: """ await self._validate_cache() key = self._to_typestring(key) + + log.trace(f"Attempting to delete {key}.") return await self._redis.hdel(self._namespace, key) async def contains(self, key: RedisType) -> bool: @@ -237,7 +257,10 @@ class RedisCache: """ await self._validate_cache() key = self._to_typestring(key) - return await self._redis.hexists(self._namespace, key) + exists = await self._redis.hexists(self._namespace, key) + + log.trace(f"Testing if {key} exists in the RedisCache - Result is {exists}") + return exists async def items(self) -> ItemsView: """ @@ -256,14 +279,19 @@ class RedisCache: # Iterate like a normal dictionary """ await self._validate_cache() - return self._dict_from_typestring( + items = self._dict_from_typestring( await self._redis.hgetall(self._namespace) ).items() + log.trace(f"Retrieving all key/value pairs from cache, total of {len(items)} items.") + return items + async def length(self) -> int: """Return the number of items in the Redis cache.""" await self._validate_cache() - return await self._redis.hlen(self._namespace) + number_of_items = await self._redis.hlen(self._namespace) + log.trace(f"Returning length. Result is {number_of_items}.") + return number_of_items async def to_dict(self) -> Dict: """Convert to dict and return.""" @@ -272,15 +300,18 @@ class RedisCache: async def clear(self) -> None: """Deletes the entire hash from the Redis cache.""" await self._validate_cache() + log.trace("Clearing the cache of all key/value pairs.") await self._redis.delete(self._namespace) async def pop(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: """Get the item, remove it from the cache, and provide a default if not found.""" + log.trace(f"Attempting to pop {key}.") value = await self.get(key, default) # No need to try to delete something that doesn't exist, # that's just a superfluous API call. if value != default: + log.trace(f"Key {key} exists, deleting it from the cache.") await self.delete(key) return value @@ -288,6 +319,7 @@ class RedisCache: async def update(self, items: Dict) -> None: """Update the Redis cache with multiple values.""" await self._validate_cache() + log.trace(f"Updating the cache with the following items:\n{items}") await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) async def increment(self, key: RedisType, amount: Optional[int, float] = 1) -> None: @@ -300,12 +332,15 @@ class RedisCache: This also supports negative amounts, although it would provide better readability to use .decrement() for that. """ + log.trace(f"Attempting to increment/decrement the value with the key {key} by {amount}.") + # Since this has several API calls, we need a lock to prevent race conditions async with self.increment_lock: value = await self.get(key) # Can't increment a non-existing value if value is None: + log.exception("Attempt to increment/decrement value for non-existent key.") raise RuntimeError("The provided key does not exist!") # If it does exist, and it's an int or a float, increment and set it. @@ -313,6 +348,7 @@ class RedisCache: value += amount await self.set(key, value) else: + log.exception("Attempt to increment/decrement non-numerical value.") raise TypeError("You may only increment or decrement values that are integers or floats.") async def decrement(self, key: RedisType, amount: Optional[int, float] = 1) -> None: -- cgit v1.2.3 From c5e6e8f796265ee6faebdd3d02c839972cd028a9 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 13:49:20 +0200 Subject: MockBot needs to be aware of redis_ready Forgot to update the additional_spec_asyncs when changing the name of this Bot attribute to be public. --- tests/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers.py b/tests/helpers.py index 5ad826156..13283339b 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -287,7 +287,7 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): For more information, see the `MockGuild` docstring. """ spec_set = Bot(command_prefix=unittest.mock.MagicMock(), loop=_get_mock_loop()) - additional_spec_asyncs = ("wait_for", "_redis_ready") + additional_spec_asyncs = ("wait_for", "redis_ready") def __init__(self, **kwargs) -> None: super().__init__(**kwargs) -- cgit v1.2.3 From 66d273f0b8f1de6850760f4d561db446c027fdfe Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 15:21:35 +0200 Subject: Add an option to use fakeredis in Bot. Without this option, all contributors would need to set up a Redis server in order to run the bot. But with use_fakeredis set to True, this is no longer necessary because it will just set up a fakeredis redis pool instead of trying to contact an actual server. This is more than good enough for most local testing purposes, since data persistence across restarts isn't really relevant for them. This also means we need to move fakeredis into our real dependency list instead of having it as a dev dependency, so there's a minor change for that as well. I also made a small kaizen change to sort all the dependencies in the Pipfile alphabetically. --- Pipfile | 28 +++++++++++++------------- Pipfile.lock | 58 +++++++++++++++++++++++++++--------------------------- bot/bot.py | 26 +++++++++++++++++++----- bot/constants.py | 1 + config-default.yml | 1 + 5 files changed, 66 insertions(+), 48 deletions(-) diff --git a/Pipfile b/Pipfile index cd2f2ad7a..b42ca6d58 100644 --- a/Pipfile +++ b/Pipfile @@ -4,30 +4,30 @@ verify_ssl = true name = "pypi" [packages] -discord.py = "~=1.3.2" +aio-pika = "~=6.1" aiodns = "~=2.0" aiohttp = "~=3.5" -sphinx = "~=2.2" -markdownify = "~=0.4" -lxml = "~=4.4" -pyyaml = "~=5.1" +aioredis = "~=1.3.1" +beautifulsoup4 = "~=4.9" +colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} +coloredlogs = "~=14.0" +deepdiff = "~=4.0" +discord.py = "~=1.3.2" +fakeredis = "~=1.4" +feedparser = "~=5.2" fuzzywuzzy = "~=0.17" -aio-pika = "~=6.1" +lxml = "~=4.4" +markdownify = "~=0.4" +more_itertools = "~=8.2" python-dateutil = "~=2.8" -deepdiff = "~=4.0" +pyyaml = "~=5.1" requests = "~=2.22" -more_itertools = "~=8.2" sentry-sdk = "~=0.14" -coloredlogs = "~=14.0" -colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} +sphinx = "~=2.2" statsd = "~=3.3" -feedparser = "~=5.2" -beautifulsoup4 = "~=4.9" -aioredis = "~=1.3.1" [dev-packages] coverage = "~=5.0" -fakeredis = "~=1.4" flake8 = "~=3.7" flake8-annotations = "~=2.0" flake8-bugbear = "~=20.1" diff --git a/Pipfile.lock b/Pipfile.lock index 1941f6887..0e591710c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c0b3e4d3e2c9ddb6ba28d2c09d521fe90ad4ea3df5c7ea7cd3a8b679fb3f85f9" + "sha256": "0297accc3d614d3da8080b89d56ef7fe489c28a0ada8102df396a604af7ee330" }, "pipfile-spec": 6, "requires": { @@ -196,6 +196,14 @@ ], "version": "==0.16" }, + "fakeredis": { + "hashes": [ + "sha256:4d170886865a91dbc8b7f8cbd4e5d488f4c5f2f25dfae127f001617bbe9e8f97", + "sha256:647b2593d349d9d4e566c8dadb2e4c71ba35be5bdc4f1f7ac2d565a12a965053" + ], + "index": "pypi", + "version": "==1.4.1" + }, "feedparser": { "hashes": [ "sha256:bd030652c2d08532c034c27fcd7c85868e7fa3cb2b17f230a44a6bbc92519bf9", @@ -501,6 +509,13 @@ "index": "pypi", "version": "==5.3.1" }, + "redis": { + "hashes": [ + "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", + "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" + ], + "version": "==3.5.2" + }, "requests": { "hashes": [ "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", @@ -531,6 +546,13 @@ ], "version": "==2.0.0" }, + "sortedcontainers": { + "hashes": [ + "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", + "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" + ], + "version": "==2.1.0" + }, "soupsieve": { "hashes": [ "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", @@ -718,14 +740,6 @@ ], "version": "==0.3.0" }, - "fakeredis": { - "hashes": [ - "sha256:4d170886865a91dbc8b7f8cbd4e5d488f4c5f2f25dfae127f001617bbe9e8f97", - "sha256:647b2593d349d9d4e566c8dadb2e4c71ba35be5bdc4f1f7ac2d565a12a965053" - ], - "index": "pypi", - "version": "==1.4.1" - }, "filelock": { "hashes": [ "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", @@ -735,11 +749,11 @@ }, "flake8": { "hashes": [ - "sha256:6c1193b0c3f853ef763969238f6c81e9e63ace9d024518edc020d5f1d6d93195", - "sha256:ea6623797bf9a52f4c9577d780da0bb17d65f870213f7b5bcc9fca82540c31d5" + "sha256:c69ac1668e434d37a2d2880b3ca9aafd54b3a10a3ac1ab101d22f29e29cf8634", + "sha256:ccaa799ef9893cebe69fdfefed76865aeaefbb94cb8545617b2298786a4de9a5" ], "index": "pypi", - "version": "==3.8.1" + "version": "==3.8.2" }, "flake8-annotations": { "hashes": [ @@ -805,10 +819,10 @@ }, "identify": { "hashes": [ - "sha256:23c18d97bb50e05be1a54917ee45cc61d57cb96aedc06aabb2b02331edf0dbf0", - "sha256:88ed90632023e52a6495749c6732e61e08ec9f4f04e95484a5c37b9caf40283c" + "sha256:0f3c3aac62b51b86fea6ff52fe8ff9e06f57f10411502443809064d23e16f1c2", + "sha256:f9ad3d41f01e98eb066b6e05c5b184fd1e925fadec48eb165b4e01c72a1ef3a7" ], - "version": "==1.4.15" + "version": "==1.4.16" }, "mccabe": { "hashes": [ @@ -877,13 +891,6 @@ "index": "pypi", "version": "==5.3.1" }, - "redis": { - "hashes": [ - "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", - "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" - ], - "version": "==3.5.2" - }, "six": { "hashes": [ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", @@ -898,13 +905,6 @@ ], "version": "==2.0.0" }, - "sortedcontainers": { - "hashes": [ - "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", - "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" - ], - "version": "==2.1.0" - }, "toml": { "hashes": [ "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", diff --git a/bot/bot.py b/bot/bot.py index 0d423201b..f1365d532 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -7,6 +7,7 @@ from typing import Optional import aiohttp import aioredis import discord +import fakeredis.aioredis from discord.ext import commands from sentry_sdk import push_scope @@ -48,11 +49,26 @@ class Bot(commands.Bot): self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") async def _create_redis_session(self) -> None: - """Create the Redis connection pool, and then open the redis event gate.""" - self.redis_session = await aioredis.create_redis_pool( - address=(constants.Redis.host, constants.Redis.port), - password=constants.Redis.password, - ) + """ + Create the Redis connection pool, and then open the redis event gate. + + If constants.Redis.use_fakeredis is True, we'll set up a fake redis pool instead + of attempting to communicate with a real Redis server. This is useful because it + means contributors don't necessarily need to get Redis running locally just + to run the bot. + + The fakeredis cache won't have persistence across restarts, but that + usually won't matter for local bot testing. + """ + if constants.Redis.use_fakeredis: + log.info("Using fakeredis instead of communicating with a real Redis server.") + self.redis_session = await fakeredis.aioredis.create_redis_pool() + else: + self.redis_session = await aioredis.create_redis_pool( + address=(constants.Redis.host, constants.Redis.port), + password=constants.Redis.password, + ) + self.redis_ready.set() def add_cog(self, cog: commands.Cog) -> None: diff --git a/bot/constants.py b/bot/constants.py index 5d854dd7a..75d394b6a 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -208,6 +208,7 @@ class Redis(metaclass=YAMLGetter): host: str port: int password: str + use_fakeredis: bool # If this is True, Bot will use fakeredis.aioredis class Filter(metaclass=YAMLGetter): diff --git a/config-default.yml b/config-default.yml index 5be393463..cee955f20 100644 --- a/config-default.yml +++ b/config-default.yml @@ -7,6 +7,7 @@ bot: host: "redis" port: 6379 password: !ENV "REDIS_PASSWORD" + use_fakeredis: false stats: statsd_host: "graphite" -- cgit v1.2.3 From ad8b1fa455e141074daec5047682e82ed96db1f5 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 19:09:45 +0200 Subject: Improve error and error testing for increment Changed a RuntimeError to a KeyError (thanks @MarkKoz), and also added some tests to ensure that the right errors are raised whenever this method is used incorrectly. --- bot/utils/redis_cache.py | 2 +- tests/bot/utils/test_redis_cache.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 5fc34d464..b91d663f3 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -341,7 +341,7 @@ class RedisCache: # Can't increment a non-existing value if value is None: log.exception("Attempt to increment/decrement value for non-existent key.") - raise RuntimeError("The provided key does not exist!") + raise KeyError("The provided key does not exist!") # If it does exist, and it's an int or a float, increment and set it. if isinstance(value, int) or isinstance(value, float): diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index dbbaef018..7405487ed 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -207,3 +207,11 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): # This should convert the type into a float. await self.redis.decrement("entropic", -2.5) self.assertEqual(await self.redis.get("entropic"), 7.5) + + # Let's test that they raise the right errors + with self.assertRaises(KeyError): + await self.redis.increment("doesn't_exist!") + + await self.redis.set("stringthing", "stringthing") + with self.assertRaises(TypeError): + await self.redis.increment("stringthing") -- cgit v1.2.3 From 185c9e84b5fde13ab21de614564eee94963d05b5 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Sun, 24 May 2020 22:46:48 +0100 Subject: Add discord.gift to URL blacklist, closes #958 --- config-default.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config-default.yml b/config-default.yml index 2e98186f1..9a28be700 100644 --- a/config-default.yml +++ b/config-default.yml @@ -318,6 +318,7 @@ filter: - poweredbysecurity.online - ssteam.site - steamwalletgift.com + - discord.gift word_watchlist: - goo+ks* -- cgit v1.2.3 From 856cecbd2354d4cbdbace5a39b7eb9e3d3bf23c7 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 24 May 2020 19:29:13 -0700 Subject: Add support for Union type annotations for constants Note that `Optional[x]` is just an alias for `Union[None, x]` so this effectively supports `Optional` too. This was especially troublesome because the redis password must be unset/None in order to avoid authentication, but the test would complain that `None` isn't a `str`. Setting to an empty string would pass the test but then make redis authenticate and fail. --- bot/constants.py | 14 +++++++------- tests/bot/test_constants.py | 17 ++++++++++++----- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/bot/constants.py b/bot/constants.py index 75d394b6a..145ae54db 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -15,7 +15,7 @@ import os from collections.abc import Mapping from enum import Enum from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional import yaml @@ -198,7 +198,7 @@ class Bot(metaclass=YAMLGetter): prefix: str token: str - sentry_dsn: str + sentry_dsn: Optional[str] class Redis(metaclass=YAMLGetter): @@ -207,7 +207,7 @@ class Redis(metaclass=YAMLGetter): host: str port: int - password: str + password: Optional[str] use_fakeredis: bool # If this is True, Bot will use fakeredis.aioredis @@ -459,7 +459,7 @@ class Guild(metaclass=YAMLGetter): class Keys(metaclass=YAMLGetter): section = "keys" - site_api: str + site_api: Optional[str] class URLs(metaclass=YAMLGetter): @@ -502,8 +502,8 @@ class Reddit(metaclass=YAMLGetter): section = "reddit" subreddits: list - client_id: str - secret: str + client_id: Optional[str] + secret: Optional[str] class Wolfram(metaclass=YAMLGetter): @@ -511,7 +511,7 @@ class Wolfram(metaclass=YAMLGetter): user_limit_day: int guild_limit_day: int - key: str + key: Optional[str] class AntiSpam(metaclass=YAMLGetter): diff --git a/tests/bot/test_constants.py b/tests/bot/test_constants.py index dae7c066c..db9a9bcb0 100644 --- a/tests/bot/test_constants.py +++ b/tests/bot/test_constants.py @@ -1,4 +1,5 @@ import inspect +import typing import unittest from bot import constants @@ -8,7 +9,7 @@ class ConstantsTests(unittest.TestCase): """Tests for our constants.""" def test_section_configuration_matches_type_specification(self): - """The section annotations should match the actual types of the sections.""" + """"The section annotations should match the actual types of the sections.""" sections = ( cls @@ -19,8 +20,14 @@ class ConstantsTests(unittest.TestCase): for name, annotation in section.__annotations__.items(): with self.subTest(section=section, name=name, annotation=annotation): value = getattr(section, name) + annotation_args = typing.get_args(annotation) - if getattr(annotation, '_name', None) in ('Dict', 'List'): - self.skipTest("Cannot validate containers yet.") - - self.assertIsInstance(value, annotation) + if not annotation_args: + self.assertIsInstance(value, annotation) + else: + origin = typing.get_origin(annotation) + if origin is typing.Union: + is_instance = any(isinstance(value, arg) for arg in annotation_args) + self.assertTrue(is_instance) + else: + self.skipTest(f"Validating type {annotation} is unsupported.") -- cgit v1.2.3 From 0ede719d7beb36f476ac26f948ab940882978476 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 25 May 2020 20:44:35 +0200 Subject: AntiMalware tests - Switched from monkeypatch to unittest.patch --- tests/bot/cogs/test_antimalware.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index fab063201..f219fc1ba 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,5 +1,5 @@ import unittest -from unittest.mock import AsyncMock, Mock +from unittest.mock import AsyncMock, Mock, patch from discord import NotFound @@ -10,6 +10,7 @@ from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole MODULE = "bot.cogs.antimalware" +@patch(f"{MODULE}.AntiMalwareConfig.whitelist", new=[".first", ".second", ".third"]) class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): """Test the AntiMalware cog.""" @@ -18,7 +19,6 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.bot = MockBot() self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() - AntiMalwareConfig.whitelist = [".first", ".second", ".third"] async def test_message_with_allowed_attachment(self): """Messages with allowed extensions should not be deleted""" -- cgit v1.2.3 From 9b9aa9b2adbdcd0e0b8c4f4ad38f112a9566fa2f Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 12:03:09 -0700 Subject: Support validating collection types for constants This is a simple validation that only check the type of the collection. It does not validate the types inside the collection because that has proven to be quite complex. --- tests/bot/test_constants.py | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/tests/bot/test_constants.py b/tests/bot/test_constants.py index db9a9bcb0..2937b6189 100644 --- a/tests/bot/test_constants.py +++ b/tests/bot/test_constants.py @@ -5,6 +5,31 @@ import unittest from bot import constants +def is_annotation_instance(value: typing.Any, annotation: typing.Any) -> bool: + """ + Return True if `value` is an instance of the type represented by `annotation`. + + This doesn't account for things like Unions or checking for homogenous types in collections. + """ + origin = typing.get_origin(annotation) + + # This is done in case a bare e.g. `typing.List` is used. + # In such case, for the assertion to pass, the type needs to be normalised to e.g. `list`. + # `get_origin()` does this normalisation for us. + type_ = annotation if origin is None else origin + + return isinstance(value, type_) + + +def is_any_instance(value: typing.Any, types: typing.Collection) -> bool: + """Return True if `value` is an instance of any type in `types`.""" + for type_ in types: + if is_annotation_instance(value, type_): + return True + + return False + + class ConstantsTests(unittest.TestCase): """Tests for our constants.""" @@ -20,14 +45,13 @@ class ConstantsTests(unittest.TestCase): for name, annotation in section.__annotations__.items(): with self.subTest(section=section, name=name, annotation=annotation): value = getattr(section, name) + origin = typing.get_origin(annotation) annotation_args = typing.get_args(annotation) + failure_msg = f"{value} is not an instance of {annotation}" - if not annotation_args: - self.assertIsInstance(value, annotation) + if origin is typing.Union: + is_instance = is_any_instance(value, annotation_args) + self.assertTrue(is_instance, failure_msg) else: - origin = typing.get_origin(annotation) - if origin is typing.Union: - is_instance = any(isinstance(value, arg) for arg in annotation_args) - self.assertTrue(is_instance) - else: - self.skipTest(f"Validating type {annotation} is unsupported.") + is_instance = is_annotation_instance(value, annotation) + self.assertTrue(is_instance, failure_msg) -- cgit v1.2.3 From 87d42add019e8ef1bad5d9593f6ed5a803e4d153 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 12:04:50 -0700 Subject: Improve output of section name in config validation subtests --- tests/bot/test_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/test_constants.py b/tests/bot/test_constants.py index 2937b6189..f10d6fbe8 100644 --- a/tests/bot/test_constants.py +++ b/tests/bot/test_constants.py @@ -43,7 +43,7 @@ class ConstantsTests(unittest.TestCase): ) for section in sections: for name, annotation in section.__annotations__.items(): - with self.subTest(section=section, name=name, annotation=annotation): + with self.subTest(section=section.__name__, name=name, annotation=annotation): value = getattr(section, name) origin = typing.get_origin(annotation) annotation_args = typing.get_args(annotation) -- cgit v1.2.3 From 8b5c1aabf58eb3c794cc61173bd7500a696a8376 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 12:12:24 -0700 Subject: Expose the redis port to the host Useful for those that run redis with docker-compose but not the bot. The bot on the host won't have access to the Docker network in such case so the port must be exposed. --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 1bcf1008e..9884e35f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,8 @@ services: redis: image: redis:5.0.9 + ports: + - "127.0.0.1:6379:6379" web: image: pythondiscord/site:latest -- cgit v1.2.3 From 6cedfdc0b24ea44b86fca039c9d7335072abede6 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Tue, 26 May 2020 02:21:58 +0100 Subject: [stats] Do not report modmail channels to stats --- bot/cogs/stats.py | 8 +++++++- bot/constants.py | 2 ++ config-default.yml | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py index 9baf222e2..14409ecb0 100644 --- a/bot/cogs/stats.py +++ b/bot/cogs/stats.py @@ -6,7 +6,7 @@ from discord.ext.commands import Cog, Context from discord.ext.tasks import loop from bot.bot import Bot -from bot.constants import Channels, Guild, Stats as StatConf +from bot.constants import Categories, Channels, Guild, Stats as StatConf CHANNEL_NAME_OVERRIDES = { @@ -36,6 +36,12 @@ class Stats(Cog): if message.guild.id != Guild.id: return + if message.channel.category.id == Categories.modmail: + if message.channel.id != Channels.incidents: + # Do not report modmail channels to stats, there are too many + # of them for interesting statistics to be drawn out of this. + return + reformatted_name = message.channel.name.replace('-', '_') if CHANNEL_NAME_OVERRIDES.get(message.channel.id): diff --git a/bot/constants.py b/bot/constants.py index 3003c9d36..39de2ee41 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -365,6 +365,7 @@ class Categories(metaclass=YAMLGetter): help_available: int help_in_use: int help_dormant: int + modmail: int class Channels(metaclass=YAMLGetter): @@ -384,6 +385,7 @@ class Channels(metaclass=YAMLGetter): esoteric: int helpers: int how_to_get_help: int + incidents: int message_log: int meta: int mod_alerts: int diff --git a/config-default.yml b/config-default.yml index 9a28be700..c7d25894c 100644 --- a/config-default.yml +++ b/config-default.yml @@ -118,6 +118,7 @@ guild: help_available: 691405807388196926 help_in_use: 696958401460043776 help_dormant: 691405908919451718 + modmail: 714494672835444826 channels: announcements: 354619224620138496 @@ -164,6 +165,7 @@ guild: mod_spam: &MOD_SPAM 620607373828030464 organisation: &ORGANISATION 551789653284356126 staff_lounge: &STAFF_LOUNGE 464905259261755392 + incidents: 714214212200562749 # Voice admins_voice: &ADMINS_VOICE 500734494840717332 -- cgit v1.2.3 From 9b882c768344cc866d366dc595fbfc19bc2cb6de Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:38:00 +0200 Subject: Turn log.exception into log.error Also, refactor error messages to be consistent and DRY throughout the file. --- bot/utils/redis_cache.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index b91d663f3..da78f1431 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -130,21 +130,24 @@ class RedisCache: async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" if self.bot is None: - log.exception("Attempt to use RedisCache with no `Bot` instance.") - raise RuntimeError( + error_message = ( "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " "have a Bot instance. Please make sure that you're instantiating " "the RedisCache inside a class that has a Bot instance " "class attribute." ) + log.error(error_message) + raise RuntimeError(error_message) if self._namespace is None: - log.exception("Attempt to use RedisCache with no namespace.") - raise RuntimeError( + error_message = ( "Critical error: RedisCache has no namespace. " "Did you initialize this object as a class attribute?" ) + log.error(error_message) + raise RuntimeError(error_message) + await self.bot.redis_ready.wait() def __set_name__(self, owner: Any, attribute_name: str) -> None: @@ -176,18 +179,17 @@ class RedisCache: return self if self._namespace is None: - log.exception("RedisCache must be a class attribute.") - raise RuntimeError("RedisCache must be a class attribute.") + error_message = "RedisCache must be a class attribute." + log.error(error_message) + raise RuntimeError(error_message) if instance is None: - log.exception( - "Attempt to access RedisCache instance through the cog's class object " - "before accessing it through the cog instance." - ) - raise RuntimeError( + error_message = ( "You must access the RedisCache instance through the cog instance " "before accessing it using the cog's class object." ) + log.error(error_message) + raise RuntimeError(error_message) for attribute in vars(instance).values(): if isinstance(attribute, Bot): @@ -195,14 +197,15 @@ class RedisCache: self._redis = self.bot.redis_session return self else: - log.exception("Attempt to use RedisCache with no `Bot` instance.") - raise RuntimeError( + error_message = ( "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " "have a Bot instance. Please make sure that you're instantiating " "the RedisCache inside a class that has a Bot instance " "class attribute." ) + log.error(error_message) + raise RuntimeError(error_message) def __repr__(self) -> str: """Return a beautiful representation of this object instance.""" @@ -340,16 +343,18 @@ class RedisCache: # Can't increment a non-existing value if value is None: - log.exception("Attempt to increment/decrement value for non-existent key.") - raise KeyError("The provided key does not exist!") + error_message = "The provided key does not exist!" + log.error(error_message) + raise KeyError(error_message) # If it does exist, and it's an int or a float, increment and set it. if isinstance(value, int) or isinstance(value, float): value += amount await self.set(key, value) else: - log.exception("Attempt to increment/decrement non-numerical value.") - raise TypeError("You may only increment or decrement values that are integers or floats.") + error_message = "You may only increment or decrement values that are integers or floats." + log.error(error_message) + raise TypeError(error_message) async def decrement(self, key: RedisType, amount: Optional[int, float] = 1) -> None: """ -- cgit v1.2.3 From 723c1d3337b0a59401f1d3fc50a123f0314a5d3e Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:47:16 +0200 Subject: Fix edge case where pop might not delete. If you passed a key for a value that was the same as your optional, it would just return it but not delete it. This edge case isn't worth it, so I'm just removing that condition and letting the extra API call fly. --- bot/utils/redis_cache.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index da78f1431..dd20b5842 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -222,7 +222,7 @@ class RedisCache: log.trace(f"Setting {key} to {value}.") await self._redis.hset(self._namespace, key, value) - async def get(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: + async def get(self, key: RedisType, default: Optional[RedisType] = None) -> Optional[RedisType]: """Get an item from the Redis cache.""" await self._validate_cache() key = self._to_typestring(key) @@ -311,11 +311,11 @@ class RedisCache: log.trace(f"Attempting to pop {key}.") value = await self.get(key, default) - # No need to try to delete something that doesn't exist, - # that's just a superfluous API call. - if value != default: - log.trace(f"Key {key} exists, deleting it from the cache.") - await self.delete(key) + log.trace( + f"Attempting to delete item with key '{key}' from the cache. " + "If this key doesn't exist, nothing will happen." + ) + await self.delete(key) return value -- cgit v1.2.3 From b630aceb1adb624f45465aeb698e844f9ea340c8 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:51:53 +0200 Subject: Add better docstring for RedisCache.update --- bot/utils/redis_cache.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index dd20b5842..b77ec47a2 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -319,8 +319,18 @@ class RedisCache: return value - async def update(self, items: Dict) -> None: - """Update the Redis cache with multiple values.""" + async def update(self, items: Dict[RedisType, RedisType]) -> None: + """ + Update the Redis cache with multiple values. + + This works exactly like dict.update from a normal dictionary. You pass + a dictionary with one or more key/value pairs into this method. If the keys + do not exist in the RedisCache, they are created. If they do exist, the values + are updated with the new ones from `items`. + + Please note that both the keys and the values in the `items` dictionary + must consist of valid RedisTypes - ints, floats, or strings. + """ await self._validate_cache() log.trace(f"Updating the cache with the following items:\n{items}") await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) -- cgit v1.2.3 From 19206734df651c7d65e5114715db6db9253cb7d6 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:52:52 +0200 Subject: Make self.increment_lock private. --- bot/utils/redis_cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index b77ec47a2..89af225e2 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -81,7 +81,7 @@ class RedisCache: """Initialize the RedisCache.""" self._namespace = None self.bot = None - self.increment_lock = asyncio.Lock() + self._increment_lock = asyncio.Lock() def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" @@ -348,7 +348,7 @@ class RedisCache: log.trace(f"Attempting to increment/decrement the value with the key {key} by {amount}.") # Since this has several API calls, we need a lock to prevent race conditions - async with self.increment_lock: + async with self._increment_lock: value = await self.get(key) # Can't increment a non-existing value -- cgit v1.2.3 From 46a377deef15545d1b860e283d8d0f8291298cee Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:55:12 +0200 Subject: Improve some docstrings for RedisCache. Thanks @MarkKoz! --- bot/utils/redis_cache.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 89af225e2..a1196fcb5 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -134,8 +134,7 @@ class RedisCache: "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " "have a Bot instance. Please make sure that you're instantiating " - "the RedisCache inside a class that has a Bot instance " - "class attribute." + "the RedisCache inside a class that has a Bot instance attribute." ) log.error(error_message) raise RuntimeError(error_message) @@ -143,7 +142,7 @@ class RedisCache: if self._namespace is None: error_message = ( "Critical error: RedisCache has no namespace. " - "Did you initialize this object as a class attribute?" + "This object must be initialized as a class attribute." ) log.error(error_message) raise RuntimeError(error_message) @@ -201,8 +200,7 @@ class RedisCache: "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " "have a Bot instance. Please make sure that you're instantiating " - "the RedisCache inside a class that has a Bot instance " - "class attribute." + "the RedisCache inside a class that has a Bot instance attribute." ) log.error(error_message) raise RuntimeError(error_message) -- cgit v1.2.3 From ec8205cfd7adb5e40aabd52e497e4e387b932211 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:56:30 +0200 Subject: Swap the order for the validate_cache checks. --- bot/utils/redis_cache.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index a1196fcb5..895a12da4 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -129,20 +129,20 @@ class RedisCache: async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" - if self.bot is None: + if self._namespace is None: error_message = ( - "Critical error: RedisCache has no `Bot` instance. " - "This happens when the class RedisCache was created in doesn't " - "have a Bot instance. Please make sure that you're instantiating " - "the RedisCache inside a class that has a Bot instance attribute." + "Critical error: RedisCache has no namespace. " + "This object must be initialized as a class attribute." ) log.error(error_message) raise RuntimeError(error_message) - if self._namespace is None: + if self.bot is None: error_message = ( - "Critical error: RedisCache has no namespace. " - "This object must be initialized as a class attribute." + "Critical error: RedisCache has no `Bot` instance. " + "This happens when the class RedisCache was created in doesn't " + "have a Bot instance. Please make sure that you're instantiating " + "the RedisCache inside a class that has a Bot instance attribute." ) log.error(error_message) raise RuntimeError(error_message) -- cgit v1.2.3 From 1ab34dd48fce2de70db1fb2dd6da06f752460829 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 19:06:57 +0200 Subject: Add a test for RuntimeErrors. This just tests that the various RuntimeErrors are reachable - that includes the error about not having a bot instance, the one about not being a class attribute, and the one about not having instantiated the class. This test addresses a concern raised by @MarkKoz in a review. I've decided not to test that actual contents of these RuntimeErrors, because I believe that sort of testing is a bit too brittle. It shouldn't break a test just to change the content of an error string. --- tests/bot/utils/test_redis_cache.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 7405487ed..1b05ae350 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -215,3 +215,25 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): await self.redis.set("stringthing", "stringthing") with self.assertRaises(TypeError): await self.redis.increment("stringthing") + + async def test_exceptions_raised(self): + """Testing that the various RuntimeErrors are reachable.""" + class MyCog: + cache = RedisCache() + + def __init__(self): + self.other_cache = RedisCache() + + cog = MyCog() + + # Raises "No Bot instance" + with self.assertRaises(RuntimeError): + await cog.cache.get("john") + + # Raises "RedisCache has no namespace" + with self.assertRaises(RuntimeError): + await cog.other_cache.get("was") + + # Raises "You must access the RedisCache instance through the cog instance" + with self.assertRaises(RuntimeError): + await MyCog.cache.get("afraid") -- cgit v1.2.3 From 63a922e629af76692c4af3902a33942b13f784b6 Mon Sep 17 00:00:00 2001 From: Dennis Pham Date: Tue, 26 May 2020 17:11:49 -0400 Subject: Add /r/FlutterDev to the guild invite whitelist --- config-default.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config-default.yml b/config-default.yml index c7d25894c..7edfb131f 100644 --- a/config-default.yml +++ b/config-default.yml @@ -290,6 +290,7 @@ filter: - 81384788765712384 # Discord API - 613425648685547541 # Discord Developers - 185590609631903755 # Blender Hub + - 420324994703163402 # /r/FlutterDev domain_blacklist: - pornhub.com -- cgit v1.2.3 From d9190d997538f49c0a1b53d63a15bada3c89297f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 07:32:16 +0200 Subject: Refactor the in_whitelist deco to a check. We're moving the actual predicate into the `utils.checks` folder, just like we're doing with most of the other decorators. This is to allow us the flexibility to use it as a pure check, not only as a decorator. This commit doesn't actually change any functionality, just moves it around. --- bot/decorators.py | 54 +++-------------------------- bot/utils/checks.py | 81 ++++++++++++++++++++++++++++++++++++++++++-- tests/bot/test_decorators.py | 4 +-- 3 files changed, 86 insertions(+), 53 deletions(-) diff --git a/bot/decorators.py b/bot/decorators.py index 306f0830c..1e77afe60 100644 --- a/bot/decorators.py +++ b/bot/decorators.py @@ -9,37 +9,20 @@ from weakref import WeakValueDictionary from discord import Colour, Embed, Member from discord.errors import NotFound from discord.ext import commands -from discord.ext.commands import CheckFailure, Cog, Context +from discord.ext.commands import Cog, Context from bot.constants import Channels, ERROR_REPLIES, RedirectOutput -from bot.utils.checks import with_role_check, without_role_check +from bot.utils.checks import in_whitelist_check, with_role_check, without_role_check log = logging.getLogger(__name__) -class InWhitelistCheckFailure(CheckFailure): - """Raised when the `in_whitelist` check fails.""" - - def __init__(self, redirect_channel: Optional[int]) -> None: - self.redirect_channel = redirect_channel - - if redirect_channel: - redirect_message = f" here. Please use the <#{redirect_channel}> channel instead" - else: - redirect_message = "" - - error_message = f"You are not allowed to use that command{redirect_message}." - - super().__init__(error_message) - - def in_whitelist( *, channels: Container[int] = (), categories: Container[int] = (), roles: Container[int] = (), redirect: Optional[int] = Channels.bot_commands, - ) -> Callable: """ Check if a command was issued in a whitelisted context. @@ -54,36 +37,9 @@ def in_whitelist( redirected to the `redirect` channel that was passed (default: #bot-commands) or simply told that they're not allowed to use this particular command (if `None` was passed). """ - if redirect and redirect not in channels: - # It does not make sense for the channel whitelist to not contain the redirection - # channel (if applicable). That's why we add the redirection channel to the `channels` - # container if it's not already in it. As we allow any container type to be passed, - # we first create a tuple in order to safely add the redirection channel. - # - # Note: It's possible for the redirect channel to be in a whitelisted category, but - # there's no easy way to check that and as a channel can easily be moved in and out of - # categories, it's probably not wise to rely on its category in any case. - channels = tuple(channels) + (redirect,) - def predicate(ctx: Context) -> bool: - """Check if a command was issued in a whitelisted context.""" - if channels and ctx.channel.id in channels: - log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted channel.") - return True - - # Only check the category id if we have a category whitelist and the channel has a `category_id` - if categories and hasattr(ctx.channel, "category_id") and ctx.channel.category_id in categories: - log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted category.") - return True - - # Only check the roles whitelist if we have one and ensure the author's roles attribute returns - # an iterable to prevent breakage in DM channels (for if we ever decide to enable commands there). - if roles and any(r.id in roles for r in getattr(ctx.author, "roles", ())): - log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they have a whitelisted role.") - return True - - log.trace(f"{ctx.author} may not use the `{ctx.command.name}` command within this context.") - raise InWhitelistCheckFailure(redirect) + """Check if command was issued in a whitelisted context.""" + return in_whitelist_check(ctx, channels, categories, roles, redirect) return commands.check(predicate) @@ -121,7 +77,7 @@ def locked() -> Callable: embed = Embed() embed.colour = Colour.red() - log.debug(f"User tried to invoke a locked command.") + log.debug("User tried to invoke a locked command.") embed.description = ( "You're already using this command. Please wait until it is done before you use it again." ) diff --git a/bot/utils/checks.py b/bot/utils/checks.py index db56c347c..63568b29e 100644 --- a/bot/utils/checks.py +++ b/bot/utils/checks.py @@ -1,12 +1,89 @@ import datetime import logging -from typing import Callable, Iterable +from typing import Callable, Container, Iterable, Optional -from discord.ext.commands import BucketType, Cog, Command, CommandOnCooldown, Context, Cooldown, CooldownMapping +from discord.ext.commands import ( + BucketType, + CheckFailure, + Cog, + Command, + CommandOnCooldown, + Context, + Cooldown, + CooldownMapping, +) + +from bot import constants log = logging.getLogger(__name__) +class InWhitelistCheckFailure(CheckFailure): + """Raised when the `in_whitelist` check fails.""" + + def __init__(self, redirect_channel: Optional[int]) -> None: + self.redirect_channel = redirect_channel + + if redirect_channel: + redirect_message = f" here. Please use the <#{redirect_channel}> channel instead" + else: + redirect_message = "" + + error_message = f"You are not allowed to use that command{redirect_message}." + + super().__init__(error_message) + + +def in_whitelist_check( + ctx: Context, + channels: Container[int] = (), + categories: Container[int] = (), + roles: Container[int] = (), + redirect: Optional[int] = constants.Channels.bot_commands, +) -> bool: + """ + Check if a command was issued in a whitelisted context. + + The whitelists that can be provided are: + + - `channels`: a container with channel ids for whitelisted channels + - `categories`: a container with category ids for whitelisted categories + - `roles`: a container with with role ids for whitelisted roles + + If the command was invoked in a context that was not whitelisted, the member is either + redirected to the `redirect` channel that was passed (default: #bot-commands) or simply + told that they're not allowed to use this particular command (if `None` was passed). + """ + if redirect and redirect not in channels: + # It does not make sense for the channel whitelist to not contain the redirection + # channel (if applicable). That's why we add the redirection channel to the `channels` + # container if it's not already in it. As we allow any container type to be passed, + # we first create a tuple in order to safely add the redirection channel. + # + # Note: It's possible for the redirect channel to be in a whitelisted category, but + # there's no easy way to check that and as a channel can easily be moved in and out of + # categories, it's probably not wise to rely on its category in any case. + channels = tuple(channels) + (redirect,) + + if channels and ctx.channel.id in channels: + log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted channel.") + return True + + # Only check the category id if we have a category whitelist and the channel has a `category_id` + if categories and hasattr(ctx.channel, "category_id") and ctx.channel.category_id in categories: + log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted category.") + return True + + # Only check the roles whitelist if we have one and ensure the author's roles attribute returns + # an iterable to prevent breakage in DM channels (for if we ever decide to enable commands there). + if roles and any(r.id in roles for r in getattr(ctx.author, "roles", ())): + log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they have a whitelisted role.") + return True + + log.trace(f"{ctx.author} may not use the `{ctx.command.name}` command within this context.") + raise InWhitelistCheckFailure(redirect) + + def with_role_check(ctx: Context, *role_ids: int) -> bool: """Returns True if the user has any one of the roles in role_ids.""" if not ctx.guild: # Return False in a DM diff --git a/tests/bot/test_decorators.py b/tests/bot/test_decorators.py index a17dd3e16..3d450caa0 100644 --- a/tests/bot/test_decorators.py +++ b/tests/bot/test_decorators.py @@ -3,10 +3,10 @@ import unittest import unittest.mock from bot import constants -from bot.decorators import InWhitelistCheckFailure, in_whitelist +from bot.decorators import in_whitelist +from bot.utils.checks import InWhitelistCheckFailure from tests import helpers - InWhitelistTestCase = collections.namedtuple("WhitelistedContextTestCase", ("kwargs", "ctx", "description")) -- cgit v1.2.3 From b51c206e51d0906f326da1e504162920cd2d443d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 07:35:56 +0200 Subject: Allow infraction management in modmail category --- bot/cogs/moderation/management.py | 20 ++++++++++++-------- bot/constants.py | 5 +---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index edfdfd9e2..56f7c390c 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -12,7 +12,7 @@ from bot.bot import Bot from bot.converters import Expiry, InfractionSearchQuery, allowed_strings, proxy_user from bot.pagination import LinePaginator from bot.utils import time -from bot.utils.checks import in_channel_check, with_role_check +from bot.utils.checks import in_whitelist_check, with_role_check from . import utils from .infractions import Infractions from .modlog import ModLog @@ -49,8 +49,8 @@ class ModManagement(commands.Cog): async def infraction_edit( self, ctx: Context, - infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], - duration: t.Union[Expiry, allowed_strings("p", "permanent"), None], + infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], # noqa: F821 + duration: t.Union[Expiry, allowed_strings("p", "permanent"), None], # noqa: F821 *, reason: str = None ) -> None: @@ -83,14 +83,14 @@ class ModManagement(commands.Cog): "actor__id": ctx.author.id, "ordering": "-inserted_at" } - infractions = await self.bot.api_client.get(f"bot/infractions", params=params) + infractions = await self.bot.api_client.get("bot/infractions", params=params) if infractions: old_infraction = infractions[0] infraction_id = old_infraction["id"] else: await ctx.send( - f":x: Couldn't find most recent infraction; you have never given an infraction." + ":x: Couldn't find most recent infraction; you have never given an infraction." ) return else: @@ -224,7 +224,7 @@ class ModManagement(commands.Cog): ) -> None: """Send a paginated embed of infractions for the specified user.""" if not infractions: - await ctx.send(f":warning: No infractions could be found for that query.") + await ctx.send(":warning: No infractions could be found for that query.") return lines = tuple( @@ -283,10 +283,14 @@ class ModManagement(commands.Cog): # This cannot be static (must have a __func__ attribute). def cog_check(self, ctx: Context) -> bool: - """Only allow moderators from moderator channels to invoke the commands in this cog.""" + """Only allow moderators inside moderator channels to invoke the commands in this cog.""" checks = [ with_role_check(ctx, *constants.MODERATION_ROLES), - in_channel_check(ctx, *constants.MODERATION_CHANNELS) + in_whitelist_check( + ctx, + channels=constants.MODERATION_CHANNELS, + categories=[constants.Categories.modmail], + ) ] return all(checks) diff --git a/bot/constants.py b/bot/constants.py index 39de2ee41..2ce5355be 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -612,13 +612,10 @@ PROJECT_ROOT = os.path.abspath(os.path.join(BOT_DIR, os.pardir)) MODERATION_ROLES = Guild.moderation_roles STAFF_ROLES = Guild.staff_roles -# Roles combinations +# Channel combinations STAFF_CHANNELS = Guild.staff_channels - -# Default Channel combinations MODERATION_CHANNELS = Guild.moderation_channels - # Bot replies NEGATIVE_REPLIES = [ "Noooooo!!", -- cgit v1.2.3 From d310f42080278b35914bf5785fa322b97627c45f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 07:42:08 +0200 Subject: Find + change all InWhitelistCheckFailure imports --- bot/cogs/error_handler.py | 6 +++--- bot/cogs/information.py | 4 ++-- bot/cogs/verification.py | 4 ++-- tests/bot/cogs/test_information.py | 3 +-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index 23d1eed82..5de961116 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -9,7 +9,7 @@ from bot.api import ResponseCodeError from bot.bot import Bot from bot.constants import Channels from bot.converters import TagNameConverter -from bot.decorators import InWhitelistCheckFailure +from bot.utils.checks import InWhitelistCheckFailure log = logging.getLogger(__name__) @@ -166,7 +166,7 @@ class ErrorHandler(Cog): await prepared_help_command self.bot.stats.incr("errors.missing_required_argument") elif isinstance(e, errors.TooManyArguments): - await ctx.send(f"Too many arguments provided.") + await ctx.send("Too many arguments provided.") await prepared_help_command self.bot.stats.incr("errors.too_many_arguments") elif isinstance(e, errors.BadArgument): @@ -206,7 +206,7 @@ class ErrorHandler(Cog): if isinstance(e, bot_missing_errors): ctx.bot.stats.incr("errors.bot_permission_error") await ctx.send( - f"Sorry, it looks like I don't have the permissions or roles I need to do that." + "Sorry, it looks like I don't have the permissions or roles I need to do that." ) elif isinstance(e, (InWhitelistCheckFailure, errors.NoPrivateMessage)): ctx.bot.stats.incr("errors.wrong_channel_or_dm_error") diff --git a/bot/cogs/information.py b/bot/cogs/information.py index ef2f308ca..f0eb3a1ea 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -12,9 +12,9 @@ from discord.utils import escape_markdown from bot import constants from bot.bot import Bot -from bot.decorators import InWhitelistCheckFailure, in_whitelist, with_role +from bot.decorators import in_whitelist, with_role from bot.pagination import LinePaginator -from bot.utils.checks import cooldown_with_role_bypass, with_role_check +from bot.utils.checks import InWhitelistCheckFailure, cooldown_with_role_bypass, with_role_check from bot.utils.time import time_since log = logging.getLogger(__name__) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 77e8b5706..99be3cdaa 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -9,8 +9,8 @@ from discord.ext.commands import Cog, Context, command from bot import constants from bot.bot import Bot from bot.cogs.moderation import ModLog -from bot.decorators import InWhitelistCheckFailure, in_whitelist, without_role -from bot.utils.checks import without_role_check +from bot.decorators import in_whitelist, without_role +from bot.utils.checks import InWhitelistCheckFailure, without_role_check log = logging.getLogger(__name__) diff --git a/tests/bot/cogs/test_information.py b/tests/bot/cogs/test_information.py index b5f928dd6..aca6b594f 100644 --- a/tests/bot/cogs/test_information.py +++ b/tests/bot/cogs/test_information.py @@ -7,10 +7,9 @@ import discord from bot import constants from bot.cogs import information -from bot.decorators import InWhitelistCheckFailure +from bot.utils.checks import InWhitelistCheckFailure from tests import helpers - COG_PATH = "bot.cogs.information.Information" -- cgit v1.2.3 From 75622622696beee8299c24e9ddbc36f5eb4f104f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 07:46:04 +0200 Subject: No redirect for mod management. --- bot/cogs/moderation/management.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index 56f7c390c..c7c19e89d 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -290,6 +290,7 @@ class ModManagement(commands.Cog): ctx, channels=constants.MODERATION_CHANNELS, categories=[constants.Categories.modmail], + redirect=None, ) ] return all(checks) -- cgit v1.2.3 From c3cbc842dce1c26f09d774b7ca85eff613765480 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 07:51:20 +0200 Subject: Allow some commands to fail checks silently. For example, we don't want the mod commands to produce any kind of error message when run by ordinary users in regular channels - these should have the perception of being invisible and unavailable. --- bot/cogs/moderation/management.py | 1 + bot/decorators.py | 3 ++- bot/utils/checks.py | 7 ++++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index c7c19e89d..c39c7f3bc 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -291,6 +291,7 @@ class ModManagement(commands.Cog): channels=constants.MODERATION_CHANNELS, categories=[constants.Categories.modmail], redirect=None, + fail_silently=True, ) ] return all(checks) diff --git a/bot/decorators.py b/bot/decorators.py index 1e77afe60..500197c89 100644 --- a/bot/decorators.py +++ b/bot/decorators.py @@ -23,6 +23,7 @@ def in_whitelist( categories: Container[int] = (), roles: Container[int] = (), redirect: Optional[int] = Channels.bot_commands, + fail_silently: bool = False, ) -> Callable: """ Check if a command was issued in a whitelisted context. @@ -39,7 +40,7 @@ def in_whitelist( """ def predicate(ctx: Context) -> bool: """Check if command was issued in a whitelisted context.""" - return in_whitelist_check(ctx, channels, categories, roles, redirect) + return in_whitelist_check(ctx, channels, categories, roles, redirect, fail_silently) return commands.check(predicate) diff --git a/bot/utils/checks.py b/bot/utils/checks.py index 63568b29e..d5ebe4ec9 100644 --- a/bot/utils/checks.py +++ b/bot/utils/checks.py @@ -40,6 +40,7 @@ def in_whitelist_check( categories: Container[int] = (), roles: Container[int] = (), redirect: Optional[int] = constants.Channels.bot_commands, + fail_silently: bool = False, ) -> bool: """ Check if a command was issued in a whitelisted context. @@ -81,7 +82,11 @@ def in_whitelist_check( return True log.trace(f"{ctx.author} may not use the `{ctx.command.name}` command within this context.") - raise InWhitelistCheckFailure(redirect) + + # Some commands are secret, and should produce no feedback at all. + if not fail_silently: + raise InWhitelistCheckFailure(redirect) + return False def with_role_check(ctx: Context, *role_ids: int) -> bool: -- cgit v1.2.3 From 72304495f43e91eb62bb47657bc3ce4858639939 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 09:29:14 +0200 Subject: Remove all sending of avatar_hash. This is a companion commit to this PR: https://github.com/python-discord/site/pull/356 This PR must be merged before this commit. --- bot/cogs/moderation/utils.py | 1 - bot/cogs/sync/cog.py | 4 +--- bot/cogs/sync/syncers.py | 3 +-- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py index e4e0f1ec2..c99847329 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/cogs/moderation/utils.py @@ -41,7 +41,6 @@ async def post_user(ctx: Context, user: UserSnowflake) -> t.Optional[dict]: log.debug("The user being added to the DB is not a Member or User object.") payload = { - 'avatar_hash': getattr(user, 'avatar', 0), 'discriminator': int(getattr(user, 'discriminator', 0)), 'id': user.id, 'in_guild': False, diff --git a/bot/cogs/sync/cog.py b/bot/cogs/sync/cog.py index 5708be3f4..7cc3726b2 100644 --- a/bot/cogs/sync/cog.py +++ b/bot/cogs/sync/cog.py @@ -94,7 +94,6 @@ class Sync(Cog): the database, the user is added. """ packed = { - 'avatar_hash': member.avatar, 'discriminator': int(member.discriminator), 'id': member.id, 'in_guild': True, @@ -135,12 +134,11 @@ class Sync(Cog): @Cog.listener() async def on_user_update(self, before: User, after: User) -> None: """Update the user information in the database if a relevant change is detected.""" - attrs = ("name", "discriminator", "avatar") + attrs = ("name", "discriminator") if any(getattr(before, attr) != getattr(after, attr) for attr in attrs): updated_information = { "name": after.name, "discriminator": int(after.discriminator), - "avatar_hash": after.avatar, } await self.patch_user(after.id, updated_information=updated_information) diff --git a/bot/cogs/sync/syncers.py b/bot/cogs/sync/syncers.py index e55bf27fd..536455668 100644 --- a/bot/cogs/sync/syncers.py +++ b/bot/cogs/sync/syncers.py @@ -17,7 +17,7 @@ log = logging.getLogger(__name__) # These objects are declared as namedtuples because tuples are hashable, # something that we make use of when diffing site roles against guild roles. _Role = namedtuple('Role', ('id', 'name', 'colour', 'permissions', 'position')) -_User = namedtuple('User', ('id', 'name', 'discriminator', 'avatar_hash', 'roles', 'in_guild')) +_User = namedtuple('User', ('id', 'name', 'discriminator', 'roles', 'in_guild')) _Diff = namedtuple('Diff', ('created', 'updated', 'deleted')) @@ -298,7 +298,6 @@ class UserSyncer(Syncer): id=member.id, name=member.name, discriminator=int(member.discriminator), - avatar_hash=member.avatar, roles=tuple(sorted(role.id for role in member.roles)), in_guild=True ) -- cgit v1.2.3 From 8e0cdb258ea6e0f25977d18336a2e07b20b5d1ee Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 09:42:57 +0200 Subject: Fix failing tests related to avatar_hash --- tests/bot/cogs/sync/test_cog.py | 3 --- tests/bot/cogs/sync/test_users.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/tests/bot/cogs/sync/test_cog.py b/tests/bot/cogs/sync/test_cog.py index 81398c61f..14fd909c4 100644 --- a/tests/bot/cogs/sync/test_cog.py +++ b/tests/bot/cogs/sync/test_cog.py @@ -247,14 +247,12 @@ class SyncCogListenerTests(SyncCogTestCase): before_data = { "name": "old name", "discriminator": "1234", - "avatar": "old avatar", "bot": False, } subtests = ( (True, "name", "name", "new name", "new name"), (True, "discriminator", "discriminator", "8765", 8765), - (True, "avatar", "avatar_hash", "9j2e9", "9j2e9"), (False, "bot", "bot", True, True), ) @@ -295,7 +293,6 @@ class SyncCogListenerTests(SyncCogTestCase): ) data = { - "avatar_hash": member.avatar, "discriminator": int(member.discriminator), "id": member.id, "in_guild": True, diff --git a/tests/bot/cogs/sync/test_users.py b/tests/bot/cogs/sync/test_users.py index 818883012..002a947ad 100644 --- a/tests/bot/cogs/sync/test_users.py +++ b/tests/bot/cogs/sync/test_users.py @@ -10,7 +10,6 @@ def fake_user(**kwargs): kwargs.setdefault("id", 43) kwargs.setdefault("name", "bob the test man") kwargs.setdefault("discriminator", 1337) - kwargs.setdefault("avatar_hash", None) kwargs.setdefault("roles", (666,)) kwargs.setdefault("in_guild", True) @@ -32,7 +31,6 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): for member in members: member = member.copy() - member["avatar"] = member.pop("avatar_hash") del member["in_guild"] mock_member = helpers.MockMember(**member) -- cgit v1.2.3 From 35a1de37307b1745c061e490be4e96c8467de212 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 12:21:58 +0200 Subject: Clear cache in asyncSetUp instead of tests. --- tests/bot/utils/test_redis_cache.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 1b05ae350..900a6d035 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -15,6 +15,7 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): """Sets up the objects that only have to be initialized once.""" self.bot = helpers.MockBot() self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() + await self.redis.clear() def test_class_attribute_namespace(self): """Test that RedisDict creates a namespace automatically for class attributes.""" @@ -76,8 +77,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_items(self): """Test that the RedisDict can be iterated.""" - await self.redis.clear() - # Set up our test cases in the Redis cache test_cases = [ ('favorite_turtle', 'Donatello'), @@ -101,7 +100,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_length(self): """Test that we can get the correct .length from the RedisDict.""" - await self.redis.clear() await self.redis.set('one', 1) await self.redis.set('two', 2) await self.redis.set('three', 3) @@ -119,7 +117,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_clear(self): """Test that the .clear method removes the entire hash.""" - await self.redis.clear() await self.redis.set('teddy', 'with me') await self.redis.set('in my dreams', 'you have a weird hat') self.assertEqual(await self.redis.length(), 2) @@ -129,7 +126,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_pop(self): """Test that we can .pop an item from the RedisDict.""" - await self.redis.clear() await self.redis.set('john', 'was afraid') self.assertEqual(await self.redis.pop('john'), 'was afraid') @@ -138,7 +134,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_update(self): """Test that we can .update the RedisDict with multiple items.""" - await self.redis.clear() await self.redis.set("reckfried", "lona") await self.redis.set("bel air", "prince") await self.redis.update({ -- cgit v1.2.3 From b18930735e05e09ba615cb54fe1dbdfd41bb0f81 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 12:51:40 +0200 Subject: Refactor .increment and add lock test. The way we were doing the asyncio.Lock() stuff for increment was slightly problematic. @aeros has adviced us that it's better to just initialize the lock as None in __init__, and then initialize it inside the first coroutine that uses it instead. This ensures that the correct loop gets attached to the lock, so we don't end up getting errors like this one: RuntimeError: got Future attached to a different loop This happens because the lock and the actual calling coroutines aren't on the same loop. When creating a new test, test_increment_lock, we discovered that we needed a small refactor here and also in the test class to make this new test pass. So, now we're creating a DummyCog for every test method, and this will ensure the loop streams never cross. Cause we all know we must never cross the streams. --- bot/utils/redis_cache.py | 11 ++- tests/bot/utils/test_redis_cache.py | 163 ++++++++++++++++++++++-------------- 2 files changed, 109 insertions(+), 65 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 895a12da4..33e5d5852 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -81,7 +81,7 @@ class RedisCache: """Initialize the RedisCache.""" self._namespace = None self.bot = None - self._increment_lock = asyncio.Lock() + self._increment_lock = None def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" @@ -345,6 +345,15 @@ class RedisCache: """ log.trace(f"Attempting to increment/decrement the value with the key {key} by {amount}.") + # We initialize the lock here, because we need to ensure we get it + # running on the same loop as the calling coroutine. + # + # If we initialized the lock in the __init__, the loop that the coroutine this method + # would be called from might not exist yet, and so the lock would be on a different + # loop, which would raise RuntimeErrors. + if self._increment_lock is None: + self._increment_lock = asyncio.Lock() + # Since this has several API calls, we need a lock to prevent race conditions async with self._increment_lock: value = await self.get(key) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 900a6d035..efd168dac 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -1,3 +1,4 @@ +import asyncio import unittest import fakeredis.aioredis @@ -9,17 +10,30 @@ from tests import helpers class RedisCacheTests(unittest.IsolatedAsyncioTestCase): """Tests the RedisCache class from utils.redis_dict.py.""" - redis = RedisCache() - async def asyncSetUp(self): # noqa: N802 """Sets up the objects that only have to be initialized once.""" self.bot = helpers.MockBot() self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() - await self.redis.clear() + + # Okay, so this is necessary so that we can create a clean new + # class for every test method, and we want that because it will + # ensure we get a fresh loop, which is necessary for test_increment_lock + # to be able to pass. + class DummyCog: + """A dummy cog, for dummies.""" + + redis = RedisCache() + + def __init__(self, bot: helpers.MockBot): + self.bot = bot + + self.cog = DummyCog(self.bot) + + await self.cog.redis.clear() def test_class_attribute_namespace(self): """Test that RedisDict creates a namespace automatically for class attributes.""" - self.assertEqual(self.redis._namespace, "RedisCacheTests.redis") + self.assertEqual(self.cog.redis._namespace, "DummyCog.redis") async def test_class_attribute_required(self): """Test that errors are raised when not assigned as a class attribute.""" @@ -31,9 +45,13 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): def test_namespace_collision(self): """Test that we prevent colliding namespaces.""" - bad_cache = RedisCache() - bad_cache._set_namespace("RedisCacheTests.redis") - self.assertEqual(bad_cache._namespace, "RedisCacheTests.redis_") + bob_cache_1 = RedisCache() + bob_cache_1._set_namespace("BobRoss") + self.assertEqual(bob_cache_1._namespace, "BobRoss") + + bob_cache_2 = RedisCache() + bob_cache_2._set_namespace("BobRoss") + self.assertEqual(bob_cache_2._namespace, "BobRoss_") async def test_set_get_item(self): """Test that users can set and get items from the RedisDict.""" @@ -45,35 +63,35 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): # Test that we can get and set different types. for test in test_cases: - await self.redis.set(*test) - self.assertEqual(await self.redis.get(test[0]), test[1]) + await self.cog.redis.set(*test) + self.assertEqual(await self.cog.redis.get(test[0]), test[1]) # Test that .get allows a default value - self.assertEqual(await self.redis.get('favorite_nothing', "bearclaw"), "bearclaw") + self.assertEqual(await self.cog.redis.get('favorite_nothing', "bearclaw"), "bearclaw") async def test_set_item_type(self): """Test that .set rejects keys and values that are not strings, ints or floats.""" fruits = ["lemon", "melon", "apple"] with self.assertRaises(TypeError): - await self.redis.set(fruits, "nice") + await self.cog.redis.set(fruits, "nice") async def test_delete_item(self): """Test that .delete allows us to delete stuff from the RedisCache.""" # Add an item and verify that it gets added - await self.redis.set("internet", "firetruck") - self.assertEqual(await self.redis.get("internet"), "firetruck") + await self.cog.redis.set("internet", "firetruck") + self.assertEqual(await self.cog.redis.get("internet"), "firetruck") # Delete that item and verify that it gets deleted - await self.redis.delete("internet") - self.assertIs(await self.redis.get("internet"), None) + await self.cog.redis.delete("internet") + self.assertIs(await self.cog.redis.get("internet"), None) async def test_contains(self): """Test that we can check membership with .contains.""" - await self.redis.set('favorite_country', "Burkina Faso") + await self.cog.redis.set('favorite_country', "Burkina Faso") - self.assertIs(await self.redis.contains('favorite_country'), True) - self.assertIs(await self.redis.contains('favorite_dentist'), False) + self.assertIs(await self.cog.redis.contains('favorite_country'), True) + self.assertIs(await self.cog.redis.contains('favorite_dentist'), False) async def test_items(self): """Test that the RedisDict can be iterated.""" @@ -84,10 +102,10 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): ('third_favorite_turtle', 'Raphael'), ] for key, value in test_cases: - await self.redis.set(key, value) + await self.cog.redis.set(key, value) # Consume the AsyncIterator into a regular list, easier to compare that way. - redis_items = [item for item in await self.redis.items()] + redis_items = [item for item in await self.cog.redis.items()] # These sequences are probably in the same order now, but probably # isn't good enough for tests. Let's not rely on .hgetall always @@ -100,43 +118,43 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_length(self): """Test that we can get the correct .length from the RedisDict.""" - await self.redis.set('one', 1) - await self.redis.set('two', 2) - await self.redis.set('three', 3) - self.assertEqual(await self.redis.length(), 3) + await self.cog.redis.set('one', 1) + await self.cog.redis.set('two', 2) + await self.cog.redis.set('three', 3) + self.assertEqual(await self.cog.redis.length(), 3) - await self.redis.set('four', 4) - self.assertEqual(await self.redis.length(), 4) + await self.cog.redis.set('four', 4) + self.assertEqual(await self.cog.redis.length(), 4) async def test_to_dict(self): """Test that the .to_dict method returns a workable dictionary copy.""" - copy = await self.redis.to_dict() - local_copy = {key: value for key, value in await self.redis.items()} + copy = await self.cog.redis.to_dict() + local_copy = {key: value for key, value in await self.cog.redis.items()} self.assertIs(type(copy), dict) self.assertDictEqual(copy, local_copy) async def test_clear(self): """Test that the .clear method removes the entire hash.""" - await self.redis.set('teddy', 'with me') - await self.redis.set('in my dreams', 'you have a weird hat') - self.assertEqual(await self.redis.length(), 2) + await self.cog.redis.set('teddy', 'with me') + await self.cog.redis.set('in my dreams', 'you have a weird hat') + self.assertEqual(await self.cog.redis.length(), 2) - await self.redis.clear() - self.assertEqual(await self.redis.length(), 0) + await self.cog.redis.clear() + self.assertEqual(await self.cog.redis.length(), 0) async def test_pop(self): """Test that we can .pop an item from the RedisDict.""" - await self.redis.set('john', 'was afraid') + await self.cog.redis.set('john', 'was afraid') - self.assertEqual(await self.redis.pop('john'), 'was afraid') - self.assertEqual(await self.redis.pop('pete', 'breakneck'), 'breakneck') - self.assertEqual(await self.redis.length(), 0) + self.assertEqual(await self.cog.redis.pop('john'), 'was afraid') + self.assertEqual(await self.cog.redis.pop('pete', 'breakneck'), 'breakneck') + self.assertEqual(await self.cog.redis.length(), 0) async def test_update(self): """Test that we can .update the RedisDict with multiple items.""" - await self.redis.set("reckfried", "lona") - await self.redis.set("bel air", "prince") - await self.redis.update({ + await self.cog.redis.set("reckfried", "lona") + await self.cog.redis.set("bel air", "prince") + await self.cog.redis.update({ "reckfried": "jona", "mega": "hungry, though", }) @@ -146,7 +164,7 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): "bel air": "prince", "mega": "hungry, though", } - self.assertDictEqual(await self.redis.to_dict(), result) + self.assertDictEqual(await self.cog.redis.to_dict(), result) def test_typestring_conversion(self): """Test the typestring-related helper functions.""" @@ -158,58 +176,75 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): # Test conversion to typestring for _input, expected in conversion_tests: - self.assertEqual(self.redis._to_typestring(_input), expected) + self.assertEqual(self.cog.redis._to_typestring(_input), expected) # Test conversion from typestrings for _input, expected in conversion_tests: - self.assertEqual(self.redis._from_typestring(expected), _input) + self.assertEqual(self.cog.redis._from_typestring(expected), _input) # Test that exceptions are raised on invalid input with self.assertRaises(TypeError): - self.redis._to_typestring(["internet"]) - self.redis._from_typestring("o|firedog") + self.cog.redis._to_typestring(["internet"]) + self.cog.redis._from_typestring("o|firedog") async def test_increment_decrement(self): """Test .increment and .decrement methods.""" - await self.redis.set("entropic", 5) - await self.redis.set("disentropic", 12.5) + await self.cog.redis.set("entropic", 5) + await self.cog.redis.set("disentropic", 12.5) # Test default increment - await self.redis.increment("entropic") - self.assertEqual(await self.redis.get("entropic"), 6) + await self.cog.redis.increment("entropic") + self.assertEqual(await self.cog.redis.get("entropic"), 6) # Test default decrement - await self.redis.decrement("entropic") - self.assertEqual(await self.redis.get("entropic"), 5) + await self.cog.redis.decrement("entropic") + self.assertEqual(await self.cog.redis.get("entropic"), 5) # Test float increment with float - await self.redis.increment("disentropic", 2.0) - self.assertEqual(await self.redis.get("disentropic"), 14.5) + await self.cog.redis.increment("disentropic", 2.0) + self.assertEqual(await self.cog.redis.get("disentropic"), 14.5) # Test float increment with int - await self.redis.increment("disentropic", 2) - self.assertEqual(await self.redis.get("disentropic"), 16.5) + await self.cog.redis.increment("disentropic", 2) + self.assertEqual(await self.cog.redis.get("disentropic"), 16.5) # Test negative increments, because why not. - await self.redis.increment("entropic", -5) - self.assertEqual(await self.redis.get("entropic"), 0) + await self.cog.redis.increment("entropic", -5) + self.assertEqual(await self.cog.redis.get("entropic"), 0) # Negative decrements? Sure. - await self.redis.decrement("entropic", -5) - self.assertEqual(await self.redis.get("entropic"), 5) + await self.cog.redis.decrement("entropic", -5) + self.assertEqual(await self.cog.redis.get("entropic"), 5) # What about if we use a negative float to decrement an int? # This should convert the type into a float. - await self.redis.decrement("entropic", -2.5) - self.assertEqual(await self.redis.get("entropic"), 7.5) + await self.cog.redis.decrement("entropic", -2.5) + self.assertEqual(await self.cog.redis.get("entropic"), 7.5) # Let's test that they raise the right errors with self.assertRaises(KeyError): - await self.redis.increment("doesn't_exist!") + await self.cog.redis.increment("doesn't_exist!") - await self.redis.set("stringthing", "stringthing") + await self.cog.redis.set("stringthing", "stringthing") with self.assertRaises(TypeError): - await self.redis.increment("stringthing") + await self.cog.redis.increment("stringthing") + + async def test_increment_lock(self): + """Test that we can't produce a race condition in .increment.""" + await self.cog.redis.set("test_key", 0) + tasks = [] + + # Increment this a lot in different tasks + for _ in range(100): + task = asyncio.create_task( + self.cog.redis.increment("test_key", 1) + ) + tasks.append(task) + await asyncio.gather(*tasks) + + # Confirm that the value has been incremented the exact right number of times. + value = await self.cog.redis.get("test_key") + self.assertEqual(value, 100) async def test_exceptions_raised(self): """Testing that the various RuntimeErrors are reachable.""" -- cgit v1.2.3 From db0a384e91a463ff9668ab4f9ea5268aa332ab2d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 13:27:34 +0200 Subject: Remove the now deprecated in_channel_check. This check was no longer being used anywhere, having been replaced by in_whitelist_check. --- bot/utils/checks.py | 8 -------- tests/bot/utils/test_checks.py | 8 -------- 2 files changed, 16 deletions(-) diff --git a/bot/utils/checks.py b/bot/utils/checks.py index d5ebe4ec9..f0ef36302 100644 --- a/bot/utils/checks.py +++ b/bot/utils/checks.py @@ -120,14 +120,6 @@ def without_role_check(ctx: Context, *role_ids: int) -> bool: return check -def in_channel_check(ctx: Context, *channel_ids: int) -> bool: - """Checks if the command was executed inside the list of specified channels.""" - check = ctx.channel.id in channel_ids - log.trace(f"{ctx.author} tried to call the '{ctx.command.name}' command. " - f"The result of the in_channel check was {check}.") - return check - - def cooldown_with_role_bypass(rate: int, per: float, type: BucketType = BucketType.default, *, bypass_roles: Iterable[int]) -> Callable: """ diff --git a/tests/bot/utils/test_checks.py b/tests/bot/utils/test_checks.py index 9610771e5..d572b6299 100644 --- a/tests/bot/utils/test_checks.py +++ b/tests/bot/utils/test_checks.py @@ -41,11 +41,3 @@ class ChecksTests(unittest.TestCase): role_id = 42 self.ctx.author.roles.append(MockRole(id=role_id)) self.assertTrue(checks.without_role_check(self.ctx, role_id + 10)) - - def test_in_channel_check_for_correct_channel(self): - self.ctx.channel.id = 42 - self.assertTrue(checks.in_channel_check(self.ctx, *[42])) - - def test_in_channel_check_for_incorrect_channel(self): - self.ctx.channel.id = 42 + 10 - self.assertFalse(checks.in_channel_check(self.ctx, *[42])) -- cgit v1.2.3 From 876fae1856f1ad876d74036899739115fd8b86c3 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 13:39:32 +0200 Subject: Add some tests for `in_whitelist_check`. --- tests/bot/utils/test_checks.py | 48 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/bot/utils/test_checks.py b/tests/bot/utils/test_checks.py index d572b6299..de72e5748 100644 --- a/tests/bot/utils/test_checks.py +++ b/tests/bot/utils/test_checks.py @@ -1,6 +1,8 @@ import unittest +from unittest.mock import MagicMock from bot.utils import checks +from bot.utils.checks import InWhitelistCheckFailure from tests.helpers import MockContext, MockRole @@ -41,3 +43,49 @@ class ChecksTests(unittest.TestCase): role_id = 42 self.ctx.author.roles.append(MockRole(id=role_id)) self.assertTrue(checks.without_role_check(self.ctx, role_id + 10)) + + def test_in_whitelist_check_correct_channel(self): + """`in_whitelist_check` returns `True` if `Context.channel.id` is in the channel list.""" + channel_id = 3 + self.ctx.channel.id = channel_id + self.assertTrue(checks.in_whitelist_check(self.ctx, [channel_id])) + + def test_in_whitelist_check_incorrect_channel(self): + """`in_whitelist_check` raises InWhitelistCheckFailure if there's no channel match.""" + self.ctx.channel.id = 3 + with self.assertRaises(InWhitelistCheckFailure): + checks.in_whitelist_check(self.ctx, [4]) + + def test_in_whitelist_check_correct_category(self): + """`in_whitelist_check` returns `True` if `Context.channel.category_id` is in the category list.""" + category_id = 3 + self.ctx.channel.category_id = category_id + self.assertTrue(checks.in_whitelist_check(self.ctx, categories=[category_id])) + + def test_in_whitelist_check_incorrect_category(self): + """`in_whitelist_check` raises InWhitelistCheckFailure if there's no category match.""" + self.ctx.channel.category_id = 3 + with self.assertRaises(InWhitelistCheckFailure): + checks.in_whitelist_check(self.ctx, categories=[4]) + + def test_in_whitelist_check_correct_role(self): + """`in_whitelist_check` returns `True` if any of the `Context.author.roles` are in the roles list.""" + self.ctx.author.roles = (MagicMock(id=1), MagicMock(id=2)) + self.assertTrue(checks.in_whitelist_check(self.ctx, roles=[2, 6])) + + def test_in_whitelist_check_incorrect_role(self): + """`in_whitelist_check` raises InWhitelistCheckFailure if there's no role match.""" + self.ctx.author.roles = (MagicMock(id=1), MagicMock(id=2)) + with self.assertRaises(InWhitelistCheckFailure): + checks.in_whitelist_check(self.ctx, roles=[4]) + + def test_in_whitelist_check_fail_silently(self): + """`in_whitelist_check` test no exception raised if `fail_silently` is `True`""" + self.assertFalse(checks.in_whitelist_check(self.ctx, roles=[2, 6], fail_silently=True)) + + def test_in_whitelist_check_complex(self): + """`in_whitelist_check` test with multiple parameters""" + self.ctx.author.roles = (MagicMock(id=1), MagicMock(id=2)) + self.ctx.channel.category_id = 3 + self.ctx.channel.id = 5 + self.assertTrue(checks.in_whitelist_check(self.ctx, channels=[1], categories=[8], roles=[2])) -- cgit v1.2.3 From 4db313e9a7899666f1597094b0d88447c7b64311 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 20:15:19 +0200 Subject: Floats are no longer permitted as RedisCache keys. Also added a test for this. This is the DRYest approach I could find. It's a little ugly, but I think it's probably good enough. --- bot/utils/redis_cache.py | 116 ++++++++++++++++++++++++------------ tests/bot/utils/test_redis_cache.py | 13 ++-- 2 files changed, 86 insertions(+), 43 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 33e5d5852..afd37f8f8 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -2,26 +2,42 @@ from __future__ import annotations import asyncio import logging -from typing import Any, Dict, ItemsView, Optional, Union +import typing +from typing import Any, Dict, ItemsView, Optional, Tuple, Union from bot.bot import Bot log = logging.getLogger(__name__) -RedisType = Union[str, int, float] -TYPESTRING_PREFIXES = ( +# Type aliases +RedisKeyType = Union[str, int] +RedisValueType = Union[str, int, float] + +# Prefix tuples +PrefixTuple = Tuple[Tuple[str, Any]] +TYPESTRING_VALUE_PREFIXES = ( ("f|", float), ("i|", int), ("s|", str), ) +TYPESTRING_KEY_PREFIXES = ( + ("i|", int), + ("s|", str), +) # Makes a nice list like "float, int, and str" -NICE_TYPE_LIST = ", ".join(str(_type.__name__) for _, _type in TYPESTRING_PREFIXES) -NICE_TYPE_LIST = ", and ".join(NICE_TYPE_LIST.rsplit(", ", 1)) +NICE_VALUE_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisValueType)) +NICE_VALUE_TYPE_LIST = ", and ".join(NICE_VALUE_TYPE_LIST.rsplit(", ", 1)) + +NICE_KEY_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisKeyType)) +NICE_KEY_TYPE_LIST = ", and ".join(NICE_KEY_TYPE_LIST.rsplit(", ", 1)) # Makes a list like "'f|', 'i|', and 's|'" -NICE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_PREFIXES]) -NICE_PREFIX_LIST = ", and ".join(NICE_PREFIX_LIST.rsplit(", ", 1)) +NICE_VALUE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_VALUE_PREFIXES]) +NICE_VALUE_PREFIX_LIST = ", and ".join(NICE_VALUE_PREFIX_LIST.rsplit(", ", 1)) + +NICE_KEY_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_KEY_PREFIXES]) +NICE_KEY_PREFIX_LIST = ", and ".join(NICE_KEY_PREFIX_LIST.rsplit(", ", 1)) class RedisCache: @@ -99,33 +115,57 @@ class RedisCache: self._namespace = namespace @staticmethod - def _to_typestring(value: RedisType) -> str: + def _to_typestring( + key_or_value: Union[RedisKeyType, RedisValueType], + prefixes: PrefixTuple, + nice_type_list: str + ) -> str: """Turn a valid Redis type into a typestring.""" - for prefix, _type in TYPESTRING_PREFIXES: - if isinstance(value, _type): - return f"{prefix}{value}" - raise TypeError(f"RedisCache._from_typestring only supports the types {NICE_TYPE_LIST}.") + for prefix, _type in prefixes: + if isinstance(key_or_value, _type): + return f"{prefix}{key_or_value}" + raise TypeError(f"RedisCache._from_typestring only supports the types {nice_type_list}.") @staticmethod - def _from_typestring(value: Union[bytes, str]) -> RedisType: - """Turn a typestring into a valid Redis type.""" + def _from_typestring( + key_or_value: Union[bytes, str], + prefixes: PrefixTuple, + nice_prefix_list: str, + ) -> Union[RedisKeyType, RedisValueType]: + """Deserialize a typestring into a valid Redis type.""" # Stuff that comes out of Redis will be bytestrings, so let's decode those. - if isinstance(value, bytes): - value = value.decode('utf-8') + if isinstance(key_or_value, bytes): + key_or_value = key_or_value.decode('utf-8') # Now we convert our unicode string back into the type it originally was. - for prefix, _type in TYPESTRING_PREFIXES: - if value.startswith(prefix): - return _type(value[len(prefix):]) - raise TypeError(f"RedisCache._to_typestring only supports the prefixes {NICE_PREFIX_LIST}.") + for prefix, _type in prefixes: + if key_or_value.startswith(prefix): + return _type(key_or_value[len(prefix):]) + raise TypeError(f"RedisCache._to_typestring only supports the prefixes {nice_prefix_list}.") + + def _key_to_typestring(self, key: RedisKeyType) -> str: + """Serialize a RedisKeyType object into a typestring.""" + return self._to_typestring(key, TYPESTRING_KEY_PREFIXES, NICE_KEY_TYPE_LIST) + + def _value_to_typestring(self, value: RedisValueType) -> str: + """Serialize a RedisValueType object into a typestring.""" + return self._to_typestring(value, TYPESTRING_VALUE_PREFIXES, NICE_VALUE_TYPE_LIST) + + def _key_from_typestring(self, key: Union[bytes, str]) -> RedisKeyType: + """Deserialize a RedisKeyType object from a typestring.""" + return self._from_typestring(key, TYPESTRING_KEY_PREFIXES, NICE_KEY_PREFIX_LIST) + + def _value_from_typestring(self, value: Union[bytes, str]) -> RedisValueType: + """Deserialize a RedisValueType object from a typestring.""" + return self._from_typestring(value, TYPESTRING_VALUE_PREFIXES, NICE_VALUE_PREFIX_LIST) def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" - return {self._from_typestring(key): self._from_typestring(value) for key, value in dictionary.items()} + return {self._key_from_typestring(key): self._value_from_typestring(value) for key, value in dictionary.items()} def _dict_to_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into typestrings.""" - return {self._to_typestring(key): self._to_typestring(value) for key, value in dictionary.items()} + return {self._key_to_typestring(key): self._value_to_typestring(value) for key, value in dictionary.items()} async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" @@ -209,21 +249,21 @@ class RedisCache: """Return a beautiful representation of this object instance.""" return f"RedisCache(namespace={self._namespace!r})" - async def set(self, key: RedisType, value: RedisType) -> None: + async def set(self, key: RedisKeyType, value: RedisValueType) -> None: """Store an item in the Redis cache.""" await self._validate_cache() # Convert to a typestring and then set it - key = self._to_typestring(key) - value = self._to_typestring(value) + key = self._key_to_typestring(key) + value = self._value_to_typestring(value) log.trace(f"Setting {key} to {value}.") await self._redis.hset(self._namespace, key, value) - async def get(self, key: RedisType, default: Optional[RedisType] = None) -> Optional[RedisType]: + async def get(self, key: RedisKeyType, default: Optional[RedisValueType] = None) -> Optional[RedisValueType]: """Get an item from the Redis cache.""" await self._validate_cache() - key = self._to_typestring(key) + key = self._key_to_typestring(key) log.trace(f"Attempting to retrieve {key}.") value = await self._redis.hget(self._namespace, key) @@ -232,11 +272,11 @@ class RedisCache: log.trace(f"Value not found, returning default value {default}") return default else: - value = self._from_typestring(value) + value = self._value_from_typestring(value) log.trace(f"Value found, returning value {value}") return value - async def delete(self, key: RedisType) -> None: + async def delete(self, key: RedisKeyType) -> None: """ Delete an item from the Redis cache. @@ -245,19 +285,19 @@ class RedisCache: See https://redis.io/commands/hdel for more info on how this works. """ await self._validate_cache() - key = self._to_typestring(key) + key = self._key_to_typestring(key) log.trace(f"Attempting to delete {key}.") return await self._redis.hdel(self._namespace, key) - async def contains(self, key: RedisType) -> bool: + async def contains(self, key: RedisKeyType) -> bool: """ Check if a key exists in the Redis cache. Return True if the key exists, otherwise False. """ await self._validate_cache() - key = self._to_typestring(key) + key = self._key_to_typestring(key) exists = await self._redis.hexists(self._namespace, key) log.trace(f"Testing if {key} exists in the RedisCache - Result is {exists}") @@ -304,7 +344,7 @@ class RedisCache: log.trace("Clearing the cache of all key/value pairs.") await self._redis.delete(self._namespace) - async def pop(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: + async def pop(self, key: RedisKeyType, default: Optional[RedisValueType] = None) -> RedisValueType: """Get the item, remove it from the cache, and provide a default if not found.""" log.trace(f"Attempting to pop {key}.") value = await self.get(key, default) @@ -317,7 +357,7 @@ class RedisCache: return value - async def update(self, items: Dict[RedisType, RedisType]) -> None: + async def update(self, items: Dict[RedisKeyType, RedisValueType]) -> None: """ Update the Redis cache with multiple values. @@ -326,14 +366,14 @@ class RedisCache: do not exist in the RedisCache, they are created. If they do exist, the values are updated with the new ones from `items`. - Please note that both the keys and the values in the `items` dictionary - must consist of valid RedisTypes - ints, floats, or strings. + Please note that keys and the values in the `items` dictionary + must consist of valid RedisKeyTypes and RedisValueTypes. """ await self._validate_cache() log.trace(f"Updating the cache with the following items:\n{items}") await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) - async def increment(self, key: RedisType, amount: Optional[int, float] = 1) -> None: + async def increment(self, key: RedisKeyType, amount: Optional[int, float] = 1) -> None: """ Increment the value by `amount`. @@ -373,7 +413,7 @@ class RedisCache: log.error(error_message) raise TypeError(error_message) - async def decrement(self, key: RedisType, amount: Optional[int, float] = 1) -> None: + async def decrement(self, key: RedisKeyType, amount: Optional[int, float] = 1) -> None: """ Decrement the value by `amount`. diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index efd168dac..4f95dff03 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -70,12 +70,15 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(await self.cog.redis.get('favorite_nothing', "bearclaw"), "bearclaw") async def test_set_item_type(self): - """Test that .set rejects keys and values that are not strings, ints or floats.""" + """Test that .set rejects keys and values that are not permitted.""" fruits = ["lemon", "melon", "apple"] with self.assertRaises(TypeError): await self.cog.redis.set(fruits, "nice") + with self.assertRaises(TypeError): + await self.cog.redis.set(4.23, "nice") + async def test_delete_item(self): """Test that .delete allows us to delete stuff from the RedisCache.""" # Add an item and verify that it gets added @@ -176,16 +179,16 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): # Test conversion to typestring for _input, expected in conversion_tests: - self.assertEqual(self.cog.redis._to_typestring(_input), expected) + self.assertEqual(self.cog.redis._value_to_typestring(_input), expected) # Test conversion from typestrings for _input, expected in conversion_tests: - self.assertEqual(self.cog.redis._from_typestring(expected), _input) + self.assertEqual(self.cog.redis._value_from_typestring(expected), _input) # Test that exceptions are raised on invalid input with self.assertRaises(TypeError): - self.cog.redis._to_typestring(["internet"]) - self.cog.redis._from_typestring("o|firedog") + self.cog.redis._value_to_typestring(["internet"]) + self.cog.redis._value_from_typestring("o|firedog") async def test_increment_decrement(self): """Test .increment and .decrement methods.""" -- cgit v1.2.3 From b6093bf7df00be1ed04a51119a65dbdd74ae0e58 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 21:01:24 +0200 Subject: Refactor typestring converters to partialmethods. We're using functools.partialmethod to make the code a little cleaner and more readable here. Read more about them here: https://docs.python.org/3/library/functools.html#functools.partial https://docs.python.org/3/library/functools.html#functools.partialmethod --- bot/utils/redis_cache.py | 54 +++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index afd37f8f8..dd24b83e8 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import logging import typing +from functools import partialmethod from typing import Any, Dict, ItemsView, Optional, Tuple, Union from bot.bot import Bot @@ -15,29 +16,29 @@ RedisValueType = Union[str, int, float] # Prefix tuples PrefixTuple = Tuple[Tuple[str, Any]] -TYPESTRING_VALUE_PREFIXES = ( +VALUE_PREFIXES = ( ("f|", float), ("i|", int), ("s|", str), ) -TYPESTRING_KEY_PREFIXES = ( +KEY_PREFIXES = ( ("i|", int), ("s|", str), ) # Makes a nice list like "float, int, and str" -NICE_VALUE_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisValueType)) -NICE_VALUE_TYPE_LIST = ", and ".join(NICE_VALUE_TYPE_LIST.rsplit(", ", 1)) +VALUE_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisValueType)) +VALUE_TYPE_LIST = ", and ".join(VALUE_TYPE_LIST.rsplit(", ", 1)) -NICE_KEY_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisKeyType)) -NICE_KEY_TYPE_LIST = ", and ".join(NICE_KEY_TYPE_LIST.rsplit(", ", 1)) +KEY_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisKeyType)) +KEY_TYPE_LIST = ", and ".join(KEY_TYPE_LIST.rsplit(", ", 1)) # Makes a list like "'f|', 'i|', and 's|'" -NICE_VALUE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_VALUE_PREFIXES]) -NICE_VALUE_PREFIX_LIST = ", and ".join(NICE_VALUE_PREFIX_LIST.rsplit(", ", 1)) +VALUE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in VALUE_PREFIXES]) +VALUE_PREFIX_LIST = ", and ".join(VALUE_PREFIX_LIST.rsplit(", ", 1)) -NICE_KEY_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_KEY_PREFIXES]) -NICE_KEY_PREFIX_LIST = ", and ".join(NICE_KEY_PREFIX_LIST.rsplit(", ", 1)) +KEY_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in KEY_PREFIXES]) +KEY_PREFIX_LIST = ", and ".join(KEY_PREFIX_LIST.rsplit(", ", 1)) class RedisCache: @@ -118,19 +119,19 @@ class RedisCache: def _to_typestring( key_or_value: Union[RedisKeyType, RedisValueType], prefixes: PrefixTuple, - nice_type_list: str + types_string: str ) -> str: """Turn a valid Redis type into a typestring.""" for prefix, _type in prefixes: if isinstance(key_or_value, _type): return f"{prefix}{key_or_value}" - raise TypeError(f"RedisCache._from_typestring only supports the types {nice_type_list}.") + raise TypeError(f"RedisCache._from_typestring only supports the types {types_string}.") @staticmethod def _from_typestring( key_or_value: Union[bytes, str], prefixes: PrefixTuple, - nice_prefix_list: str, + prefixes_string: str, ) -> Union[RedisKeyType, RedisValueType]: """Deserialize a typestring into a valid Redis type.""" # Stuff that comes out of Redis will be bytestrings, so let's decode those. @@ -141,23 +142,16 @@ class RedisCache: for prefix, _type in prefixes: if key_or_value.startswith(prefix): return _type(key_or_value[len(prefix):]) - raise TypeError(f"RedisCache._to_typestring only supports the prefixes {nice_prefix_list}.") - - def _key_to_typestring(self, key: RedisKeyType) -> str: - """Serialize a RedisKeyType object into a typestring.""" - return self._to_typestring(key, TYPESTRING_KEY_PREFIXES, NICE_KEY_TYPE_LIST) - - def _value_to_typestring(self, value: RedisValueType) -> str: - """Serialize a RedisValueType object into a typestring.""" - return self._to_typestring(value, TYPESTRING_VALUE_PREFIXES, NICE_VALUE_TYPE_LIST) - - def _key_from_typestring(self, key: Union[bytes, str]) -> RedisKeyType: - """Deserialize a RedisKeyType object from a typestring.""" - return self._from_typestring(key, TYPESTRING_KEY_PREFIXES, NICE_KEY_PREFIX_LIST) - - def _value_from_typestring(self, value: Union[bytes, str]) -> RedisValueType: - """Deserialize a RedisValueType object from a typestring.""" - return self._from_typestring(value, TYPESTRING_VALUE_PREFIXES, NICE_VALUE_PREFIX_LIST) + raise TypeError(f"RedisCache._to_typestring only supports the prefixes {prefixes_string}.") + + # Add some nice partials to call our generic typestring converters. + # These are basically functions that will fill in some of the parameters for you, so that + # any call to _key_to_typestring will be like calling _to_typestring with those two parameters + # pre-filled. + _key_to_typestring = partialmethod(_to_typestring, prefixes=KEY_PREFIXES, types_string=KEY_TYPE_LIST) + _value_to_typestring = partialmethod(_to_typestring, prefixes=VALUE_PREFIXES, types_string=VALUE_TYPE_LIST) + _key_from_typestring = partialmethod(_from_typestring, prefixes=KEY_PREFIXES, prefixes_string=KEY_PREFIX_LIST) + _value_from_typestring = partialmethod(_from_typestring, prefixes=VALUE_PREFIXES, prefixes_string=VALUE_PREFIX_LIST) def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" -- cgit v1.2.3 From 63b81b04da3cbc4d1824e65c977ec61532dbe605 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 21:04:08 +0200 Subject: Fix ATROCIOUS comment. I should be shot. --- bot/utils/redis_cache.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index dd24b83e8..a71ad2191 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -145,9 +145,11 @@ class RedisCache: raise TypeError(f"RedisCache._to_typestring only supports the prefixes {prefixes_string}.") # Add some nice partials to call our generic typestring converters. - # These are basically functions that will fill in some of the parameters for you, so that - # any call to _key_to_typestring will be like calling _to_typestring with those two parameters - # pre-filled. + # These are basically methods that will fill in some of the parameters for you, so that + # any call to _key_to_typestring will be like calling _to_typestring with the two parameters + # at `prefixes` and `types_string` pre-filled. + # + # See https://docs.python.org/3/library/functools.html#functools.partialmethod _key_to_typestring = partialmethod(_to_typestring, prefixes=KEY_PREFIXES, types_string=KEY_TYPE_LIST) _value_to_typestring = partialmethod(_to_typestring, prefixes=VALUE_PREFIXES, types_string=VALUE_TYPE_LIST) _key_from_typestring = partialmethod(_from_typestring, prefixes=KEY_PREFIXES, prefixes_string=KEY_PREFIX_LIST) -- cgit v1.2.3 From bdb7bbc5e98bd840785def6ac08c9f5a313847cb Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 28 May 2020 00:43:58 +0200 Subject: Reduce complexity on some of the typestring stuff. - Refactor error messages in _to_typestring and _from_typestring to just print the prefix tuples instead of that custom error string. - Create a RedisKeyOrValue type to simplify some annotations. - Simplify partialmethod calls. - Make the signatures for _to_typestring and _from_typestring one-liners - Fix a typo in the errors. --- bot/utils/redis_cache.py | 40 +++++++++------------------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index a71ad2191..0b682d378 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -2,7 +2,6 @@ from __future__ import annotations import asyncio import logging -import typing from functools import partialmethod from typing import Any, Dict, ItemsView, Optional, Tuple, Union @@ -13,6 +12,7 @@ log = logging.getLogger(__name__) # Type aliases RedisKeyType = Union[str, int] RedisValueType = Union[str, int, float] +RedisKeyOrValue = Union[RedisKeyType, RedisValueType] # Prefix tuples PrefixTuple = Tuple[Tuple[str, Any]] @@ -26,20 +26,6 @@ KEY_PREFIXES = ( ("s|", str), ) -# Makes a nice list like "float, int, and str" -VALUE_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisValueType)) -VALUE_TYPE_LIST = ", and ".join(VALUE_TYPE_LIST.rsplit(", ", 1)) - -KEY_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisKeyType)) -KEY_TYPE_LIST = ", and ".join(KEY_TYPE_LIST.rsplit(", ", 1)) - -# Makes a list like "'f|', 'i|', and 's|'" -VALUE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in VALUE_PREFIXES]) -VALUE_PREFIX_LIST = ", and ".join(VALUE_PREFIX_LIST.rsplit(", ", 1)) - -KEY_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in KEY_PREFIXES]) -KEY_PREFIX_LIST = ", and ".join(KEY_PREFIX_LIST.rsplit(", ", 1)) - class RedisCache: """ @@ -116,23 +102,15 @@ class RedisCache: self._namespace = namespace @staticmethod - def _to_typestring( - key_or_value: Union[RedisKeyType, RedisValueType], - prefixes: PrefixTuple, - types_string: str - ) -> str: + def _to_typestring(key_or_value: RedisKeyOrValue, prefixes: PrefixTuple) -> str: """Turn a valid Redis type into a typestring.""" for prefix, _type in prefixes: if isinstance(key_or_value, _type): return f"{prefix}{key_or_value}" - raise TypeError(f"RedisCache._from_typestring only supports the types {types_string}.") + raise TypeError(f"RedisCache._to_typestring only supports the following: {prefixes}.") @staticmethod - def _from_typestring( - key_or_value: Union[bytes, str], - prefixes: PrefixTuple, - prefixes_string: str, - ) -> Union[RedisKeyType, RedisValueType]: + def _from_typestring(key_or_value: Union[bytes, str], prefixes: PrefixTuple) -> RedisKeyOrValue: """Deserialize a typestring into a valid Redis type.""" # Stuff that comes out of Redis will be bytestrings, so let's decode those. if isinstance(key_or_value, bytes): @@ -142,7 +120,7 @@ class RedisCache: for prefix, _type in prefixes: if key_or_value.startswith(prefix): return _type(key_or_value[len(prefix):]) - raise TypeError(f"RedisCache._to_typestring only supports the prefixes {prefixes_string}.") + raise TypeError(f"RedisCache._from_typestring only supports the following: {prefixes}.") # Add some nice partials to call our generic typestring converters. # These are basically methods that will fill in some of the parameters for you, so that @@ -150,10 +128,10 @@ class RedisCache: # at `prefixes` and `types_string` pre-filled. # # See https://docs.python.org/3/library/functools.html#functools.partialmethod - _key_to_typestring = partialmethod(_to_typestring, prefixes=KEY_PREFIXES, types_string=KEY_TYPE_LIST) - _value_to_typestring = partialmethod(_to_typestring, prefixes=VALUE_PREFIXES, types_string=VALUE_TYPE_LIST) - _key_from_typestring = partialmethod(_from_typestring, prefixes=KEY_PREFIXES, prefixes_string=KEY_PREFIX_LIST) - _value_from_typestring = partialmethod(_from_typestring, prefixes=VALUE_PREFIXES, prefixes_string=VALUE_PREFIX_LIST) + _key_to_typestring = partialmethod(_to_typestring, prefixes=KEY_PREFIXES) + _value_to_typestring = partialmethod(_to_typestring, prefixes=VALUE_PREFIXES) + _key_from_typestring = partialmethod(_from_typestring, prefixes=KEY_PREFIXES) + _value_from_typestring = partialmethod(_from_typestring, prefixes=VALUE_PREFIXES) def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" -- cgit v1.2.3 From 11542fbcc7c32fb9a18577c45ae3c331eaa12db8 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 28 May 2020 00:56:10 +0200 Subject: Make prefix consts private and more precise. --- bot/utils/redis_cache.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 0b682d378..979ea5d47 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -15,13 +15,13 @@ RedisValueType = Union[str, int, float] RedisKeyOrValue = Union[RedisKeyType, RedisValueType] # Prefix tuples -PrefixTuple = Tuple[Tuple[str, Any]] -VALUE_PREFIXES = ( +_PrefixTuple = Tuple[Tuple[str, Any], ...] +_VALUE_PREFIXES = ( ("f|", float), ("i|", int), ("s|", str), ) -KEY_PREFIXES = ( +_KEY_PREFIXES = ( ("i|", int), ("s|", str), ) @@ -102,7 +102,7 @@ class RedisCache: self._namespace = namespace @staticmethod - def _to_typestring(key_or_value: RedisKeyOrValue, prefixes: PrefixTuple) -> str: + def _to_typestring(key_or_value: RedisKeyOrValue, prefixes: _PrefixTuple) -> str: """Turn a valid Redis type into a typestring.""" for prefix, _type in prefixes: if isinstance(key_or_value, _type): @@ -110,7 +110,7 @@ class RedisCache: raise TypeError(f"RedisCache._to_typestring only supports the following: {prefixes}.") @staticmethod - def _from_typestring(key_or_value: Union[bytes, str], prefixes: PrefixTuple) -> RedisKeyOrValue: + def _from_typestring(key_or_value: Union[bytes, str], prefixes: _PrefixTuple) -> RedisKeyOrValue: """Deserialize a typestring into a valid Redis type.""" # Stuff that comes out of Redis will be bytestrings, so let's decode those. if isinstance(key_or_value, bytes): @@ -128,10 +128,10 @@ class RedisCache: # at `prefixes` and `types_string` pre-filled. # # See https://docs.python.org/3/library/functools.html#functools.partialmethod - _key_to_typestring = partialmethod(_to_typestring, prefixes=KEY_PREFIXES) - _value_to_typestring = partialmethod(_to_typestring, prefixes=VALUE_PREFIXES) - _key_from_typestring = partialmethod(_from_typestring, prefixes=KEY_PREFIXES) - _value_from_typestring = partialmethod(_from_typestring, prefixes=VALUE_PREFIXES) + _key_to_typestring = partialmethod(_to_typestring, prefixes=_KEY_PREFIXES) + _value_to_typestring = partialmethod(_to_typestring, prefixes=_VALUE_PREFIXES) + _key_from_typestring = partialmethod(_from_typestring, prefixes=_KEY_PREFIXES) + _value_from_typestring = partialmethod(_from_typestring, prefixes=_VALUE_PREFIXES) def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" -- cgit v1.2.3 From f66a63501fe1ef8fb5390dfbe42ae9f95ea2bc28 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 28 May 2020 01:29:34 +0200 Subject: Add custom exceptions for each error state. The bot can get into trouble in three distinct ways: - It has no Bot instance - It has no namespace - It has no parent instance. These happen only if you're using it wrong. To make the test more precise, and to add a little bit more readability (RuntimeError could be anything!), we'll introduce some custom exceptions for these three states. This addresses a review comment by @aeros. --- bot/utils/redis_cache.py | 22 +++++++++++++++++----- tests/bot/utils/test_redis_cache.py | 7 ++++--- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 979ea5d47..6b3c68979 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -27,6 +27,18 @@ _KEY_PREFIXES = ( ) +class NoBotInstanceError(RuntimeError): + """Raised when RedisCache is created without an available bot instance on the owner class.""" + + +class NoNamespaceError(RuntimeError): + """Raised when RedisCache has no namespace, for example if it is not assigned to a class attribute.""" + + +class NoParentInstanceError(RuntimeError): + """Raised when the parent instance is available, for example if called by accessing the parent class directly.""" + + class RedisCache: """ A simplified interface for a Redis connection. @@ -149,7 +161,7 @@ class RedisCache: "This object must be initialized as a class attribute." ) log.error(error_message) - raise RuntimeError(error_message) + raise NoNamespaceError(error_message) if self.bot is None: error_message = ( @@ -159,7 +171,7 @@ class RedisCache: "the RedisCache inside a class that has a Bot instance attribute." ) log.error(error_message) - raise RuntimeError(error_message) + raise NoBotInstanceError(error_message) await self.bot.redis_ready.wait() @@ -194,7 +206,7 @@ class RedisCache: if self._namespace is None: error_message = "RedisCache must be a class attribute." log.error(error_message) - raise RuntimeError(error_message) + raise NoNamespaceError(error_message) if instance is None: error_message = ( @@ -202,7 +214,7 @@ class RedisCache: "before accessing it using the cog's class object." ) log.error(error_message) - raise RuntimeError(error_message) + raise NoParentInstanceError(error_message) for attribute in vars(instance).values(): if isinstance(attribute, Bot): @@ -217,7 +229,7 @@ class RedisCache: "the RedisCache inside a class that has a Bot instance attribute." ) log.error(error_message) - raise RuntimeError(error_message) + raise NoBotInstanceError(error_message) def __repr__(self) -> str: """Return a beautiful representation of this object instance.""" diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 4f95dff03..8c1a40640 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -4,6 +4,7 @@ import unittest import fakeredis.aioredis from bot.utils import RedisCache +from bot.utils.redis_cache import NoBotInstanceError, NoNamespaceError, NoParentInstanceError from tests import helpers @@ -260,13 +261,13 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): cog = MyCog() # Raises "No Bot instance" - with self.assertRaises(RuntimeError): + with self.assertRaises(NoBotInstanceError): await cog.cache.get("john") # Raises "RedisCache has no namespace" - with self.assertRaises(RuntimeError): + with self.assertRaises(NoNamespaceError): await cog.other_cache.get("was") # Raises "You must access the RedisCache instance through the cog instance" - with self.assertRaises(RuntimeError): + with self.assertRaises(NoParentInstanceError): await MyCog.cache.get("afraid") -- cgit v1.2.3 From b7c30d41e263605a680cfe0f623b8e7ed5936b7d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 28 May 2020 10:35:17 +0200 Subject: Prevent a state where a coro could wait forever. This addresses a review comment by @aeros. --- bot/bot.py | 5 ++++- bot/utils/redis_cache.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bot/bot.py b/bot/bot.py index f1365d532..ba09ce207 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -32,6 +32,7 @@ class Bot(commands.Bot): self.http_session: Optional[aiohttp.ClientSession] = None self.redis_session: Optional[aioredis.Redis] = None self.redis_ready = asyncio.Event() + self.redis_closed = False self.api_client = api.APIClient(loop=self.loop) self._connector = None @@ -106,8 +107,9 @@ class Bot(commands.Bot): self.stats._transport.close() if self.redis_session: - self.redis_ready.clear() + self.redis_closed = True self.redis_session.close() + self.redis_ready.clear() await self.redis_session.wait_closed() async def login(self, *args, **kwargs) -> None: @@ -135,6 +137,7 @@ class Bot(commands.Bot): # Create the redis session self.loop.create_task(self._create_redis_session()) + self.redis_closed = False # Use AF_INET as its socket family to prevent HTTPS related problems both locally # and in production. diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 6b3c68979..de80cee84 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -173,7 +173,8 @@ class RedisCache: log.error(error_message) raise NoBotInstanceError(error_message) - await self.bot.redis_ready.wait() + if not self.bot.redis_closed: + await self.bot.redis_ready.wait() def __set_name__(self, owner: Any, attribute_name: str) -> None: """ -- cgit v1.2.3 From cc45960406f64a791a15cf9de76614103fda384b Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 28 May 2020 12:45:26 +0200 Subject: Move the `self.redis_closed` into session create. --- bot/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/bot.py b/bot/bot.py index ba09ce207..313652d11 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -70,6 +70,7 @@ class Bot(commands.Bot): password=constants.Redis.password, ) + self.redis_closed = False self.redis_ready.set() def add_cog(self, cog: commands.Cog) -> None: @@ -137,7 +138,6 @@ class Bot(commands.Bot): # Create the redis session self.loop.create_task(self._create_redis_session()) - self.redis_closed = False # Use AF_INET as its socket family to prevent HTTPS related problems both locally # and in production. -- cgit v1.2.3 From d12e84fe6834a0bc574e365a3283bc358c2ae4d9 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Fri, 29 May 2020 17:57:09 +0200 Subject: Ignore response when posting python news Sometimes a mailing list user doesn't press respond correctly to the email, and so a response is sent as a separate thread. To keep only new threads in the channel, we need to ignore those. --- bot/cogs/python_news.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/cogs/python_news.py b/bot/cogs/python_news.py index d28af4a0b..d15d0371e 100644 --- a/bot/cogs/python_news.py +++ b/bot/cogs/python_news.py @@ -153,6 +153,7 @@ class PythonNews(Cog): if ( thread_information["thread_id"] in existing_news["data"][maillist] + or 'Re: ' in thread_information["subject"] or new_date.date() < date.today() ): continue -- cgit v1.2.3 From 63f5028641e9b78c61d3bcfe3bbaa6f80c8a288a Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff Date: Fri, 29 May 2020 19:56:40 +0200 Subject: Fix `check_for_answer` breaking on missing cache The `check_for_answer` method of the HelpChannels cog relies on the channel->claimant cache being available. However, as this cache is (currently) lost during bot restarts, this method may fail with a KeyError exception. I've used `dict.get` with an `if not claimant: return` to circumvent this issue. --- bot/cogs/help_channels.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index d2a55fba6..2221132d4 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -660,10 +660,13 @@ class HelpChannels(Scheduler, commands.Cog): # Check if there is an entry in unanswered (does not persist across restarts) if channel.id in self.unanswered: - claimant_id = self.help_channel_claimants[channel].id + claimant = self.help_channel_claimants.get(channel) + if not claimant: + # The mapping for this channel was lost, we can't do anything. + return # Check the message did not come from the claimant - if claimant_id != message.author.id: + if claimant.id != message.author.id: # Mark the channel as answered self.unanswered[channel.id] = False -- cgit v1.2.3 From 28f20b969556e0ce1363ac44a5b9ff2bff2a6575 Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff Date: Fri, 29 May 2020 19:49:00 +0200 Subject: Reduce the number of help channel name changes Discord has introduced a new, strict rate limit for individual channel edits that reduces the number of allow channel name/channel topic changes to 2 per 10 minutes per channel. Unfortunately, our help channel system frequently goes over that rate limit as it edits the name and topic of a channel on all three "move" actions we have: to available, to occupied, and to dormant. In addition, our "unanswered" feature adds another channel name change on top of the move-related edits. That's why I've removed the topic/emoji changing features from the help channel system. This means we now have a generic topic that fits all three categories and no status emojis in the channel names. --- bot/cogs/help_channels.py | 32 +++----------------------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 2221132d4..70cef339a 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -24,18 +24,8 @@ ASKING_GUIDE_URL = "https://pythondiscord.com/pages/asking-good-questions/" MAX_CHANNELS_PER_CATEGORY = 50 EXCLUDED_CHANNELS = (constants.Channels.how_to_get_help,) -AVAILABLE_TOPIC = """ -This channel is available. Feel free to ask a question in order to claim this channel! -""" - -IN_USE_TOPIC = """ -This channel is currently in use. If you'd like to discuss a different problem, please claim a new \ -channel from the Help: Available category. -""" - -DORMANT_TOPIC = """ -This channel is temporarily archived. If you'd like to ask a question, please use one of the \ -channels in the Help: Available category. +HELP_CHANNEL_TOPIC = """ +This is a Python help channel. You can claim your own help channel in the Python Help: Available category. """ AVAILABLE_MSG = f""" @@ -64,11 +54,6 @@ question to maximize your chance of getting a good answer. If you're not sure ho through our guide for [asking a good question]({ASKING_GUIDE_URL}). """ -AVAILABLE_EMOJI = "✅" -IN_USE_ANSWERED_EMOJI = "⌛" -IN_USE_UNANSWERED_EMOJI = "⏳" -NAME_SEPARATOR = "｜" - CoroutineFunc = t.Callable[..., t.Coroutine] @@ -196,7 +181,7 @@ class HelpChannels(Scheduler, commands.Cog): return None log.debug(f"Creating a new dormant channel named {name}.") - return await self.dormant_category.create_text_channel(name) + return await self.dormant_category.create_text_channel(name, topic=HELP_CHANNEL_TOPIC) def create_name_queue(self) -> deque: """Return a queue of element names to use for creating new channels.""" @@ -542,8 +527,6 @@ class HelpChannels(Scheduler, commands.Cog): await self.move_to_bottom_position( channel=channel, category_id=constants.Categories.help_available, - name=f"{AVAILABLE_EMOJI}{NAME_SEPARATOR}{self.get_clean_channel_name(channel)}", - topic=AVAILABLE_TOPIC, ) self.report_stats() @@ -559,8 +542,6 @@ class HelpChannels(Scheduler, commands.Cog): await self.move_to_bottom_position( channel=channel, category_id=constants.Categories.help_dormant, - name=self.get_clean_channel_name(channel), - topic=DORMANT_TOPIC, ) self.bot.stats.incr(f"help.dormant_calls.{caller}") @@ -593,8 +574,6 @@ class HelpChannels(Scheduler, commands.Cog): await self.move_to_bottom_position( channel=channel, category_id=constants.Categories.help_in_use, - name=f"{IN_USE_UNANSWERED_EMOJI}{NAME_SEPARATOR}{self.get_clean_channel_name(channel)}", - topic=IN_USE_TOPIC, ) timeout = constants.HelpChannels.idle_minutes * 60 @@ -670,11 +649,6 @@ class HelpChannels(Scheduler, commands.Cog): # Mark the channel as answered self.unanswered[channel.id] = False - # Change the emoji in the channel name to signify activity - log.trace(f"#{channel} ({channel.id}) has been answered; changing its emoji") - name = self.get_clean_channel_name(channel) - await channel.edit(name=f"{IN_USE_ANSWERED_EMOJI}{NAME_SEPARATOR}{name}") - @commands.Cog.listener() async def on_message(self, message: discord.Message) -> None: """Move an available channel to the In Use category and replace it with a dormant one.""" -- cgit v1.2.3 From aa46d01fa6c6fd14a9613412783ce377fe7e967d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Fri, 29 May 2020 23:52:56 +0200 Subject: Clean up channel counts and add staff channels. Cleaning up a particularly dirty line by turning it into like 10 lines, and also adding the number of channels that are hidden to the `@everyone` role - which we're classifying as "Staff channels". --- bot/cogs/information.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index f0eb3a1ea..8309cff4b 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -104,8 +104,27 @@ class Information(Cog): member_count = ctx.guild.member_count # How many of each type of channel? - channels = Counter(c.type for c in ctx.guild.channels) - channel_counts = "".join(sorted(f"{str(ch).title()} channels: {channels[ch]}\n" for ch in channels)).strip() + channel_counter = Counter(c.type for c in ctx.guild.channels) + channel_type_list = [] + for channel in channel_counter: + channel_type = str(channel).title() + channel_type_list.append(f"{channel_type} channels: {channel_counter[channel]}") + + channel_type_list = sorted(channel_type_list) + channel_counts = "\n".join(channel_type_list).strip() + + # How many channels are for staff only? + everyone_role = ctx.guild.roles[0] + hidden_channels = 0 + + for channel in ctx.guild.channels: + overwrites = channel.overwrites_for(everyone_role) + if overwrites.is_empty(): + continue + + for perm, value in overwrites: + if perm == 'read_messages' and value is False: + hidden_channels += 1 # How many of each user status? statuses = Counter(member.status for member in ctx.guild.members) @@ -126,6 +145,7 @@ class Information(Cog): Members: {member_count:,} Roles: {roles} $channel_counts + Staff channels: {hidden_channels} **Members** {constants.Emojis.status_online} {statuses[Status.online]:,} -- cgit v1.2.3 From 171c1e2713355570baf50c687e7466daea834b89 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 00:02:47 +0200 Subject: Adding staff member count to !server. --- bot/cogs/information.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index 8309cff4b..d830806c1 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -130,6 +130,9 @@ class Information(Cog): statuses = Counter(member.status for member in ctx.guild.members) embed = Embed(colour=Colour.blurple()) + # How many staff members? + staff_members = len(ctx.guild.get_role(constants.Roles.helpers).members) + # Because channel_counts lacks leading whitespace, it breaks the dedent if it's inserted directly by the # f-string. While this is correctly formated by Discord, it makes unit testing difficult. To keep the formatting # without joining a tuple of strings we can use a Template string to insert the already-formatted channel_counts @@ -141,13 +144,16 @@ class Information(Cog): Voice region: {region} Features: {features} - **Counts** - Members: {member_count:,} - Roles: {roles} + **Channel counts** $channel_counts Staff channels: {hidden_channels} - **Members** + **Member counts** + Members: {member_count:,} + Staff members: {staff_members} + Roles: {roles} + + **Member statuses** {constants.Emojis.status_online} {statuses[Status.online]:,} {constants.Emojis.status_idle} {statuses[Status.idle]:,} {constants.Emojis.status_dnd} {statuses[Status.dnd]:,} -- cgit v1.2.3 From b258f77d1c9a1b62fef26e7fecdb89e57719dfac Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 00:12:44 +0200 Subject: Removing the periodic ping from verification. It's no longer needed, and causes problems with anti-raid and anti-spam. --- bot/cogs/verification.py | 44 +------------------------------------------- 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 99be3cdaa..0a087cee9 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -1,9 +1,7 @@ import logging from contextlib import suppress -from datetime import datetime from discord import Colour, Forbidden, Message, NotFound, Object -from discord.ext import tasks from discord.ext.commands import Cog, Context, command from bot import constants @@ -34,14 +32,6 @@ If you'd like to unsubscribe from the announcement notifications, simply send `! <#{constants.Channels.bot_commands}>. """ -if constants.DEBUG_MODE: - PERIODIC_PING = "Periodic checkpoint message successfully sent." -else: - PERIODIC_PING = ( - f"@everyone To verify that you have read our rules, please type `{constants.Bot.prefix}accept`." - " If you encounter any problems during the verification process, " - f"send a direct message to a staff member." - ) BOT_MESSAGE_DELETE_DELAY = 10 @@ -50,7 +40,6 @@ class Verification(Cog): def __init__(self, bot: Bot): self.bot = bot - self.periodic_ping.start() @property def mod_log(self) -> ModLog: @@ -65,10 +54,7 @@ class Verification(Cog): if message.author.bot: # They're a bot, delete their message after the delay. - # But not the periodic ping; we like that one. - if message.content != PERIODIC_PING: - await message.delete(delay=BOT_MESSAGE_DELETE_DELAY) - return + await message.delete(delay=BOT_MESSAGE_DELETE_DELAY) # if a user mentions a role or guild member # alert the mods in mod-alerts channel @@ -198,34 +184,6 @@ class Verification(Cog): else: return True - @tasks.loop(hours=12) - async def periodic_ping(self) -> None: - """Every week, mention @everyone to remind them to verify.""" - messages = self.bot.get_channel(constants.Channels.verification).history(limit=10) - need_to_post = True # True if a new message needs to be sent. - - async for message in messages: - if message.author == self.bot.user and message.content == PERIODIC_PING: - delta = datetime.utcnow() - message.created_at # Time since last message. - if delta.days >= 7: # Message is older than a week. - await message.delete() - else: - need_to_post = False - - break - - if need_to_post: - await self.bot.get_channel(constants.Channels.verification).send(PERIODIC_PING) - - @periodic_ping.before_loop - async def before_ping(self) -> None: - """Only start the loop when the bot is ready.""" - await self.bot.wait_until_guild_available() - - def cog_unload(self) -> None: - """Cancel the periodic ping task when the cog is unloaded.""" - self.periodic_ping.cancel() - def setup(bot: Bot) -> None: """Load the Verification cog.""" -- cgit v1.2.3 From f7fe7df271b0e0930fa8b997c087bb3be141d016 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 11 Apr 2020 07:43:56 +0200 Subject: Tags: explicitly use UTF-8 to read files Not all operating systems use UTF-8 as the default encoding. For systems that don't, reading tag files with Unicode would cause an unhandled exception. (cherry picked from commit adc75ff9bbcf8b905bd78c78f253522ae5e42fc3) --- bot/cogs/tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index bc7f53f68..6f03a3475 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -44,7 +44,7 @@ class Tags(Cog): tag = { "title": tag_title, "embed": { - "description": file.read_text(), + "description": file.read_text(encoding="utf8"), }, "restricted_to": "developers", } -- cgit v1.2.3 From 89752c5ff15bc55bb1986d131f562bedfdf9e63a Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 01:35:19 +0200 Subject: More precise staff-channel check. We now check: - Does the @everyone role have explicit read deny permissions? - Do staff roles have explicit read allow permissions? If the answer to both of these are yes, it's a staff channel. By 'staff roles', I mean Helpers, Moderators or Admins. --- bot/cogs/information.py | 66 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index d830806c1..715623620 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -1,11 +1,13 @@ import colorsys +import functools import logging import pprint import textwrap from collections import Counter, defaultdict from string import Template -from typing import Any, Mapping, Optional, Union +from typing import Any, List, Mapping, Optional, Union +import more_itertools from discord import Colour, Embed, Member, Message, Role, Status, utils from discord.ext.commands import BucketType, Cog, Context, Paginator, command, group from discord.utils import escape_markdown @@ -26,6 +28,34 @@ class Information(Cog): def __init__(self, bot: Bot): self.bot = bot + @staticmethod + def _get_channels_with_role_permission(ctx: Context, role: Role, perm: str, value: Optional[bool]) -> List[int]: + """Get a list of channel IDs where a role has a specific permission set to a specific value.""" + channel_ids = [] + + for channel in ctx.guild.channels: + overwrites = channel.overwrites_for(role) + if overwrites.is_empty(): + continue + + for _perm, _value in overwrites: + if _perm == perm and _value is value: + channel_ids.append(channel.id) + + return channel_ids + + _get_channels_where_role_can_read = functools.partialmethod( + _get_channels_with_role_permission, + perm='read_messages', + value=True + ) + + _get_channels_where_role_cannot_read = functools.partialmethod( + _get_channels_with_role_permission, + perm='read_messages', + value=False + ) + @with_role(*constants.MODERATION_ROLES) @command(name="roles") async def roles_info(self, ctx: Context) -> None: @@ -114,17 +144,27 @@ class Information(Cog): channel_counts = "\n".join(channel_type_list).strip() # How many channels are for staff only? - everyone_role = ctx.guild.roles[0] - hidden_channels = 0 - - for channel in ctx.guild.channels: - overwrites = channel.overwrites_for(everyone_role) - if overwrites.is_empty(): - continue - - for perm, value in overwrites: - if perm == 'read_messages' and value is False: - hidden_channels += 1 + # We need to know two things about a channel: + # - Does the @everyone role have explicit read deny permissions? + # - Do staff roles have explicit read allow permissions? + # + # If the answer to both of these questions is yes, it's a staff channel. + helpers = ctx.guild.get_role(constants.Roles.helpers) + moderators = ctx.guild.get_role(constants.Roles.moderators) + admins = ctx.guild.get_role(constants.Roles.admins) + everyone = ctx.guild.roles[0] + + # Let's build some lists of channels. + everyone_denied = self._get_channels_where_role_cannot_read(ctx, everyone) + staff_allowed = more_itertools.flatten([ + self._get_channels_where_role_can_read(ctx, admins), # Admins has explicit read message allow + self._get_channels_where_role_can_read(ctx, moderators), # Moderators has explicit read message allow + self._get_channels_where_role_can_read(ctx, helpers), # Helpers has explicit read message allow + ]) + + # Now we need to check which channels are both denied for everyone and permitted for staff + staff_channels = [cid for cid in staff_allowed if cid in everyone_denied] + staff_channel_count = len(staff_channels) # How many of each user status? statuses = Counter(member.status for member in ctx.guild.members) @@ -146,7 +186,7 @@ class Information(Cog): **Channel counts** $channel_counts - Staff channels: {hidden_channels} + Staff channels: {staff_channel_count} **Member counts** Members: {member_count:,} -- cgit v1.2.3 From f59e63454ffa582765847e8a26d9d97dcd9ff7b2 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 01:42:02 +0200 Subject: Fix busted test_information test. I wish this test didn't exist. --- tests/bot/cogs/test_information.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/bot/cogs/test_information.py b/tests/bot/cogs/test_information.py index aca6b594f..79c0e0ad3 100644 --- a/tests/bot/cogs/test_information.py +++ b/tests/bot/cogs/test_information.py @@ -148,14 +148,18 @@ class InformationCogTests(unittest.TestCase): Voice region: {self.ctx.guild.region} Features: {', '.join(self.ctx.guild.features)} - **Counts** - Members: {self.ctx.guild.member_count:,} - Roles: {len(self.ctx.guild.roles)} + **Channel counts** Category channels: 1 Text channels: 1 Voice channels: 1 + Staff channels: 0 + + **Member counts** + Members: {self.ctx.guild.member_count:,} + Staff members: 0 + Roles: {len(self.ctx.guild.roles)} - **Members** + **Member statuses** {constants.Emojis.status_online} 2 {constants.Emojis.status_idle} 1 {constants.Emojis.status_dnd} 4 -- cgit v1.2.3 From 96b026198a4ca2074f4fd7ea68e8a09acd5b38e4 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:34:39 +0300 Subject: Simplify infraction reason truncation tests --- tests/bot/cogs/moderation/test_infractions.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 5548d9f68..ad3c95958 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -27,15 +27,14 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): self.cog.apply_infraction = AsyncMock() self.bot.get_cog.return_value = AsyncMock() self.cog.mod_log.ignore = Mock() + self.ctx.guild.ban = Mock() await self.cog.apply_ban(self.ctx, self.target, "foo bar" * 3000) - ban = self.cog.apply_infraction.call_args[0][3] - self.assertEqual( - ban.cr_frame.f_locals["kwargs"]["reason"], - textwrap.shorten("foo bar" * 3000, 512, placeholder="...") + self.ctx.guild.ban.assert_called_once_with( + self.target, + reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="..."), + delete_message_days=0 ) - # Await ban to avoid not awaited coroutine warning - await ban @patch("bot.cogs.moderation.utils.post_infraction") async def test_apply_kick_reason_truncation(self, post_infraction_mock): @@ -44,12 +43,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): self.cog.apply_infraction = AsyncMock() self.cog.mod_log.ignore = Mock() + self.target.kick = Mock() await self.cog.apply_kick(self.ctx, self.target, "foo bar" * 3000) - kick = self.cog.apply_infraction.call_args[0][3] - self.assertEqual( - kick.cr_frame.f_locals["kwargs"]["reason"], - textwrap.shorten("foo bar" * 3000, 512, placeholder="...") - ) - # Await kick to avoid not awaited coroutine warning - await kick + self.target.kick.assert_called_once_with(reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="...")) -- cgit v1.2.3 From 7f827abfa1922a4ec81d2f49fa2811471588269d Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:44:01 +0300 Subject: Scheduler: Move inline f-string if-else statement to normal if statement --- bot/cogs/moderation/scheduler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index b65048f4c..80a58484c 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -174,12 +174,15 @@ class InfractionScheduler(Scheduler): dm_result = f"{constants.Emojis.failmail} " log.trace(f"Deleted infraction {infraction['id']} from database because applying infraction failed.") await self.bot.api_client.delete(f"bot/infractions/{infraction['id']}") + infr_message = "" + else: + infr_message = f"**{infr_type}** to {user.mention}{expiry_msg}{end_msg}" # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} confirmation message.") await ctx.send( f"{dm_result}{confirm_msg} " - f"{f'**{infr_type}** to {user.mention}{expiry_msg}{end_msg}' if not failed else ''}." + f"{infr_message}." ) # Send a log message to the mod log. -- cgit v1.2.3 From 136ef112999b9387f04c3e0b800a3008ac07934f Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:44:36 +0300 Subject: Scheduler: Remove invalid comment --- bot/cogs/moderation/scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 80a58484c..7e8455740 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -84,7 +84,6 @@ class InfractionScheduler(Scheduler): """Apply an infraction to the user, log the infraction, and optionally notify the user.""" infr_type = infraction["type"] icon = utils.INFRACTION_ICONS[infr_type][0] - # Truncate reason when it's too long to avoid raising error on sending ModLog entry reason = infraction["reason"] expiry = time.format_infraction_with_duration(infraction["expires_at"]) id_ = infraction['id'] -- cgit v1.2.3 From f42ddf64abfd487fd69eec275d1011112eb76166 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:53:38 +0300 Subject: Scheduler: Add try-except to infraction deletion --- bot/cogs/moderation/scheduler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 7e8455740..dcc0001f8 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -172,7 +172,12 @@ class InfractionScheduler(Scheduler): dm_log_text = "\nDM: **Canceled**" dm_result = f"{constants.Emojis.failmail} " log.trace(f"Deleted infraction {infraction['id']} from database because applying infraction failed.") - await self.bot.api_client.delete(f"bot/infractions/{infraction['id']}") + try: + await self.bot.api_client.delete(f"bot/infractions/{id_}") + except ResponseCodeError as e: + confirm_msg += f" and failed to delete" + log_title += " and failed to delete" + log.error(f"Deletion of {infr_type} infraction #{id_} failed with error code {e.status}.") infr_message = "" else: infr_message = f"**{infr_type}** to {user.mention}{expiry_msg}{end_msg}" -- cgit v1.2.3 From e71beb79f6f78b348ff17974844806c981da3c2d Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:56:37 +0300 Subject: Scheduler: Remove unnecessary `f` before string --- bot/cogs/moderation/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index dcc0001f8..0d4f0ffba 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -175,7 +175,7 @@ class InfractionScheduler(Scheduler): try: await self.bot.api_client.delete(f"bot/infractions/{id_}") except ResponseCodeError as e: - confirm_msg += f" and failed to delete" + confirm_msg += " and failed to delete" log_title += " and failed to delete" log.error(f"Deletion of {infr_type} infraction #{id_} failed with error code {e.status}.") infr_message = "" -- cgit v1.2.3 From 854b27593e13f1e35810c37f4be91e5c8c4516b2 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:58:14 +0300 Subject: Scheduler: Fix spaces for modlog text Co-authored-by: Mark --- bot/cogs/moderation/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 0d4f0ffba..8b28afa69 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -198,7 +198,7 @@ class InfractionScheduler(Scheduler): thumbnail=user.avatar_url_as(static_format="png"), text=textwrap.dedent(f""" Member: {user.mention} (`{user.id}`) - Actor: {ctx.message.author}{dm_log_text} {expiry_log_text} + Actor: {ctx.message.author}{dm_log_text}{expiry_log_text} Reason: {reason} """), content=log_content, -- cgit v1.2.3 From 323317496310ef474a39d468e273703106e44768 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 10:07:21 +0300 Subject: Infr. Tests: Add `apply_infraction` awaiting assertion with args --- tests/bot/cogs/moderation/test_infractions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index ad3c95958..da4e92ccc 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -35,6 +35,9 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="..."), delete_message_days=0 ) + self.cog.apply_infraction.assert_awaited_once_with( + self.ctx, {"foo": "bar"}, self.target, self.ctx.guild.ban.return_value + ) @patch("bot.cogs.moderation.utils.post_infraction") async def test_apply_kick_reason_truncation(self, post_infraction_mock): @@ -47,3 +50,6 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): await self.cog.apply_kick(self.ctx, self.target, "foo bar" * 3000) self.target.kick.assert_called_once_with(reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="...")) + self.cog.apply_infraction.assert_awaited_once_with( + self.ctx, {"foo": "bar"}, self.target, self.target.kick.return_value + ) -- cgit v1.2.3 From e236113612c560326176da91f5a743c514ac988b Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 10:10:34 +0300 Subject: Scheduler: Remove line splitting from `ctx.send` after 7f827ab --- bot/cogs/moderation/scheduler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 8b28afa69..3679561a3 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -184,10 +184,7 @@ class InfractionScheduler(Scheduler): # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} confirmation message.") - await ctx.send( - f"{dm_result}{confirm_msg} " - f"{infr_message}." - ) + await ctx.send(f"{dm_result}{confirm_msg} {infr_message}.") # Send a log message to the mod log. log.trace(f"Sending apply mod log for infraction #{id_}.") -- cgit v1.2.3 From f1f2c488dc29d731b3343c949fe49cc3eaced842 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 10:17:05 +0300 Subject: Scheduler: Move space from f-string of `ctx.send` to `infr_message` --- bot/cogs/moderation/scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 3679561a3..1c7786df4 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -180,11 +180,11 @@ class InfractionScheduler(Scheduler): log.error(f"Deletion of {infr_type} infraction #{id_} failed with error code {e.status}.") infr_message = "" else: - infr_message = f"**{infr_type}** to {user.mention}{expiry_msg}{end_msg}" + infr_message = f" **{infr_type}** to {user.mention}{expiry_msg}{end_msg}" # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} confirmation message.") - await ctx.send(f"{dm_result}{confirm_msg} {infr_message}.") + await ctx.send(f"{dm_result}{confirm_msg}{infr_message}.") # Send a log message to the mod log. log.trace(f"Sending apply mod log for infraction #{id_}.") -- cgit v1.2.3 From fa3cd5fef8e2f80a85ebc460cffb4e59c8e6387a Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 10:37:41 +0200 Subject: Prevent duplicates, and break into function. - We're using a set comprehension and flipping the order for counting the number of channels that are both staff allow and @everyone deny. - We're breaking the staff channel count stuff into a separate helper function so it doesn't crowd the server_info() scope. These fixes are both to address the code review from @MarkKoz, thanks Mark. --- bot/cogs/information.py | 59 +++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index 715623620..9ebb89300 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -30,7 +30,7 @@ class Information(Cog): @staticmethod def _get_channels_with_role_permission(ctx: Context, role: Role, perm: str, value: Optional[bool]) -> List[int]: - """Get a list of channel IDs where a role has a specific permission set to a specific value.""" + """Get a list of channel IDs where one of the specified roles can read.""" channel_ids = [] for channel in ctx.guild.channels: @@ -56,6 +56,33 @@ class Information(Cog): value=False ) + def _get_staff_channel_count(self, ctx: Context) -> int: + """ + Get number of channels that are staff-only. + + We need to know two things about a channel: + - Does the @everyone role have explicit read deny permissions? + - Do staff roles have explicit read allow permissions? + + If the answer to both of these questions is yes, it's a staff channel. + """ + helpers = ctx.guild.get_role(constants.Roles.helpers) + moderators = ctx.guild.get_role(constants.Roles.moderators) + admins = ctx.guild.get_role(constants.Roles.admins) + everyone = ctx.guild.default_role + + # Let's build some lists of channels. + everyone_denied = self._get_channels_where_role_cannot_read(ctx, everyone) + staff_allowed = more_itertools.flatten([ + self._get_channels_where_role_can_read(ctx, admins), # Admins has explicit read message allow + self._get_channels_where_role_can_read(ctx, moderators), # Moderators has explicit read message allow + self._get_channels_where_role_can_read(ctx, helpers), # Helpers has explicit read message allow + ]) + + # Now we need to check which channels are both denied for @everyone and permitted for staff + staff_channels = set(cid for cid in everyone_denied if cid in staff_allowed) + return len(staff_channels) + @with_role(*constants.MODERATION_ROLES) @command(name="roles") async def roles_info(self, ctx: Context) -> None: @@ -143,35 +170,13 @@ class Information(Cog): channel_type_list = sorted(channel_type_list) channel_counts = "\n".join(channel_type_list).strip() - # How many channels are for staff only? - # We need to know two things about a channel: - # - Does the @everyone role have explicit read deny permissions? - # - Do staff roles have explicit read allow permissions? - # - # If the answer to both of these questions is yes, it's a staff channel. - helpers = ctx.guild.get_role(constants.Roles.helpers) - moderators = ctx.guild.get_role(constants.Roles.moderators) - admins = ctx.guild.get_role(constants.Roles.admins) - everyone = ctx.guild.roles[0] - - # Let's build some lists of channels. - everyone_denied = self._get_channels_where_role_cannot_read(ctx, everyone) - staff_allowed = more_itertools.flatten([ - self._get_channels_where_role_can_read(ctx, admins), # Admins has explicit read message allow - self._get_channels_where_role_can_read(ctx, moderators), # Moderators has explicit read message allow - self._get_channels_where_role_can_read(ctx, helpers), # Helpers has explicit read message allow - ]) - - # Now we need to check which channels are both denied for everyone and permitted for staff - staff_channels = [cid for cid in staff_allowed if cid in everyone_denied] - staff_channel_count = len(staff_channels) - # How many of each user status? statuses = Counter(member.status for member in ctx.guild.members) embed = Embed(colour=Colour.blurple()) - # How many staff members? - staff_members = len(ctx.guild.get_role(constants.Roles.helpers).members) + # How many staff members and staff channels do we have? + staff_member_count = len(ctx.guild.get_role(constants.Roles.helpers).members) + staff_channel_count = self._get_staff_channel_count() # Because channel_counts lacks leading whitespace, it breaks the dedent if it's inserted directly by the # f-string. While this is correctly formated by Discord, it makes unit testing difficult. To keep the formatting @@ -190,7 +195,7 @@ class Information(Cog): **Member counts** Members: {member_count:,} - Staff members: {staff_members} + Staff members: {staff_member_count} Roles: {roles} **Member statuses** -- cgit v1.2.3 From 8562ed2feb6ef465b8b502c725566de4f0a06cbb Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 10:49:38 +0200 Subject: Don't membership check in an itertools.chain. We're using the set comprehension to prevent duplicates anyway, so flipping these back makes more sense. Also added a missing ctx and tested ok. --- bot/cogs/information.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index 9ebb89300..d3a2768d4 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -80,7 +80,7 @@ class Information(Cog): ]) # Now we need to check which channels are both denied for @everyone and permitted for staff - staff_channels = set(cid for cid in everyone_denied if cid in staff_allowed) + staff_channels = set(cid for cid in staff_allowed if cid in everyone_denied) return len(staff_channels) @with_role(*constants.MODERATION_ROLES) @@ -176,7 +176,7 @@ class Information(Cog): # How many staff members and staff channels do we have? staff_member_count = len(ctx.guild.get_role(constants.Roles.helpers).members) - staff_channel_count = self._get_staff_channel_count() + staff_channel_count = self._get_staff_channel_count(ctx) # Because channel_counts lacks leading whitespace, it breaks the dedent if it's inserted directly by the # f-string. While this is correctly formated by Discord, it makes unit testing difficult. To keep the formatting -- cgit v1.2.3 From 4bbb5b127315727b1534c5a0e77bfdd48173847b Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sat, 30 May 2020 21:05:13 +0200 Subject: Free tag: link #how-to-get-help This creates a clickable link in the response embed. Referencing the category is no longer necessary. --- bot/resources/tags/free.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/resources/tags/free.md b/bot/resources/tags/free.md index 582cca9da..1493076c7 100644 --- a/bot/resources/tags/free.md +++ b/bot/resources/tags/free.md @@ -1,5 +1,5 @@ **We have a new help channel system!** -We recently moved to a new help channel system. You can now use any channel in the **<#691405807388196926>** category to ask your question. +Please see <#704250143020417084> for further information. -For more information, check out [our website](https://pythondiscord.com/pages/resources/guides/help-channels/). +A more detailed guide can be found on [our website](https://pythondiscord.com/pages/resources/guides/help-channels/). -- cgit v1.2.3 From 4549fa3defb7b9aba22505b438493bf03e74378d Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 30 May 2020 12:43:11 -0700 Subject: Simplify counting of staff channels and improve efficiency Simplification comes from being able to access permissions as attributes on the overwrite object. This removes the need to iterate all permissions. Efficiency comes from checking all roles within a single iteration of all channels. This also removes the need to flatten and filter the channels afterwards, which required additional iterations. --- bot/cogs/information.py | 75 +++++++++++++++++-------------------------------- 1 file changed, 26 insertions(+), 49 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index d3a2768d4..887c7c127 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -1,14 +1,13 @@ import colorsys -import functools import logging import pprint import textwrap from collections import Counter, defaultdict from string import Template -from typing import Any, List, Mapping, Optional, Union +from typing import Any, Mapping, Optional, Union -import more_itertools -from discord import Colour, Embed, Member, Message, Role, Status, utils +from discord import ChannelType, Colour, Embed, Guild, Member, Message, Role, Status, utils +from discord.abc import GuildChannel from discord.ext.commands import BucketType, Cog, Context, Paginator, command, group from discord.utils import escape_markdown @@ -29,36 +28,14 @@ class Information(Cog): self.bot = bot @staticmethod - def _get_channels_with_role_permission(ctx: Context, role: Role, perm: str, value: Optional[bool]) -> List[int]: - """Get a list of channel IDs where one of the specified roles can read.""" - channel_ids = [] + def role_can_read(channel: GuildChannel, role: Role) -> bool: + """Return True if `role` can read messages in `channel`.""" + overwrites = channel.overwrites_for(role) + return overwrites.read_messages is True - for channel in ctx.guild.channels: - overwrites = channel.overwrites_for(role) - if overwrites.is_empty(): - continue - - for _perm, _value in overwrites: - if _perm == perm and _value is value: - channel_ids.append(channel.id) - - return channel_ids - - _get_channels_where_role_can_read = functools.partialmethod( - _get_channels_with_role_permission, - perm='read_messages', - value=True - ) - - _get_channels_where_role_cannot_read = functools.partialmethod( - _get_channels_with_role_permission, - perm='read_messages', - value=False - ) - - def _get_staff_channel_count(self, ctx: Context) -> int: + def get_staff_channel_count(self, guild: Guild) -> int: """ - Get number of channels that are staff-only. + Get the number of channels that are staff-only. We need to know two things about a channel: - Does the @everyone role have explicit read deny permissions? @@ -66,22 +43,22 @@ class Information(Cog): If the answer to both of these questions is yes, it's a staff channel. """ - helpers = ctx.guild.get_role(constants.Roles.helpers) - moderators = ctx.guild.get_role(constants.Roles.moderators) - admins = ctx.guild.get_role(constants.Roles.admins) - everyone = ctx.guild.default_role - - # Let's build some lists of channels. - everyone_denied = self._get_channels_where_role_cannot_read(ctx, everyone) - staff_allowed = more_itertools.flatten([ - self._get_channels_where_role_can_read(ctx, admins), # Admins has explicit read message allow - self._get_channels_where_role_can_read(ctx, moderators), # Moderators has explicit read message allow - self._get_channels_where_role_can_read(ctx, helpers), # Helpers has explicit read message allow - ]) - - # Now we need to check which channels are both denied for @everyone and permitted for staff - staff_channels = set(cid for cid in staff_allowed if cid in everyone_denied) - return len(staff_channels) + channel_ids = set() + for channel in guild.channels: + if channel.type is ChannelType.category: + continue + + if channel in channel_ids: + continue # Only one of the roles has to have read permissions, not all + + everyone_can_read = self.role_can_read(channel, guild.default_role) + + for role in constants.STAFF_ROLES: + role_can_read = self.role_can_read(channel, guild.get_role(role)) + if role_can_read and everyone_can_read is False: + channel_ids.add(channel.id) + + return len(channel_ids) @with_role(*constants.MODERATION_ROLES) @command(name="roles") @@ -176,7 +153,7 @@ class Information(Cog): # How many staff members and staff channels do we have? staff_member_count = len(ctx.guild.get_role(constants.Roles.helpers).members) - staff_channel_count = self._get_staff_channel_count(ctx) + staff_channel_count = self.get_staff_channel_count(ctx.guild) # Because channel_counts lacks leading whitespace, it breaks the dedent if it's inserted directly by the # f-string. While this is correctly formated by Discord, it makes unit testing difficult. To keep the formatting -- cgit v1.2.3 From 8c6219cd668c814f945418000c6df896de581dc1 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 30 May 2020 12:54:22 -0700 Subject: Move counting of channels to a separate method This de-clutters the main `server_info` function and improves its readability. --- bot/cogs/information.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index 887c7c127..7c39dce5f 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -60,6 +60,18 @@ class Information(Cog): return len(channel_ids) + @staticmethod + def get_channel_type_counts(guild: Guild) -> str: + """Return the total amounts of the various types of channels in `guild`.""" + channel_counter = Counter(c.type for c in guild.channels) + channel_type_list = [] + for channel in channel_counter: + channel_type = str(channel).title() + channel_type_list.append(f"{channel_type} channels: {channel_counter[channel]}") + + channel_type_list = sorted(channel_type_list) + return "\n".join(channel_type_list).strip() + @with_role(*constants.MODERATION_ROLES) @command(name="roles") async def roles_info(self, ctx: Context) -> None: @@ -136,16 +148,7 @@ class Information(Cog): roles = len(ctx.guild.roles) member_count = ctx.guild.member_count - - # How many of each type of channel? - channel_counter = Counter(c.type for c in ctx.guild.channels) - channel_type_list = [] - for channel in channel_counter: - channel_type = str(channel).title() - channel_type_list.append(f"{channel_type} channels: {channel_counter[channel]}") - - channel_type_list = sorted(channel_type_list) - channel_counts = "\n".join(channel_type_list).strip() + channel_counts = self.get_channel_type_counts(ctx.guild) # How many of each user status? statuses = Counter(member.status for member in ctx.guild.members) -- cgit v1.2.3 From 795dea3c8030955736984cdab372595c4799f5e9 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 22:36:34 +0200 Subject: Add multichannel !purge via commands.Greedy We can now pass in as many channel mentions as we want after any !purge command - for example `!purge all 5 #python-general #python-language` --- bot/cogs/clean.py | 70 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index b5d9132cb..91e69ee89 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -1,9 +1,10 @@ import logging import random import re -from typing import Optional +from typing import Iterable, Optional from discord import Colour, Embed, Message, TextChannel, User +from discord.ext import commands from discord.ext.commands import Cog, Context, group from bot.bot import Bot @@ -41,10 +42,11 @@ class Clean(Cog): self, amount: int, ctx: Context, + channels: Iterable[TextChannel], bots_only: bool = False, user: User = None, regex: Optional[str] = None, - channel: Optional[TextChannel] = None + ) -> None: """A helper function that does the actual message cleaning.""" def predicate_bots_only(message: Message) -> bool: @@ -110,8 +112,8 @@ class Clean(Cog): predicate = None # Delete all messages # Default to using the invoking context's channel - if not channel: - channel = ctx.channel + if not channels: + channels = [ctx.channel] # Look through the history and retrieve message data messages = [] @@ -120,23 +122,24 @@ class Clean(Cog): invocation_deleted = False # To account for the invocation message, we index `amount + 1` messages. - async for message in channel.history(limit=amount + 1): + for channel in channels: + async for message in channel.history(limit=amount + 1): - # If at any point the cancel command is invoked, we should stop. - if not self.cleaning: - return + # If at any point the cancel command is invoked, we should stop. + if not self.cleaning: + return - # Always start by deleting the invocation - if not invocation_deleted: - self.mod_log.ignore(Event.message_delete, message.id) - await message.delete() - invocation_deleted = True - continue + # Always start by deleting the invocation + if not invocation_deleted: + self.mod_log.ignore(Event.message_delete, message.id) + await message.delete() + invocation_deleted = True + continue - # If the message passes predicate, let's save it. - if predicate is None or predicate(message): - message_ids.append(message.id) - messages.append(message) + # If the message passes predicate, let's save it. + if predicate is None or predicate(message): + message_ids.append(message.id) + messages.append(message) self.cleaning = False @@ -144,10 +147,11 @@ class Clean(Cog): self.mod_log.ignore(Event.message_delete, *message_ids) # Use bulk delete to actually do the cleaning. It's far faster. - await channel.purge( - limit=amount, - check=predicate - ) + for channel in channels: + await channel.purge( + limit=amount, + check=predicate + ) # Reverse the list to restore chronological order if messages: @@ -163,8 +167,12 @@ class Clean(Cog): return # Build the embed and send it + if len(channels) > 1: + target_channels = ", ".join([f"<#{channel.id}>" for channel in channels]) + else: + target_channels = f"<#{channels[0].id}>" message = ( - f"**{len(message_ids)}** messages deleted in <#{channel.id}> by **{ctx.author.name}**\n\n" + f"**{len(message_ids)}** messages deleted in {target_channels} by **{ctx.author.name}**\n\n" f"A log of the deleted messages can be found [here]({log_url})." ) @@ -189,10 +197,10 @@ class Clean(Cog): ctx: Context, user: User, amount: Optional[int] = 10, - channel: TextChannel = None + channels: commands.Greedy[TextChannel] = None ) -> None: """Delete messages posted by the provided user, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, user=user, channel=channel) + await self._clean_messages(amount, ctx, user=user, channels=channels) @clean_group.command(name="all", aliases=["everything"]) @with_role(*MODERATION_ROLES) @@ -200,10 +208,10 @@ class Clean(Cog): self, ctx: Context, amount: Optional[int] = 10, - channel: TextChannel = None + channels: commands.Greedy[TextChannel] = None ) -> None: """Delete all messages, regardless of poster, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, channel=channel) + await self._clean_messages(amount, ctx, channels=channels) @clean_group.command(name="bots", aliases=["bot"]) @with_role(*MODERATION_ROLES) @@ -211,10 +219,10 @@ class Clean(Cog): self, ctx: Context, amount: Optional[int] = 10, - channel: TextChannel = None + channels: commands.Greedy[TextChannel] = None ) -> None: """Delete all messages posted by a bot, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, bots_only=True, channel=channel) + await self._clean_messages(amount, ctx, bots_only=True, channels=channels) @clean_group.command(name="regex", aliases=["word", "expression"]) @with_role(*MODERATION_ROLES) @@ -223,10 +231,10 @@ class Clean(Cog): ctx: Context, regex: str, amount: Optional[int] = 10, - channel: TextChannel = None + channels: commands.Greedy[TextChannel] = None ) -> None: """Delete all messages that match a certain regex, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, regex=regex, channel=channel) + await self._clean_messages(amount, ctx, regex=regex, channels=channels) @clean_group.command(name="stop", aliases=["cancel", "abort"]) @with_role(*MODERATION_ROLES) -- cgit v1.2.3 From 5926f75c8d2d4b683139606bd0a39b07d28529e1 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 22:43:22 +0200 Subject: Remove a completely unacceptable newline. --- bot/cogs/clean.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index 91e69ee89..8b0b8ed05 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -46,7 +46,6 @@ class Clean(Cog): bots_only: bool = False, user: User = None, regex: Optional[str] = None, - ) -> None: """A helper function that does the actual message cleaning.""" def predicate_bots_only(message: Message) -> bool: -- cgit v1.2.3 From 1cc1b3851871dfff5690432960ade09fe8ab5794 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 23:14:21 +0200 Subject: Oops, add the return back. We do not wanna process bot messages. --- bot/cogs/verification.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 0a087cee9..ae156cf70 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -55,6 +55,7 @@ class Verification(Cog): if message.author.bot: # They're a bot, delete their message after the delay. await message.delete(delay=BOT_MESSAGE_DELETE_DELAY) + return # if a user mentions a role or guild member # alert the mods in mod-alerts channel -- cgit v1.2.3 From d7123487230d70b855c84fb5d99ec45f6bee6859 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 31 May 2020 18:18:18 +0200 Subject: Better channel mentions Co-authored-by: Mark --- bot/cogs/clean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index 8b0b8ed05..571a5ced8 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -167,7 +167,7 @@ class Clean(Cog): # Build the embed and send it if len(channels) > 1: - target_channels = ", ".join([f"<#{channel.id}>" for channel in channels]) + target_channels = ", ".join(channel.mention for channel in channels) else: target_channels = f"<#{channels[0].id}>" message = ( -- cgit v1.2.3 From d637053eb19a6bf33e765b25b3dff9963d7b7735 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 31 May 2020 18:19:36 +0200 Subject: Remove unnecessary conditional. Thanks @MarkKoz! --- bot/cogs/clean.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index 571a5ced8..02216a4af 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -166,10 +166,8 @@ class Clean(Cog): return # Build the embed and send it - if len(channels) > 1: - target_channels = ", ".join(channel.mention for channel in channels) - else: - target_channels = f"<#{channels[0].id}>" + target_channels = ", ".join(channel.mention for channel in channels) + message = ( f"**{len(message_ids)}** messages deleted in {target_channels} by **{ctx.author.name}**\n\n" f"A log of the deleted messages can be found [here]({log_url})." -- cgit v1.2.3 From 0737b1a63ca359e88ef580143e8e4e6a879c482e Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 31 May 2020 18:34:36 +0200 Subject: Add a mod_log.ignore_all context manager. This new context manager makes it easier to make the mod_log ignore actions like message deletions. The only existing method is the `ignore()` method, which requires that you pass all the messages you want to ignore into it. This one just ignores everything inside its scope. This isn't the DRYest approach, but it's low-cost and improves the readability of clean.py quite a bit. Ideally we should go through and give modlog a proper cleanup, because it's kinda ugly right now. --- bot/cogs/moderation/modlog.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index 9d28030d9..b3ae8e215 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -3,6 +3,7 @@ import difflib import itertools import logging import typing as t +from contextlib import contextmanager from datetime import datetime from itertools import zip_longest @@ -40,6 +41,7 @@ class ModLog(Cog, name="ModLog"): def __init__(self, bot: Bot): self.bot = bot self._ignored = {event: [] for event in Event} + self._ignore_all = False self._cached_deletes = [] self._cached_edits = [] @@ -81,6 +83,15 @@ class ModLog(Cog, name="ModLog"): if item not in self._ignored[event]: self._ignored[event].append(item) + @contextmanager + def ignore_all(self) -> None: + """Ignore all events while inside this context scope.""" + self._ignore_all = True + try: + yield + finally: + self._ignore_all = False + async def send_log_message( self, icon_url: t.Optional[str], @@ -191,6 +202,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.guild_channel_update].remove(before.id) return + if self._ignore_all: + return + # Two channel updates are sent for a single edit: 1 for topic and 1 for category change. # TODO: remove once support is added for ignoring multiple occurrences for the same channel. help_categories = (Categories.help_available, Categories.help_dormant, Categories.help_in_use) @@ -386,6 +400,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_ban].remove(member.id) return + if self._ignore_all: + return + await self.send_log_message( Icons.user_ban, Colours.soft_red, "User banned", f"{member} (`{member.id}`)", @@ -426,6 +443,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_remove].remove(member.id) return + if self._ignore_all: + return + member_str = escape_markdown(str(member)) await self.send_log_message( Icons.sign_out, Colours.soft_red, @@ -444,6 +464,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_unban].remove(member.id) return + if self._ignore_all: + return + member_str = escape_markdown(str(member)) await self.send_log_message( Icons.user_unban, Colour.blurple(), @@ -462,6 +485,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_update].remove(before.id) return + if self._ignore_all: + return + diff = DeepDiff(before, after) changes = [] done = [] @@ -564,6 +590,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.message_delete].remove(message.id) return + if self._ignore_all: + return + if author.bot: return @@ -623,6 +652,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.message_delete].remove(event.message_id) return + if self._ignore_all: + return + channel = self.bot.get_channel(event.channel_id) if channel.category: @@ -797,6 +829,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.voice_state_update].remove(member.id) return + if self._ignore_all: + return + # Exclude all channel attributes except the name. diff = DeepDiff( before, -- cgit v1.2.3 From f344dd8a72024e05577a5aeba25e2f98501417af Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 31 May 2020 18:37:56 +0200 Subject: Fix a bug with invocation deletion. This command was written to support only a single channel, and with the move to multi-channel purges, we need to rethink the way the invocation deletion happens. We may be invoking this command from a completely different channel, so we can't necessarily look inside the channels we're targeting for the invocation. So, we're solving this by just deleting the invocation by using ctx.message. We do this before we start iterating message history, and then we only need to iterate the number of messages that was passed into the command. A much cleaner approach, which solves the bug reported and identified by @MarkKoz. --- bot/cogs/clean.py | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index 02216a4af..892c638b8 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -10,8 +10,7 @@ from discord.ext.commands import Cog, Context, group from bot.bot import Bot from bot.cogs.moderation import ModLog from bot.constants import ( - Channels, CleanMessages, Colours, Event, - Icons, MODERATION_ROLES, NEGATIVE_REPLIES + Channels, CleanMessages, Colours, Icons, MODERATION_ROLES, NEGATIVE_REPLIES ) from bot.decorators import with_role @@ -114,27 +113,23 @@ class Clean(Cog): if not channels: channels = [ctx.channel] + # Delete the invocation first + with self.mod_log.ignore_all(): + await ctx.message.delete() + # Look through the history and retrieve message data + # This is only done so we can create a log to upload. messages = [] message_ids = [] self.cleaning = True - invocation_deleted = False - # To account for the invocation message, we index `amount + 1` messages. for channel in channels: - async for message in channel.history(limit=amount + 1): + async for message in channel.history(limit=amount): # If at any point the cancel command is invoked, we should stop. if not self.cleaning: return - # Always start by deleting the invocation - if not invocation_deleted: - self.mod_log.ignore(Event.message_delete, message.id) - await message.delete() - invocation_deleted = True - continue - # If the message passes predicate, let's save it. if predicate is None or predicate(message): message_ids.append(message.id) @@ -142,15 +137,13 @@ class Clean(Cog): self.cleaning = False - # We should ignore the ID's we stored, so we don't get mod-log spam. - self.mod_log.ignore(Event.message_delete, *message_ids) - - # Use bulk delete to actually do the cleaning. It's far faster. - for channel in channels: - await channel.purge( - limit=amount, - check=predicate - ) + # Now let's delete the actual messages with purge. + with self.mod_log.ignore_all(): + for channel in channels: + await channel.purge( + limit=amount, + check=predicate + ) # Reverse the list to restore chronological order if messages: -- cgit v1.2.3 From b225791c917b582fcfbed0aee30b2cf7d3fd9ac4 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 31 May 2020 18:50:16 +0200 Subject: Fix a bad check in get_staff_channel_count. This also changes a few aesthetic problems pointed out in review by @MarkKoz and @kwzrd. --- bot/cogs/information.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/bot/cogs/information.py b/bot/cogs/information.py index 7c39dce5f..f0bd1afdb 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -48,15 +48,13 @@ class Information(Cog): if channel.type is ChannelType.category: continue - if channel in channel_ids: - continue # Only one of the roles has to have read permissions, not all - everyone_can_read = self.role_can_read(channel, guild.default_role) for role in constants.STAFF_ROLES: role_can_read = self.role_can_read(channel, guild.get_role(role)) - if role_can_read and everyone_can_read is False: + if role_can_read and not everyone_can_read: channel_ids.add(channel.id) + break return len(channel_ids) @@ -65,12 +63,12 @@ class Information(Cog): """Return the total amounts of the various types of channels in `guild`.""" channel_counter = Counter(c.type for c in guild.channels) channel_type_list = [] - for channel in channel_counter: + for channel, count in channel_counter.items(): channel_type = str(channel).title() - channel_type_list.append(f"{channel_type} channels: {channel_counter[channel]}") + channel_type_list.append(f"{channel_type} channels: {count}") channel_type_list = sorted(channel_type_list) - return "\n".join(channel_type_list).strip() + return "\n".join(channel_type_list) @with_role(*constants.MODERATION_ROLES) @command(name="roles") -- cgit v1.2.3 From 3139991c3dcf4ec981a49aefa3d3cd75eed93fd8 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 31 May 2020 23:33:05 +0200 Subject: Revert "Add a mod_log.ignore_all context manager." This reverts commit 0737b1a6 This isn't gonna work, because async is a thing. --- bot/cogs/moderation/modlog.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index b3ae8e215..9d28030d9 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -3,7 +3,6 @@ import difflib import itertools import logging import typing as t -from contextlib import contextmanager from datetime import datetime from itertools import zip_longest @@ -41,7 +40,6 @@ class ModLog(Cog, name="ModLog"): def __init__(self, bot: Bot): self.bot = bot self._ignored = {event: [] for event in Event} - self._ignore_all = False self._cached_deletes = [] self._cached_edits = [] @@ -83,15 +81,6 @@ class ModLog(Cog, name="ModLog"): if item not in self._ignored[event]: self._ignored[event].append(item) - @contextmanager - def ignore_all(self) -> None: - """Ignore all events while inside this context scope.""" - self._ignore_all = True - try: - yield - finally: - self._ignore_all = False - async def send_log_message( self, icon_url: t.Optional[str], @@ -202,9 +191,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.guild_channel_update].remove(before.id) return - if self._ignore_all: - return - # Two channel updates are sent for a single edit: 1 for topic and 1 for category change. # TODO: remove once support is added for ignoring multiple occurrences for the same channel. help_categories = (Categories.help_available, Categories.help_dormant, Categories.help_in_use) @@ -400,9 +386,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_ban].remove(member.id) return - if self._ignore_all: - return - await self.send_log_message( Icons.user_ban, Colours.soft_red, "User banned", f"{member} (`{member.id}`)", @@ -443,9 +426,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_remove].remove(member.id) return - if self._ignore_all: - return - member_str = escape_markdown(str(member)) await self.send_log_message( Icons.sign_out, Colours.soft_red, @@ -464,9 +444,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_unban].remove(member.id) return - if self._ignore_all: - return - member_str = escape_markdown(str(member)) await self.send_log_message( Icons.user_unban, Colour.blurple(), @@ -485,9 +462,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_update].remove(before.id) return - if self._ignore_all: - return - diff = DeepDiff(before, after) changes = [] done = [] @@ -590,9 +564,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.message_delete].remove(message.id) return - if self._ignore_all: - return - if author.bot: return @@ -652,9 +623,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.message_delete].remove(event.message_id) return - if self._ignore_all: - return - channel = self.bot.get_channel(event.channel_id) if channel.category: @@ -829,9 +797,6 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.voice_state_update].remove(member.id) return - if self._ignore_all: - return - # Exclude all channel attributes except the name. diff = DeepDiff( before, -- cgit v1.2.3 From 345fda6b88fef50e9bc47298085a10d8acb4fdff Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 31 May 2020 23:36:28 +0200 Subject: Revert message ignore approach. We're removing the context manager due to async concerns, so we'll go back to the old approach again of ignoring specific messages and iterating history. --- bot/cogs/clean.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index 892c638b8..b164cf232 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -10,7 +10,7 @@ from discord.ext.commands import Cog, Context, group from bot.bot import Bot from bot.cogs.moderation import ModLog from bot.constants import ( - Channels, CleanMessages, Colours, Icons, MODERATION_ROLES, NEGATIVE_REPLIES + Channels, CleanMessages, Colours, Event, Icons, MODERATION_ROLES, NEGATIVE_REPLIES ) from bot.decorators import with_role @@ -114,11 +114,10 @@ class Clean(Cog): channels = [ctx.channel] # Delete the invocation first - with self.mod_log.ignore_all(): - await ctx.message.delete() + self.mod_log.ignore(Event.message_delete, ctx.message.id) + await ctx.message.delete() # Look through the history and retrieve message data - # This is only done so we can create a log to upload. messages = [] message_ids = [] self.cleaning = True @@ -138,12 +137,12 @@ class Clean(Cog): self.cleaning = False # Now let's delete the actual messages with purge. - with self.mod_log.ignore_all(): - for channel in channels: - await channel.purge( - limit=amount, - check=predicate - ) + self.mod_log.ignore(Event.message_delete, *message_ids) + for channel in channels: + await channel.purge( + limit=amount, + check=predicate + ) # Reverse the list to restore chronological order if messages: -- cgit v1.2.3 From 196ce8a828a0fed7450cad1ee0bba25ef608214a Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 31 May 2020 15:27:53 -0700 Subject: Use the messages returned by `purge` to upload message logs This ensures that only what was actually deleted will be uploaded. I managed to get a 400 response from our API when purging twice in quick succession. Searching the history manually for these messages is unreliable cause of some sort of race condition. --- bot/cogs/clean.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index b164cf232..368d91c85 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -117,11 +117,11 @@ class Clean(Cog): self.mod_log.ignore(Event.message_delete, ctx.message.id) await ctx.message.delete() - # Look through the history and retrieve message data messages = [] message_ids = [] self.cleaning = True + # Find the IDs of the messages to delete. IDs are needed in order to ignore mod log events. for channel in channels: async for message in channel.history(limit=amount): @@ -132,21 +132,17 @@ class Clean(Cog): # If the message passes predicate, let's save it. if predicate is None or predicate(message): message_ids.append(message.id) - messages.append(message) self.cleaning = False # Now let's delete the actual messages with purge. self.mod_log.ignore(Event.message_delete, *message_ids) for channel in channels: - await channel.purge( - limit=amount, - check=predicate - ) + messages += await channel.purge(limit=amount, check=predicate) # Reverse the list to restore chronological order if messages: - messages = list(reversed(messages)) + messages = reversed(messages) log_url = await self.mod_log.upload_log(messages, ctx.author.id) else: # Can't build an embed, nothing to clean! -- cgit v1.2.3 From 629817eaa87d869cc7857d5bde48d53cce6bcdc0 Mon Sep 17 00:00:00 2001 From: Rasmus Moorats Date: Tue, 2 Jun 2020 17:41:13 +0300 Subject: add modmail tag --- bot/resources/tags/modmail.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 bot/resources/tags/modmail.md diff --git a/bot/resources/tags/modmail.md b/bot/resources/tags/modmail.md new file mode 100644 index 000000000..7545419ee --- /dev/null +++ b/bot/resources/tags/modmail.md @@ -0,0 +1,9 @@ +**Contacting the moderation team via ModMail** + +<@!683001325440860340> is a bot that will relay your messages to our moderation team, so that you can start a conversation with the moderation team. Your messages will be relayed to the entire moderator team, who will be able to respond to you via the bot. + +It supports attachments, codeblocks, and reactions. As communication happens over direct messages, the conversation will stay between you and the mod team. + +**To use it, simply send a direct message to the bot.** + +Should there be an urgent and immediate need for a moderator or admin to look at a channel, feel free to ping the <@&267629731250176001> or <@&267628507062992896> role instead. -- cgit v1.2.3 From 758edf044bfc24aeb8e00c8e244a770c6a247d42 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 4 Jun 2020 23:14:51 -0700 Subject: Fix AttributeError for category check Not all channels will have a category attribute. This may be fine in production, but it does cause periodic errors when testing locally. --- bot/cogs/stats.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py index 4ebb6423c..d42f55466 100644 --- a/bot/cogs/stats.py +++ b/bot/cogs/stats.py @@ -36,7 +36,8 @@ class Stats(Cog): if message.guild.id != Guild.id: return - if message.channel.category.id == Categories.modmail: + cat = getattr(message.channel, "category", None) + if cat is not None and cat.id == Categories.modmail: if message.channel.id != Channels.incidents: # Do not report modmail channels to stats, there are too many # of them for interesting statistics to be drawn out of this. -- cgit v1.2.3 From 3c305dadf7ae745fcf2ba9375d577ce750408fd3 Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff Date: Fri, 5 Jun 2020 14:55:41 +0200 Subject: Send infraction DM before applying infraction I've "reverted" the change that reversed the order of DM'ing a user about their infraction and applying the actual infraction. A recent PR reversed the order to stop us from sending DMs when applying the infraction failed. However, in order to DM a user, the bot has to share a guild with the recipient and kicking them off of our server first does not help with that. That's why I reverted the change and reverted some other minor changes made in relation to this change. Note: I did not change the code sending the DM itself; I merely moved it back to where it belongs and added a comment about the necessity of doing the DM'ing first. I couldn't cleanly revert a commit to do this, as changes were spread out over and included in multiple commits that also contained changes not related to the `DM->apply infraction` order. --- bot/cogs/moderation/scheduler.py | 41 ++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index f0a3ad1b1..b03d89537 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -106,6 +106,27 @@ class InfractionScheduler(Scheduler): log_content = None failed = False + # DM the user about the infraction if it's not a shadow/hidden infraction. + # This needs to happen before we apply the infraction, as the bot cannot + # send DMs to user that it doesn't share a guild with. If we were to + # apply kick/ban infractions first, this would mean that we'd make it + # impossible for us to deliver a DM. See python-discord/bot#982. + if not infraction["hidden"]: + dm_result = f"{constants.Emojis.failmail} " + dm_log_text = "\nDM: **Failed**" + + # Sometimes user is a discord.Object; make it a proper user. + try: + if not isinstance(user, (discord.Member, discord.User)): + user = await self.bot.fetch_user(user.id) + except discord.HTTPException as e: + log.error(f"Failed to DM {user.id}: could not fetch user (status {e.status})") + else: + # Accordingly display whether the user was successfully notified via DM. + if await utils.notify_infraction(user, infr_type, expiry, reason, icon): + dm_result = ":incoming_envelope: " + dm_log_text = "\nDM: Sent" + if infraction["actor"] == self.bot.user.id: log.trace( f"Infraction #{id_} actor is bot; including the reason in the confirmation message." @@ -150,27 +171,7 @@ class InfractionScheduler(Scheduler): log.exception(log_msg) failed = True - # DM the user about the infraction if it's not a shadow/hidden infraction. - # Don't send DM when applying failed. - if not infraction["hidden"] and not failed: - dm_result = f"{constants.Emojis.failmail} " - dm_log_text = "\nDM: **Failed**" - - # Sometimes user is a discord.Object; make it a proper user. - try: - if not isinstance(user, (discord.Member, discord.User)): - user = await self.bot.fetch_user(user.id) - except discord.HTTPException as e: - log.error(f"Failed to DM {user.id}: could not fetch user (status {e.status})") - else: - # Accordingly display whether the user was successfully notified via DM. - if await utils.notify_infraction(user, infr_type, expiry, reason, icon): - dm_result = ":incoming_envelope: " - dm_log_text = "\nDM: Sent" - if failed: - dm_log_text = "\nDM: **Canceled**" - dm_result = f"{constants.Emojis.failmail} " log.trace(f"Deleted infraction {infraction['id']} from database because applying infraction failed.") try: await self.bot.api_client.delete(f"bot/infractions/{id_}") -- cgit v1.2.3 From 5801480f0a98aabcbc13e7ee1d90d2f226d9e2a9 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 5 Jun 2020 16:16:50 +0300 Subject: Stats: Implement rules stats Increase every shown rule uses count when command called. --- bot/cogs/site.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot/cogs/site.py b/bot/cogs/site.py index 7fc2a9c34..e61cd5003 100644 --- a/bot/cogs/site.py +++ b/bot/cogs/site.py @@ -133,6 +133,9 @@ class Site(Cog): await ctx.send(f":x: Invalid rule indices: {indices}") return + for rule in rules: + self.bot.stats.incr(f"rule_uses.{rule}") + final_rules = tuple(f"**{pick}.** {full_rules[pick - 1]}" for pick in rules) await LinePaginator.paginate(final_rules, ctx, rules_embed, max_lines=3) -- cgit v1.2.3 From 93be87cea7cde7333042e2bb9529867723f567a7 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 6 Jun 2020 12:49:50 +0200 Subject: Enable the 'redis' / 'aiohttp' Sentry integrations This will provide breadcrumbs for these systems in all our Sentry events, if applicable. Closes #989. --- bot/__main__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bot/__main__.py b/bot/__main__.py index aa1d1aee8..4e0d4a111 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -3,7 +3,9 @@ import logging import discord import sentry_sdk from discord.ext.commands import when_mentioned_or +from sentry_sdk.integrations.aiohttp import AioHttpIntegration from sentry_sdk.integrations.logging import LoggingIntegration +from sentry_sdk.integrations.redis import RedisIntegration from bot import constants, patches from bot.bot import Bot @@ -15,7 +17,11 @@ sentry_logging = LoggingIntegration( sentry_sdk.init( dsn=constants.Bot.sentry_dsn, - integrations=[sentry_logging] + integrations=[ + sentry_logging, + AioHttpIntegration(), + RedisIntegration(), + ] ) bot = Bot( -- cgit v1.2.3 From 290f0982be7bf0f0a709d2c65bee413b11430ba3 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Tue, 9 Jun 2020 01:29:02 +0100 Subject: Add Python Atlanta to guild whitelists --- config-default.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config-default.yml b/config-default.yml index 2c85f5ef3..3a1bdae54 100644 --- a/config-default.yml +++ b/config-default.yml @@ -297,6 +297,7 @@ filter: - 613425648685547541 # Discord Developers - 185590609631903755 # Blender Hub - 420324994703163402 # /r/FlutterDev + - 488751051629920277 # Python Atlanta domain_blacklist: - pornhub.com -- cgit v1.2.3