From ce5fcdab852600342fe69211b038426ce2821107 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 4 Apr 2020 14:51:13 +0300 Subject: (Banning): Added logging and truncating to correct length for Discord Audit Log when ban reason length is more than 512 characters. --- bot/cogs/moderation/infractions.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index efa19f59e..f41484711 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -244,7 +244,14 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - action = ctx.guild.ban(user, reason=reason, delete_message_days=0) + if len(reason) > 512: + log.info("Ban reason is longer than 512 characters. Reason will be truncated for Audit Log.") + + action = ctx.guild.ban( + user, + reason=f"{reason[:509]}..." if len(reason) > 512 else reason, + delete_message_days=0 + ) await self.apply_infraction(ctx, infraction, user, action) if infraction.get('expires_at') is not None: -- cgit v1.2.3 From 6ad86ace9f34245180aefeed9553c407602602e8 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 4 Apr 2020 14:53:11 +0300 Subject: (Kick Command): Added logging and truncating to correct length for Discord Audit Log when kick reason length is more than 512 characters. --- bot/cogs/moderation/infractions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index f41484711..f8c3e8da3 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -225,7 +225,10 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - action = user.kick(reason=reason) + if len(reason) > 512: + log.info("Kick reason is longer than 512 characters. Reason will be truncated for Audit Log.") + + action = user.kick(reason=f"{reason[:509]}..." if len(reason) > 512 else reason) await self.apply_infraction(ctx, infraction, user, action) @respect_role_hierarchy() -- cgit v1.2.3 From 80e483a7ebaf57e6544429343c94cf0eed8821ef Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 4 Apr 2020 16:03:58 +0300 Subject: (Ban and Kick): Replaced force reason truncating with `textwrap.shorten`. --- bot/cogs/moderation/infractions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index f8c3e8da3..a0bdf0d97 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -1,4 +1,5 @@ import logging +import textwrap import typing as t import discord @@ -228,7 +229,7 @@ class Infractions(InfractionScheduler, commands.Cog): if len(reason) > 512: log.info("Kick reason is longer than 512 characters. Reason will be truncated for Audit Log.") - action = user.kick(reason=f"{reason[:509]}..." if len(reason) > 512 else reason) + action = user.kick(textwrap.shorten(reason, width=509, placeholder="...") if len(reason) > 512 else reason) await self.apply_infraction(ctx, infraction, user, action) @respect_role_hierarchy() @@ -252,7 +253,7 @@ class Infractions(InfractionScheduler, commands.Cog): action = ctx.guild.ban( user, - reason=f"{reason[:509]}..." if len(reason) > 512 else reason, + reason=textwrap.shorten(reason, width=509, placeholder="...") if len(reason) > 512 else reason, delete_message_days=0 ) await self.apply_infraction(ctx, infraction, user, action) -- cgit v1.2.3 From ec8cc8b02b0823deaa4ea2c97801d16d6aef5244 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sun, 5 Apr 2020 18:13:38 +0300 Subject: (Ban and Kick): Applied simplification to reason truncating. --- bot/cogs/moderation/infractions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index a0bdf0d97..5bdea5755 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -229,7 +229,7 @@ class Infractions(InfractionScheduler, commands.Cog): if len(reason) > 512: log.info("Kick reason is longer than 512 characters. Reason will be truncated for Audit Log.") - action = user.kick(textwrap.shorten(reason, width=509, placeholder="...") if len(reason) > 512 else reason) + action = user.kick(reason=textwrap.shorten(reason, width=509, placeholder="...")) await self.apply_infraction(ctx, infraction, user, action) @respect_role_hierarchy() @@ -253,7 +253,7 @@ class Infractions(InfractionScheduler, commands.Cog): action = ctx.guild.ban( user, - reason=textwrap.shorten(reason, width=509, placeholder="...") if len(reason) > 512 else reason, + reason=textwrap.shorten(reason, width=509, placeholder="..."), delete_message_days=0 ) await self.apply_infraction(ctx, infraction, user, action) -- cgit v1.2.3 From aad67373d3ac6d3d3a236d6734a6c22019dce120 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sun, 5 Apr 2020 18:17:16 +0300 Subject: (Mod Scheduler): Added reason truncations to Scheduler's `apply_infraction` --- bot/cogs/moderation/scheduler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 917697be9..45e9d58ad 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -84,7 +84,8 @@ class InfractionScheduler(Scheduler): """Apply an infraction to the user, log the infraction, and optionally notify the user.""" infr_type = infraction["type"] icon = utils.INFRACTION_ICONS[infr_type][0] - reason = infraction["reason"] + # Truncate reason when it's too long to avoid raising error on sending ModLog entry + reason = textwrap.shorten(infraction["reason"], width=1900, placeholder="...") expiry = time.format_infraction_with_duration(infraction["expires_at"]) id_ = infraction['id'] -- cgit v1.2.3 From 72768b432b07acd3b1bfd5533c55241126329886 Mon Sep 17 00:00:00 2001 From: rohanjnr Date: Sun, 5 Apr 2020 21:22:28 +0530 Subject: Add feature to restrict tags to specific role(s) --- bot/cogs/tags.py | 52 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index 539105017..9c897ad36 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -4,7 +4,7 @@ import time from pathlib import Path from typing import Callable, Dict, Iterable, List, Optional -from discord import Colour, Embed +from discord import Colour, Embed, Member from discord.ext.commands import Cog, Context, group from bot import constants @@ -36,18 +36,32 @@ class Tags(Cog): """Get all tags.""" # Save all tags in memory. cache = {} - tag_files = Path("bot", "resources", "tags").iterdir() + tag_files = Path("bot", "resources", "tags").glob("**/*") for file in tag_files: - tag_title = file.stem - tag = { - "title": tag_title, - "embed": { - "description": file.read_text() + file_path = str(file).split("/") + if file.is_file(): + tag_title = file.stem + tag = { + "title": tag_title, + "embed": { + "description": file.read_text() + }, + "restricted_to": "developers" } - } - cache[tag_title] = tag + if len(file_path) == 5: + restricted_to = file_path[3] + tag["restricted_to"] = restricted_to + + cache[tag_title] = tag return cache + @staticmethod + def check_accessibility(user: Member, tag: dict) -> bool: + """Check if user can access a tag.""" + if tag["restricted_to"].lower() in [role.name.lower() for role in user.roles]: + return True + return False + @staticmethod def _fuzzy_search(search: str, target: str) -> float: """A simple scoring algorithm based on how many letters are found / total, with order in mind.""" @@ -92,7 +106,7 @@ class Tags(Cog): return self._get_suggestions(tag_name) return found - def _get_tags_via_content(self, check: Callable[[Iterable], bool], keywords: str) -> list: + def _get_tags_via_content(self, check: Callable[[Iterable], bool], keywords: str, user: Member) -> list: """ Search for tags via contents. @@ -113,8 +127,9 @@ class Tags(Cog): matching_tags = [] for tag in self._cache.values(): - if check(query in tag['embed']['description'].casefold() for query in keywords_processed): - matching_tags.append(tag) + if self.check_accessibility(user, tag): + if check(query in tag['embed']['description'].casefold() for query in keywords_processed): + matching_tags.append(tag) return matching_tags @@ -151,7 +166,7 @@ class Tags(Cog): Only search for tags that has ALL the keywords. """ - matching_tags = self._get_tags_via_content(all, keywords) + matching_tags = self._get_tags_via_content(all, keywords, ctx.author) await self._send_matching_tags(ctx, keywords, matching_tags) @search_tag_content.command(name='any') @@ -161,7 +176,7 @@ class Tags(Cog): Search for tags that has ANY of the keywords. """ - matching_tags = self._get_tags_via_content(any, keywords or 'any') + matching_tags = self._get_tags_via_content(any, keywords or 'any', ctx.author) await self._send_matching_tags(ctx, keywords, matching_tags) @tags_group.command(name='get', aliases=('show', 'g')) @@ -198,6 +213,10 @@ class Tags(Cog): if tag_name is not None: founds = self._get_tag(tag_name) + for found_tag in founds: + if not self.check_accessibility(ctx.author, found_tag): + founds.remove(found_tag) + if len(founds) == 1: tag = founds[0] if ctx.channel.id not in TEST_CHANNELS: @@ -222,7 +241,10 @@ class Tags(Cog): else: embed: Embed = Embed(title="**Current tags**") await LinePaginator.paginate( - sorted(f"**»** {tag['title']}" for tag in tags), + sorted( + f"**»** {tag['title']}" for tag in tags + if self.check_accessibility(ctx.author, tag) + ), ctx, embed, footer_text=FOOTER_TEXT, -- cgit v1.2.3 From 00d22a316041a8670903eb5fd4b4a7d143993330 Mon Sep 17 00:00:00 2001 From: rohanjnr Date: Sun, 5 Apr 2020 22:04:51 +0530 Subject: Remove unnecessary variable creation and join two if statements --- bot/cogs/tags.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index 9c897ad36..bb74ab1ca 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -49,8 +49,7 @@ class Tags(Cog): "restricted_to": "developers" } if len(file_path) == 5: - restricted_to = file_path[3] - tag["restricted_to"] = restricted_to + tag["restricted_to"] = file_path[3] cache[tag_title] = tag return cache @@ -127,9 +126,11 @@ class Tags(Cog): matching_tags = [] for tag in self._cache.values(): - if self.check_accessibility(user, tag): - if check(query in tag['embed']['description'].casefold() for query in keywords_processed): - matching_tags.append(tag) + if ( + self.check_accessibility(user, tag) + and check(query in tag['embed']['description'].casefold() for query in keywords_processed) + ): + matching_tags.append(tag) return matching_tags -- cgit v1.2.3 From 025857541b7a0cbb77adf2a0282873c9e116169a Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 6 Apr 2020 08:48:32 +0300 Subject: (Ban and Kick): Changed length in `textwrap.shorten` from 309 to 312 because shorten already include `placeholder` to length. --- bot/cogs/moderation/infractions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 5bdea5755..7a044fc1c 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -229,7 +229,7 @@ class Infractions(InfractionScheduler, commands.Cog): if len(reason) > 512: log.info("Kick reason is longer than 512 characters. Reason will be truncated for Audit Log.") - action = user.kick(reason=textwrap.shorten(reason, width=509, placeholder="...")) + action = user.kick(reason=textwrap.shorten(reason, width=512, placeholder="...")) await self.apply_infraction(ctx, infraction, user, action) @respect_role_hierarchy() @@ -253,7 +253,7 @@ class Infractions(InfractionScheduler, commands.Cog): action = ctx.guild.ban( user, - reason=textwrap.shorten(reason, width=509, placeholder="..."), + reason=textwrap.shorten(reason, width=512, placeholder="..."), delete_message_days=0 ) await self.apply_infraction(ctx, infraction, user, action) -- cgit v1.2.3 From c4f7359d2e301e6ab19666a6867c9cb69892da0b Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Mon, 6 Apr 2020 08:50:46 +0300 Subject: (Ban and Kick): Added space to `textwrap.shorten` `placeholder`. --- bot/cogs/moderation/infractions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 7a044fc1c..2c809535b 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -253,7 +253,7 @@ class Infractions(InfractionScheduler, commands.Cog): action = ctx.guild.ban( user, - reason=textwrap.shorten(reason, width=512, placeholder="..."), + reason=textwrap.shorten(reason, width=512, placeholder=" ..."), delete_message_days=0 ) await self.apply_infraction(ctx, infraction, user, action) -- cgit v1.2.3 From 482c3f4b475cdbe16b377dd5bb85910be0387166 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 09:04:15 +0300 Subject: (Mod Utils): Added shortening reason on embed creation in `notify_infraction`. --- bot/cogs/moderation/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py index 3598f3b1f..9811d059f 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/cogs/moderation/utils.py @@ -135,7 +135,7 @@ async def notify_infraction( description=textwrap.dedent(f""" **Type:** {infr_type.capitalize()} **Expires:** {expires_at or "N/A"} - **Reason:** {reason or "No reason provided."} + **Reason:** {textwrap.shorten(reason, width=1500, placeholder="...") or "No reason provided."} """), colour=Colours.soft_red ) -- cgit v1.2.3 From a59092659271832a46c8ab0166031bffdc68c0a6 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 09:06:17 +0300 Subject: (Infractions): Removed unnecessary logging that notify when reason will be truncated for Audit Log. --- bot/cogs/moderation/infractions.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 2c809535b..d1e77311c 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -226,9 +226,6 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - if len(reason) > 512: - log.info("Kick reason is longer than 512 characters. Reason will be truncated for Audit Log.") - action = user.kick(reason=textwrap.shorten(reason, width=512, placeholder="...")) await self.apply_infraction(ctx, infraction, user, action) @@ -248,9 +245,6 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - if len(reason) > 512: - log.info("Ban reason is longer than 512 characters. Reason will be truncated for Audit Log.") - action = ctx.guild.ban( user, reason=textwrap.shorten(reason, width=512, placeholder=" ..."), -- cgit v1.2.3 From 42e18061a21e0ec1b8a4a692bc8d96f8ef1fd45b Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 09:09:10 +0300 Subject: (Infractions): Moved truncated reason to variable instead on ban coroutine creating. --- bot/cogs/moderation/infractions.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index d1e77311c..3340744b0 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -245,11 +245,9 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - action = ctx.guild.ban( - user, - reason=textwrap.shorten(reason, width=512, placeholder=" ..."), - delete_message_days=0 - ) + truncated_reason = textwrap.shorten(reason, width=512, placeholder=" ...") + + action = ctx.guild.ban(user, reason=truncated_reason, delete_message_days=0) await self.apply_infraction(ctx, infraction, user, action) if infraction.get('expires_at') is not None: -- cgit v1.2.3 From 10ea74bc5ed390c36d64a0f7413b8422f158708a Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 09:16:36 +0300 Subject: (Superstarify, Scheduler): Added reason shortening for ModLog. --- bot/cogs/moderation/scheduler.py | 2 +- bot/cogs/moderation/superstarify.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 45e9d58ad..7404ec8ac 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -326,7 +326,7 @@ class InfractionScheduler(Scheduler): log_text = { "Member": f"<@{user_id}>", "Actor": str(self.bot.get_user(actor) or actor), - "Reason": infraction["reason"], + "Reason": textwrap.shorten(infraction["reason"], width=1500, placeholder="..."), "Created": created, } diff --git a/bot/cogs/moderation/superstarify.py b/bot/cogs/moderation/superstarify.py index ca3dc4202..d77e61e6b 100644 --- a/bot/cogs/moderation/superstarify.py +++ b/bot/cogs/moderation/superstarify.py @@ -183,7 +183,7 @@ class Superstarify(InfractionScheduler, Cog): text=textwrap.dedent(f""" Member: {member.mention} (`{member.id}`) Actor: {ctx.message.author} - Reason: {reason} + Reason: {textwrap.shorten(reason, width=1500, placeholder="...")} Expires: {expiry_str} Old nickname: `{old_nick}` New nickname: `{forced_nick}` -- cgit v1.2.3 From 27e15c42e71f3d2df828d13a5e92c53c664b4431 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 09:20:04 +0300 Subject: (Scheduler): Changed reason truncating in `apply_infraction` from 1900 chars to 1500, added shortening to end message too. --- bot/cogs/moderation/scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 7404ec8ac..345f08f19 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -85,7 +85,7 @@ class InfractionScheduler(Scheduler): infr_type = infraction["type"] icon = utils.INFRACTION_ICONS[infr_type][0] # Truncate reason when it's too long to avoid raising error on sending ModLog entry - reason = textwrap.shorten(infraction["reason"], width=1900, placeholder="...") + reason = textwrap.shorten(infraction["reason"], width=1500, placeholder="...") expiry = time.format_infraction_with_duration(infraction["expires_at"]) id_ = infraction['id'] @@ -128,7 +128,7 @@ class InfractionScheduler(Scheduler): f"Infraction #{id_} actor is bot; including the reason in the confirmation message." ) - end_msg = f" (reason: {infraction['reason']})" + end_msg = f" (reason: {textwrap.shorten(infraction['reason'], width=1500, placeholder='...')})" elif ctx.channel.id not in STAFF_CHANNELS: log.trace( f"Infraction #{id_} context is not in a staff channel; omitting infraction count." -- cgit v1.2.3 From 1100dba71b789bdc35c6c42d1b4003c7d28dcbb0 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 15:10:40 +0300 Subject: (ModLog): Added mod log item embed description truncating when it's too long. --- bot/cogs/moderation/modlog.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index c63b4bab9..e15a80c6d 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -2,6 +2,7 @@ import asyncio import difflib import itertools import logging +import textwrap import typing as t from datetime import datetime from itertools import zip_longest @@ -98,7 +99,7 @@ class ModLog(Cog, name="ModLog"): footer: t.Optional[str] = None, ) -> Context: """Generate log embed and send to logging channel.""" - embed = discord.Embed(description=text) + embed = discord.Embed(description=textwrap.shorten(text, width=2048, placeholder="...")) if title and icon_url: embed.set_author(name=title, icon_url=icon_url) -- cgit v1.2.3 From b765ecb280a3ba0ca017350a4c69dc9c07f97a67 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 15:12:56 +0300 Subject: (Scheduler): Removed reason truncation from `apply_infraction`, changed order of ModLog embed description item in same function. --- bot/cogs/moderation/scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 345f08f19..fbb2d457b 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -85,7 +85,7 @@ class InfractionScheduler(Scheduler): infr_type = infraction["type"] icon = utils.INFRACTION_ICONS[infr_type][0] # Truncate reason when it's too long to avoid raising error on sending ModLog entry - reason = textwrap.shorten(infraction["reason"], width=1500, placeholder="...") + reason = infraction["reason"] expiry = time.format_infraction_with_duration(infraction["expires_at"]) id_ = infraction['id'] @@ -182,8 +182,8 @@ class InfractionScheduler(Scheduler): text=textwrap.dedent(f""" Member: {user.mention} (`{user.id}`) Actor: {ctx.message.author}{dm_log_text} - Reason: {reason} {expiry_log_text} + Reason: {reason} """), content=log_content, footer=f"ID {infraction['id']}" -- cgit v1.2.3 From 03028eab84a09c047c0ef879fc06fccacbe30420 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 15:18:38 +0300 Subject: (Mod Utils): Removed truncation of reason itself and added truncation to whole embed in `notify_infraction`. --- bot/cogs/moderation/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py index 9811d059f..0423e5373 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/cogs/moderation/utils.py @@ -132,11 +132,11 @@ async def notify_infraction( log.trace(f"Sending {user} a DM about their {infr_type} infraction.") embed = discord.Embed( - description=textwrap.dedent(f""" + description=textwrap.shorten(textwrap.dedent(f""" **Type:** {infr_type.capitalize()} **Expires:** {expires_at or "N/A"} - **Reason:** {textwrap.shorten(reason, width=1500, placeholder="...") or "No reason provided."} - """), + **Reason:** {reason or "No reason provided."} + """), width=2048, placeholder="..."), colour=Colours.soft_red ) -- cgit v1.2.3 From 36f947c1cb52faedc1d2e00869e109f2cde12c11 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 8 Apr 2020 15:19:44 +0300 Subject: (Superstarify): Removed unnecessary truncation on `superstarify` command, reordered ModLog text. --- bot/cogs/moderation/superstarify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/superstarify.py b/bot/cogs/moderation/superstarify.py index d77e61e6b..e221ad909 100644 --- a/bot/cogs/moderation/superstarify.py +++ b/bot/cogs/moderation/superstarify.py @@ -183,10 +183,10 @@ class Superstarify(InfractionScheduler, Cog): text=textwrap.dedent(f""" Member: {member.mention} (`{member.id}`) Actor: {ctx.message.author} - Reason: {textwrap.shorten(reason, width=1500, placeholder="...")} Expires: {expiry_str} Old nickname: `{old_nick}` New nickname: `{forced_nick}` + Reason: {reason} """), footer=f"ID {id_}" ) -- cgit v1.2.3 From 8d283fb8eefd7be288702f88653aaebcdcda37c3 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 08:12:09 +0300 Subject: (Mod Utils): Moved embed description to variable. --- bot/cogs/moderation/utils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py index 0423e5373..fc8c26031 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/cogs/moderation/utils.py @@ -131,12 +131,14 @@ async def notify_infraction( """DM a user about their new infraction and return True if the DM is successful.""" log.trace(f"Sending {user} a DM about their {infr_type} infraction.") + text = textwrap.dedent(f""" + **Type:** {infr_type.capitalize()} + **Expires:** {expires_at or "N/A"} + **Reason:** {reason or "No reason provided."} + """) + embed = discord.Embed( - description=textwrap.shorten(textwrap.dedent(f""" - **Type:** {infr_type.capitalize()} - **Expires:** {expires_at or "N/A"} - **Reason:** {reason or "No reason provided."} - """), width=2048, placeholder="..."), + description=textwrap.shorten(text, width=2048, placeholder="..."), colour=Colours.soft_red ) -- cgit v1.2.3 From b148beeec8c897d91fa100d0bbd1cb4965f58e6e Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 08:27:10 +0300 Subject: (Scheduler): Move reason to end of log text to avoid truncating keys. --- bot/cogs/moderation/scheduler.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index fbb2d457b..3352806e7 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -283,6 +283,9 @@ class InfractionScheduler(Scheduler): f"{log_text.get('Failure', '')}" ) + # Move reason to end of entry to avoid cutting out some keys + log_text["Reason"] = log_text.pop("Reason") + # Send a log message to the mod log. await self.mod_log.send_log_message( icon_url=utils.INFRACTION_ICONS[infr_type][1], @@ -326,7 +329,7 @@ class InfractionScheduler(Scheduler): log_text = { "Member": f"<@{user_id}>", "Actor": str(self.bot.get_user(actor) or actor), - "Reason": textwrap.shorten(infraction["reason"], width=1500, placeholder="..."), + "Reason": infraction["reason"], "Created": created, } @@ -396,6 +399,9 @@ class InfractionScheduler(Scheduler): user = self.bot.get_user(user_id) avatar = user.avatar_url_as(static_format="png") if user else None + # Move reason to end so when reason is too long, this is not gonna cut out required items. + log_text["Reason"] = log_text.pop("Reason") + log.trace(f"Sending deactivation mod log for infraction #{id_}.") await self.mod_log.send_log_message( icon_url=utils.INFRACTION_ICONS[type_][1], @@ -405,7 +411,6 @@ class InfractionScheduler(Scheduler): text="\n".join(f"{k}: {v}" for k, v in log_text.items()), footer=f"ID: {id_}", content=log_content, - ) return log_text -- cgit v1.2.3 From 816e76fe8c5e5231cdc85ab974294c1d2fb4a87c Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 08:28:44 +0300 Subject: (Scheduler): Replaced `infraction['reason']` with `reason` variable using in `end_msg`. --- bot/cogs/moderation/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 3352806e7..b238cf4e2 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -128,7 +128,7 @@ class InfractionScheduler(Scheduler): f"Infraction #{id_} actor is bot; including the reason in the confirmation message." ) - end_msg = f" (reason: {textwrap.shorten(infraction['reason'], width=1500, placeholder='...')})" + end_msg = f" (reason: {textwrap.shorten(reason, width=1500, placeholder='...')})" elif ctx.channel.id not in STAFF_CHANNELS: log.trace( f"Infraction #{id_} context is not in a staff channel; omitting infraction count." -- cgit v1.2.3 From 3bbd10ac91e9677e24588734d256a0558c0b46a2 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 09:52:19 +0300 Subject: (Talent Pool): Applied reason shortening. --- bot/cogs/watchchannels/talentpool.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/cogs/watchchannels/talentpool.py b/bot/cogs/watchchannels/talentpool.py index ad0c51fa6..15af7e34d 100644 --- a/bot/cogs/watchchannels/talentpool.py +++ b/bot/cogs/watchchannels/talentpool.py @@ -106,8 +106,8 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): if history: total = f"({len(history)} previous nominations in total)" - start_reason = f"Watched: {history[0]['reason']}" - end_reason = f"Unwatched: {history[0]['end_reason']}" + start_reason = f"Watched: {textwrap.shorten(history[0]['reason'], width=500, placeholder='...')}" + end_reason = f"Unwatched: {textwrap.shorten(history[0]['end_reason'], width=500, placeholder='...')}" msg += f"\n\nUser's previous watch reasons {total}:```{start_reason}\n\n{end_reason}```" await ctx.send(msg) @@ -224,7 +224,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): Status: **Active** Date: {start_date} Actor: {actor.mention if actor else actor_id} - Reason: {nomination_object["reason"]} + Reason: {textwrap.shorten(nomination_object["reason"], width=200, placeholder="...")} Nomination ID: `{nomination_object["id"]}` =============== """ @@ -237,10 +237,10 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): Status: Inactive Date: {start_date} Actor: {actor.mention if actor else actor_id} - Reason: {nomination_object["reason"]} + Reason: {textwrap.shorten(nomination_object["reason"], width=200, placeholder="...")} End date: {end_date} - Unwatch reason: {nomination_object["end_reason"]} + Unwatch reason: {textwrap.shorten(nomination_object["end_reason"], width=200, placeholder="...")} Nomination ID: `{nomination_object["id"]}` =============== """ -- cgit v1.2.3 From 2c9bc9f6fe5174096a6177560acd91f869c296ef Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 11:30:51 +0300 Subject: (Watchchannel): Added footer shortening. --- bot/cogs/watchchannels/watchchannel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/watchchannels/watchchannel.py b/bot/cogs/watchchannels/watchchannel.py index 479820444..ac1aa38ee 100644 --- a/bot/cogs/watchchannels/watchchannel.py +++ b/bot/cogs/watchchannels/watchchannel.py @@ -280,8 +280,9 @@ class WatchChannel(metaclass=CogABCMeta): else: message_jump = f"in [#{msg.channel.name}]({msg.jump_url})" + footer = f"Added {time_delta} by {actor} | Reason: {reason}" embed = Embed(description=f"{msg.author.mention} {message_jump}") - embed.set_footer(text=f"Added {time_delta} by {actor} | Reason: {reason}") + embed.set_footer(text=textwrap.shorten(footer, width=128, placeholder="...")) await self.webhook_send(embed=embed, username=msg.author.display_name, avatar_url=msg.author.avatar_url) -- cgit v1.2.3 From 472b58b3fb1b3d8695d9de1a13ce92f129e6bcc4 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 9 Apr 2020 11:34:54 +0300 Subject: (Big Brother): Added truncating reason. --- bot/cogs/watchchannels/bigbrother.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bot/cogs/watchchannels/bigbrother.py b/bot/cogs/watchchannels/bigbrother.py index 903c87f85..69df849f0 100644 --- a/bot/cogs/watchchannels/bigbrother.py +++ b/bot/cogs/watchchannels/bigbrother.py @@ -1,4 +1,5 @@ import logging +import textwrap from collections import ChainMap from discord.ext.commands import Cog, Context, group @@ -97,8 +98,8 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): if len(history) > 1: total = f"({len(history) // 2} previous infractions in total)" - end_reason = history[0]["reason"] - start_reason = f"Watched: {history[1]['reason']}" + end_reason = textwrap.shorten(history[0]["reason"], width=500, placeholder="...") + start_reason = f"Watched: {textwrap.shorten(history[1]['reason'], width=500, placeholder='...')}" msg += f"\n\nUser's previous watch reasons {total}:```{start_reason}\n\n{end_reason}```" else: msg = ":x: Failed to post the infraction: response was empty." -- cgit v1.2.3 From 7195a928952c1550a82d27706a57165606aa5f4a Mon Sep 17 00:00:00 2001 From: rohanjnr Date: Tue, 14 Apr 2020 18:57:39 +0530 Subject: Feature now is cross-platform Instead of using string methods to split the file path at `/` which is not cross-platform, I am now entirely using pathlib methods to get the parent folder and restrict the tags. --- bot/cogs/tags.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index bb74ab1ca..a59d28600 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -38,7 +38,7 @@ class Tags(Cog): cache = {} tag_files = Path("bot", "resources", "tags").glob("**/*") for file in tag_files: - file_path = str(file).split("/") + if file.is_file(): tag_title = file.stem tag = { @@ -48,8 +48,9 @@ class Tags(Cog): }, "restricted_to": "developers" } - if len(file_path) == 5: - tag["restricted_to"] = file_path[3] + parent_folder = file.parent.stem + if parent_folder != "tags": + tag["restricted_to"] = parent_folder cache[tag_title] = tag return cache @@ -212,11 +213,13 @@ class Tags(Cog): return if tag_name is not None: - founds = self._get_tag(tag_name) + temp_founds = self._get_tag(tag_name) + + founds = [] - for found_tag in founds: - if not self.check_accessibility(ctx.author, found_tag): - founds.remove(found_tag) + for found_tag in temp_founds: + if self.check_accessibility(ctx.author, found_tag): + founds.append(found_tag) if len(founds) == 1: tag = founds[0] -- cgit v1.2.3 From 39bac8873120801eb51a2c1a996d2760d9af64a4 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 14 Apr 2020 16:32:43 +0300 Subject: (ModLog): Applied force embed description truncating in `send_log_message` to avoid removing newlines. --- bot/cogs/moderation/modlog.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index e15a80c6d..fcc9d4e0a 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -99,7 +99,10 @@ class ModLog(Cog, name="ModLog"): footer: t.Optional[str] = None, ) -> Context: """Generate log embed and send to logging channel.""" - embed = discord.Embed(description=textwrap.shorten(text, width=2048, placeholder="...")) + # Truncate string directly here to avoid removing newlines + embed = discord.Embed( + description=text[:2046] + "..." if len(text) > 2048 else text + ) if title and icon_url: embed.set_author(name=title, icon_url=icon_url) -- cgit v1.2.3 From 0b719040e8eb52fd030322b720de5f5ffbcc9919 Mon Sep 17 00:00:00 2001 From: Rohan Reddy Alleti Date: Tue, 14 Apr 2020 19:02:58 +0530 Subject: simplify if statement Co-Authored-By: Mark --- bot/cogs/tags.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index a59d28600..79bea3b63 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -127,10 +127,8 @@ class Tags(Cog): matching_tags = [] for tag in self._cache.values(): - if ( - self.check_accessibility(user, tag) - and check(query in tag['embed']['description'].casefold() for query in keywords_processed) - ): + matches = (query in tag['embed']['description'].casefold() for query in keywords_processed) + if self.check_accessibility(user, tag) and check(matches): matching_tags.append(tag) return matching_tags -- cgit v1.2.3 From 8bf1df9438ecf456b02725e3f9689c5bd885b2a7 Mon Sep 17 00:00:00 2001 From: Rohan Reddy Alleti Date: Tue, 14 Apr 2020 19:03:38 +0530 Subject: simpl Co-Authored-By: Mark --- bot/cogs/tags.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index 79bea3b63..e5492971d 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -58,9 +58,7 @@ class Tags(Cog): @staticmethod def check_accessibility(user: Member, tag: dict) -> bool: """Check if user can access a tag.""" - if tag["restricted_to"].lower() in [role.name.lower() for role in user.roles]: - return True - return False + return tag["restricted_to"].lower() in [role.name.lower() for role in user.roles] @staticmethod def _fuzzy_search(search: str, target: str) -> float: -- cgit v1.2.3 From 6928c492c1d989567c47dfd49a396730c6c8bb27 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 14 Apr 2020 18:29:55 +0300 Subject: (Scheduler): Removed empty line when expiration not specified in `apply_infraction`. --- bot/cogs/moderation/scheduler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index b238cf4e2..5b59b4d4b 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -102,7 +102,7 @@ class InfractionScheduler(Scheduler): dm_result = "" dm_log_text = "" - expiry_log_text = f"Expires: {expiry}" if expiry else "" + expiry_log_text = f"\nExpires: {expiry}" if expiry else "" log_title = "applied" log_content = None @@ -181,8 +181,7 @@ class InfractionScheduler(Scheduler): thumbnail=user.avatar_url_as(static_format="png"), text=textwrap.dedent(f""" Member: {user.mention} (`{user.id}`) - Actor: {ctx.message.author}{dm_log_text} - {expiry_log_text} + Actor: {ctx.message.author}{dm_log_text} {expiry_log_text} Reason: {reason} """), content=log_content, -- cgit v1.2.3 From 32a5bf97e31addb32d17bd0479bc4cf2f4dd9eb7 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 14 Apr 2020 18:50:24 +0300 Subject: (Scheduler): Added removal of infraction in DB, when applying infraction fail. Also don't send DM in this case. --- bot/cogs/moderation/scheduler.py | 46 ++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 5b59b4d4b..58e363da6 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -105,23 +105,7 @@ class InfractionScheduler(Scheduler): expiry_log_text = f"\nExpires: {expiry}" if expiry else "" log_title = "applied" log_content = None - - # DM the user about the infraction if it's not a shadow/hidden infraction. - if not infraction["hidden"]: - dm_result = f"{constants.Emojis.failmail} " - dm_log_text = "\nDM: **Failed**" - - # Sometimes user is a discord.Object; make it a proper user. - try: - if not isinstance(user, (discord.Member, discord.User)): - user = await self.bot.fetch_user(user.id) - except discord.HTTPException as e: - log.error(f"Failed to DM {user.id}: could not fetch user (status {e.status})") - else: - # Accordingly display whether the user was successfully notified via DM. - if await utils.notify_infraction(user, infr_type, expiry, reason, icon): - dm_result = ":incoming_envelope: " - dm_log_text = "\nDM: Sent" + failed = False if infraction["actor"] == self.bot.user.id: log.trace( @@ -165,11 +149,37 @@ class InfractionScheduler(Scheduler): log.warning(f"{log_msg}: bot lacks permissions.") else: log.exception(log_msg) + failed = True + + # DM the user about the infraction if it's not a shadow/hidden infraction. + # Don't send DM when applying failed. + if not infraction["hidden"] and not failed: + dm_result = f"{constants.Emojis.failmail} " + dm_log_text = "\nDM: **Failed**" + + # Sometimes user is a discord.Object; make it a proper user. + try: + if not isinstance(user, (discord.Member, discord.User)): + user = await self.bot.fetch_user(user.id) + except discord.HTTPException as e: + log.error(f"Failed to DM {user.id}: could not fetch user (status {e.status})") + else: + # Accordingly display whether the user was successfully notified via DM. + if await utils.notify_infraction(user, infr_type, expiry, reason, icon): + dm_result = ":incoming_envelope: " + dm_log_text = "\nDM: Sent" + + if failed: + dm_log_text = "\nDM: **Canceled**" + dm_result = f"{constants.Emojis.failmail} " + log.trace(f"Deleted infraction {infraction['id']} from database because applying infraction failed.") + await self.bot.api_client.delete(f"bot/infractions/{infraction['id']}") # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} confirmation message.") await ctx.send( - f"{dm_result}{confirm_msg} **{infr_type}** to {user.mention}{expiry_msg}{end_msg}." + f"{dm_result}{confirm_msg} " + f"{f'**{infr_type}** to {user.mention}{expiry_msg}{end_msg}' if not failed else ''}." ) # Send a log message to the mod log. -- cgit v1.2.3 From 085decd12867f89a0803806928741fe6dd3c76bb Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 15 Apr 2020 08:18:19 +0300 Subject: (Test Helpers): Added `__ge__` function to `MockRole` for comparing. --- tests/helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/helpers.py b/tests/helpers.py index 8e13f0f28..227bac95f 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -205,6 +205,10 @@ class MockRole(CustomMockMixin, unittest.mock.Mock, ColourMixin, HashableMixin): """Simplified position-based comparisons similar to those of `discord.Role`.""" return self.position < other.position + def __ge__(self, other): + """Simplified position-based comparisons similar to those of `discord.Role`.""" + return self.position >= other.position + # Create a Member instance to get a realistic Mock of `discord.Member` member_data = {'user': 'lemon', 'roles': [1]} -- cgit v1.2.3 From 81f6efc2f4e9e157e2f7fb9f191ea410af066632 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 16 Apr 2020 11:15:16 +0300 Subject: (Infraction Tests): Created reason shortening tests for ban and kick. --- tests/bot/cogs/moderation/test_infractions.py | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/bot/cogs/moderation/test_infractions.py diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py new file mode 100644 index 000000000..39ea93952 --- /dev/null +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -0,0 +1,54 @@ +import textwrap +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from bot.cogs.moderation.infractions import Infractions +from tests.helpers import MockBot, MockContext, MockGuild, MockMember, MockRole + + +class ShorteningTests(unittest.IsolatedAsyncioTestCase): + """Tests for ban and kick command reason shortening.""" + + def setUp(self): + self.bot = MockBot() + self.cog = Infractions(self.bot) + self.user = MockMember(id=1234, top_role=MockRole(id=3577, position=10)) + self.target = MockMember(id=1265, top_role=MockRole(id=9876, position=0)) + self.guild = MockGuild(id=4567) + self.ctx = MockContext(bot=self.bot, author=self.user, guild=self.guild) + + @patch("bot.cogs.moderation.utils.has_active_infraction") + @patch("bot.cogs.moderation.utils.post_infraction") + async def test_apply_ban_reason_shortening(self, post_infraction_mock, has_active_mock): + """Should truncate reason for `ctx.guild.ban`.""" + has_active_mock.return_value = False + post_infraction_mock.return_value = {"foo": "bar"} + + self.cog.apply_infraction = AsyncMock() + self.bot.get_cog.return_value = AsyncMock() + self.cog.mod_log.ignore = Mock() + + await self.cog.apply_ban(self.ctx, self.target, "foo bar" * 3000) + ban = self.cog.apply_infraction.call_args[0][3] + self.assertEqual( + ban.cr_frame.f_locals["kwargs"]["reason"], + textwrap.shorten("foo bar" * 3000, 512, placeholder=" ...") + ) + # Await ban to avoid warning + await ban + + @patch("bot.cogs.moderation.utils.post_infraction") + async def test_apply_kick_reason_shortening(self, post_infraction_mock) -> None: + """Should truncate reason for `Member.kick`.""" + post_infraction_mock.return_value = {"foo": "bar"} + + self.cog.apply_infraction = AsyncMock() + self.cog.mod_log.ignore = Mock() + + await self.cog.apply_kick(self.ctx, self.target, "foo bar" * 3000) + kick = self.cog.apply_infraction.call_args[0][3] + self.assertEqual( + kick.cr_frame.f_locals["kwargs"]["reason"], + textwrap.shorten("foo bar" * 3000, 512, placeholder="...") + ) + await kick -- cgit v1.2.3 From 216953044a870f2440fe44fcd2f9ca3ee7cf37e9 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 16 Apr 2020 11:30:09 +0300 Subject: (ModLog Tests): Created reason shortening tests for `send_log_message`. --- tests/bot/cogs/moderation/test_modlog.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tests/bot/cogs/moderation/test_modlog.py diff --git a/tests/bot/cogs/moderation/test_modlog.py b/tests/bot/cogs/moderation/test_modlog.py new file mode 100644 index 000000000..46e01d2ea --- /dev/null +++ b/tests/bot/cogs/moderation/test_modlog.py @@ -0,0 +1,29 @@ +import unittest + +import discord + +from bot.cogs.moderation.modlog import ModLog +from tests.helpers import MockBot, MockTextChannel + + +class ModLogTests(unittest.IsolatedAsyncioTestCase): + """Tests for moderation logs.""" + + def setUp(self): + self.bot = MockBot() + self.cog = ModLog(self.bot) + self.channel = MockTextChannel() + + async def test_log_entry_description_shortening(self): + """Should truncate embed description for ModLog entry.""" + self.bot.get_channel.return_value = self.channel + await self.cog.send_log_message( + icon_url="foo", + colour=discord.Colour.blue(), + title="bar", + text="foo bar" * 3000 + ) + embed = self.channel.send.call_args[1]["embed"] + self.assertEqual( + embed.description, ("foo bar" * 3000)[:2046] + "..." + ) -- cgit v1.2.3 From b463cb2d21683b7184698f788419d325e2f5f5cc Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 16 Apr 2020 11:33:21 +0300 Subject: (ModLog): Removed unused `textwrap` import. --- bot/cogs/moderation/modlog.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index fcc9d4e0a..eb8bd65cf 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -2,7 +2,6 @@ import asyncio import difflib import itertools import logging -import textwrap import typing as t from datetime import datetime from itertools import zip_longest -- cgit v1.2.3 From 1a3fa6a395141c4fcdd1d388d6ce3e7bd89bcbf0 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 16 Apr 2020 13:40:47 +0300 Subject: (Infractions and ModLog Tests): Replaced `shortening` with `truncation`, removed unnecessary type hint and added comment to kick truncation test about awaiting `kick`. --- tests/bot/cogs/moderation/test_infractions.py | 9 +++++---- tests/bot/cogs/moderation/test_modlog.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 39ea93952..51a8cc645 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -6,8 +6,8 @@ from bot.cogs.moderation.infractions import Infractions from tests.helpers import MockBot, MockContext, MockGuild, MockMember, MockRole -class ShorteningTests(unittest.IsolatedAsyncioTestCase): - """Tests for ban and kick command reason shortening.""" +class TruncationTests(unittest.IsolatedAsyncioTestCase): + """Tests for ban and kick command reason truncation.""" def setUp(self): self.bot = MockBot() @@ -19,7 +19,7 @@ class ShorteningTests(unittest.IsolatedAsyncioTestCase): @patch("bot.cogs.moderation.utils.has_active_infraction") @patch("bot.cogs.moderation.utils.post_infraction") - async def test_apply_ban_reason_shortening(self, post_infraction_mock, has_active_mock): + async def test_apply_ban_reason_truncation(self, post_infraction_mock, has_active_mock): """Should truncate reason for `ctx.guild.ban`.""" has_active_mock.return_value = False post_infraction_mock.return_value = {"foo": "bar"} @@ -38,7 +38,7 @@ class ShorteningTests(unittest.IsolatedAsyncioTestCase): await ban @patch("bot.cogs.moderation.utils.post_infraction") - async def test_apply_kick_reason_shortening(self, post_infraction_mock) -> None: + async def test_apply_kick_reason_truncation(self, post_infraction_mock): """Should truncate reason for `Member.kick`.""" post_infraction_mock.return_value = {"foo": "bar"} @@ -51,4 +51,5 @@ class ShorteningTests(unittest.IsolatedAsyncioTestCase): kick.cr_frame.f_locals["kwargs"]["reason"], textwrap.shorten("foo bar" * 3000, 512, placeholder="...") ) + # Await kick to avoid warning await kick diff --git a/tests/bot/cogs/moderation/test_modlog.py b/tests/bot/cogs/moderation/test_modlog.py index 46e01d2ea..d60836474 100644 --- a/tests/bot/cogs/moderation/test_modlog.py +++ b/tests/bot/cogs/moderation/test_modlog.py @@ -14,7 +14,7 @@ class ModLogTests(unittest.IsolatedAsyncioTestCase): self.cog = ModLog(self.bot) self.channel = MockTextChannel() - async def test_log_entry_description_shortening(self): + async def test_log_entry_description_truncation(self): """Should truncate embed description for ModLog entry.""" self.bot.get_channel.return_value = self.channel await self.cog.send_log_message( -- cgit v1.2.3 From ce0ec7db55a9f12f55ac5ba019a95bc0b7d5cbd7 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 18 Apr 2020 21:00:38 -0700 Subject: Tags: always use top-most folder for role restrictions Ensures that nested directories aren't used as the value for the role name. --- bot/cogs/tags.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index e5492971d..1c124b25a 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -34,25 +34,28 @@ class Tags(Cog): @staticmethod def get_tags() -> dict: """Get all tags.""" - # Save all tags in memory. cache = {} - tag_files = Path("bot", "resources", "tags").glob("**/*") - for file in tag_files: + base_path = Path("bot", "resources", "tags") + for file in base_path.glob("**/*"): if file.is_file(): tag_title = file.stem tag = { "title": tag_title, "embed": { - "description": file.read_text() + "description": file.read_text(), }, - "restricted_to": "developers" + "restricted_to": "developers", } - parent_folder = file.parent.stem - if parent_folder != "tags": - tag["restricted_to"] = parent_folder + + # Convert to a list to allow negative indexing. + parents = list(file.relative_to(base_path).parents) + if len(parents) > 1: + # -1 would be '.' hence -2 is used as the index. + tag["restricted_to"] = parents[-2].name cache[tag_title] = tag + return cache @staticmethod -- cgit v1.2.3 From 601ff03823deb842d74f4689fecb68f7ce1693e6 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 18:11:44 +0200 Subject: AntiMalware Tests - Added unittest for message without attachment --- tests/bot/cogs/test_antimalware.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/bot/cogs/test_antimalware.py diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py new file mode 100644 index 000000000..41ca19e17 --- /dev/null +++ b/tests/bot/cogs/test_antimalware.py @@ -0,0 +1,20 @@ +import asyncio +import unittest + +from bot.cogs import antimalware +from tests.helpers import MockBot, MockMessage + + +class AntiMalwareCogTests(unittest.TestCase): + """Test the AntiMalware cog.""" + + def setUp(self): + """Sets up fresh objects for each test.""" + self.bot = MockBot() + self.cog = antimalware.AntiMalware(self.bot) + self.message = MockMessage() + + def test_message_without_attachment(self): + """Messages without attachments should result in no action.""" + coroutine = self.cog.on_message(self.message) + self.assertIsNone(asyncio.run(coroutine)) -- cgit v1.2.3 From 9889f0fdd1ba403ae50ba20be38feca0932d1dda Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 19:06:46 +0200 Subject: AntiMalware Tests - Added unittests for deletion of message and ignoring of dms --- tests/bot/cogs/test_antimalware.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 41ca19e17..ebf3a1277 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,8 +1,9 @@ import asyncio import unittest +from unittest.mock import AsyncMock from bot.cogs import antimalware -from tests.helpers import MockBot, MockMessage +from tests.helpers import MockAttachment, MockBot, MockMessage class AntiMalwareCogTests(unittest.TestCase): @@ -13,8 +14,27 @@ class AntiMalwareCogTests(unittest.TestCase): self.bot = MockBot() self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() + self.message.delete = AsyncMock() def test_message_without_attachment(self): """Messages without attachments should result in no action.""" coroutine = self.cog.on_message(self.message) self.assertIsNone(asyncio.run(coroutine)) + self.message.delete.assert_not_called() + + def test_direct_message_with_attachment(self): + """Direct messages should have no action taken.""" + attachment = MockAttachment(filename="python.asdfsff") + self.message.attachments = [attachment] + self.message.guild = None + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + self.message.delete.assert_not_called() + + def test_message_with_illegal_extension_gets_deleted(self): + """A message containing an illegal extension should send an embed.""" + attachment = MockAttachment(filename="python.asdfsff") + self.message.attachments = [attachment] + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + self.message.delete.assert_called_once() -- cgit v1.2.3 From 90d2ce0e3717d4ddf79eb986e22f7542ca1770e1 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 19:13:45 +0200 Subject: AntiMalware Tests - Added unittest for messages send by staff --- tests/bot/cogs/test_antimalware.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index ebf3a1277..e3fd477fa 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -3,7 +3,8 @@ import unittest from unittest.mock import AsyncMock from bot.cogs import antimalware -from tests.helpers import MockAttachment, MockBot, MockMessage +from bot.constants import Roles +from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole class AntiMalwareCogTests(unittest.TestCase): @@ -38,3 +39,13 @@ class AntiMalwareCogTests(unittest.TestCase): coroutine = self.cog.on_message(self.message) asyncio.run(coroutine) self.message.delete.assert_called_once() + + def test_message_send_by_staff(self): + """A message send by a member of staff should be ignored.""" + moderator_role = MockRole(name="Moderator", id=Roles.moderators) + self.message.author.roles.append(moderator_role) + attachment = MockAttachment(filename="python.asdfsff") + self.message.attachments = [attachment] + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + self.message.delete.assert_not_called() -- cgit v1.2.3 From 3913a8eba46bf98bd09e13145da33f7a09f77960 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 19:45:57 +0200 Subject: AntiMalware Tests - Added unittest for the embed for a python file. --- tests/bot/cogs/test_antimalware.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index e3fd477fa..0bb5af943 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -3,7 +3,7 @@ import unittest from unittest.mock import AsyncMock from bot.cogs import antimalware -from bot.constants import Roles +from bot.constants import Roles, URLs from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole @@ -28,16 +28,20 @@ class AntiMalwareCogTests(unittest.TestCase): attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] self.message.guild = None + coroutine = self.cog.on_message(self.message) asyncio.run(coroutine) + self.message.delete.assert_not_called() def test_message_with_illegal_extension_gets_deleted(self): """A message containing an illegal extension should send an embed.""" attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] + coroutine = self.cog.on_message(self.message) asyncio.run(coroutine) + self.message.delete.assert_called_once() def test_message_send_by_staff(self): @@ -46,6 +50,25 @@ class AntiMalwareCogTests(unittest.TestCase): self.message.author.roles.append(moderator_role) attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] + coroutine = self.cog.on_message(self.message) asyncio.run(coroutine) + self.message.delete.assert_not_called() + + def test_python_file_redirect_embed(self): + """A message containing a .python file should result in an embed redirecting the user to our paste site""" + attachment = MockAttachment(filename="python.py") + self.message.attachments = [attachment] + self.message.channel.send = AsyncMock() + + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + args, kwargs = self.message.channel.send.call_args + embed = kwargs.pop("embed") + + self.assertEqual(args[0], f"Hey {self.message.author.mention}!") + self.assertEqual(embed.description, ( + "It looks like you tried to attach a Python file - " + f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" + )) -- cgit v1.2.3 From 19c15d957040b6857a4141e15c32fd0526f9920d Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 20:15:17 +0200 Subject: AntiMalware Tests - Added unittest for messages that were deleted in the meantime. --- tests/bot/cogs/test_antimalware.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 0bb5af943..da5cd9d11 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,6 +1,9 @@ import asyncio +import logging import unittest -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, Mock + +from discord import NotFound from bot.cogs import antimalware from bot.constants import Roles, URLs @@ -72,3 +75,18 @@ class AntiMalwareCogTests(unittest.TestCase): "It looks like you tried to attach a Python file - " f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" )) + + def test_removing_deleted_message_logs(self): + """Removing an already deleted message logs the correct message""" + attachment = MockAttachment(filename="python.py") + self.message.attachments = [attachment] + self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) + + coroutine = self.cog.on_message(self.message) + logger = logging.getLogger("bot.cogs.antimalware") + + with self.assertLogs(logger=logger, level="INFO") as logs: + asyncio.run(coroutine) + self.assertIn( + f"INFO:bot.cogs.antimalware:Tried to delete message `{self.message.id}`, but message could not be found.", + logs.output) -- cgit v1.2.3 From 4a0b3ea1ef182ddbbb1f9d731b28768a049a531d Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 20:23:00 +0200 Subject: AntiMalware Tests - Added unittest for cog setup --- tests/bot/cogs/test_antimalware.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index da5cd9d11..67c640d23 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -90,3 +90,13 @@ class AntiMalwareCogTests(unittest.TestCase): self.assertIn( f"INFO:bot.cogs.antimalware:Tried to delete message `{self.message.id}`, but message could not be found.", logs.output) + + +class AntiMalwareSetupTests(unittest.TestCase): + """Tests setup of the `AntiMalware` cog.""" + + def test_setup(self): + """Setup of the extension should call add_cog.""" + bot = MockBot() + antimalware.setup(bot) + bot.add_cog.assert_called_once() -- cgit v1.2.3 From 3090141f673279f2836cb3aca95397eb9950ad0f Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 20:41:31 +0200 Subject: AntiMalware Tests - Added unittest message deletion log --- tests/bot/cogs/test_antimalware.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 67c640d23..b4e31b5ce 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,14 +1,17 @@ import asyncio import logging import unittest +from os.path import splitext from unittest.mock import AsyncMock, Mock from discord import NotFound from bot.cogs import antimalware -from bot.constants import Roles, URLs +from bot.constants import AntiMalware as AntiMalwareConfig, Roles, URLs from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole +MODULE = "bot.cogs.antimalware" + class AntiMalwareCogTests(unittest.TestCase): """Test the AntiMalware cog.""" @@ -78,17 +81,38 @@ class AntiMalwareCogTests(unittest.TestCase): def test_removing_deleted_message_logs(self): """Removing an already deleted message logs the correct message""" - attachment = MockAttachment(filename="python.py") + attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) coroutine = self.cog.on_message(self.message) - logger = logging.getLogger("bot.cogs.antimalware") + logger = logging.getLogger(MODULE) with self.assertLogs(logger=logger, level="INFO") as logs: asyncio.run(coroutine) self.assertIn( - f"INFO:bot.cogs.antimalware:Tried to delete message `{self.message.id}`, but message could not be found.", + f"INFO:{MODULE}:Tried to delete message `{self.message.id}`, but message could not be found.", + logs.output) + + def test_message_with_illegal_attachment_logs(self): + """Deleting a message with an illegal attachment should result in a log.""" + attachment = MockAttachment(filename="python.asdfsff") + self.message.attachments = [attachment] + + coroutine = self.cog.on_message(self.message) + file_extensions = {splitext(attachment.filename.lower())[1] for attachment in self.message.attachments} + extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) + blocked_extensions_str = ', '.join(extensions_blocked) + logger = logging.getLogger(MODULE) + + with self.assertLogs(logger=logger, level="INFO") as logs: + asyncio.run(coroutine) + self.assertEqual( + [ + f"INFO:{MODULE}:" + f"User '{self.message.author}' ({self.message.author.id}) " + f"uploaded blacklisted file(s): {blocked_extensions_str}" + ], logs.output) -- cgit v1.2.3 From f0bc9d800dd141b9126c48251a80618e138d61f1 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 20:46:15 +0200 Subject: AntiMalware Tests - Added unittest for valid attachment --- tests/bot/cogs/test_antimalware.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index b4e31b5ce..407fa05c1 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -23,6 +23,15 @@ class AntiMalwareCogTests(unittest.TestCase): self.message = MockMessage() self.message.delete = AsyncMock() + def test_message_with_allowed_attachment(self): + """Messages with allowed extensions should not be deleted""" + attachment = MockAttachment(filename=f"python.{AntiMalwareConfig.whitelist[0]}") + self.message.attachments = [attachment] + + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + self.message.delete.assert_not_called() + def test_message_without_attachment(self): """Messages without attachments should result in no action.""" coroutine = self.cog.on_message(self.message) -- cgit v1.2.3 From 75f6ca6bd9b695a5deb4a4d78311bc63eb2a74d0 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 21:04:47 +0200 Subject: AntiMalware Tests - Added unittest for txt file attachment --- tests/bot/cogs/test_antimalware.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 407fa05c1..eba439afb 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -7,7 +7,7 @@ from unittest.mock import AsyncMock, Mock from discord import NotFound from bot.cogs import antimalware -from bot.constants import AntiMalware as AntiMalwareConfig, Roles, URLs +from bot.constants import AntiMalware as AntiMalwareConfig, Channels, Roles, URLs from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole MODULE = "bot.cogs.antimalware" @@ -21,7 +21,6 @@ class AntiMalwareCogTests(unittest.TestCase): self.bot = MockBot() self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() - self.message.delete = AsyncMock() def test_message_with_allowed_attachment(self): """Messages with allowed extensions should not be deleted""" @@ -88,6 +87,28 @@ class AntiMalwareCogTests(unittest.TestCase): f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" )) + def test_txt_file_redirect_embed(self): + attachment = MockAttachment(filename="python.txt") + self.message.attachments = [attachment] + self.message.channel.send = AsyncMock() + + coroutine = self.cog.on_message(self.message) + asyncio.run(coroutine) + args, kwargs = self.message.channel.send.call_args + embed = kwargs.pop("embed") + cmd_channel = self.bot.get_channel(Channels.bot_commands) + + self.assertEqual(args[0], f"Hey {self.message.author.mention}!") + self.assertEqual(embed.description, ( + "**Uh-oh!** It looks like your message got zapped by our spam filter. " + "We currently don't allow `.txt` attachments, so here are some tips to help you travel safely: \n\n" + "• If you attempted to send a message longer than 2000 characters, try shortening your message " + "to fit within the character limit or use a pasting service (see below) \n\n" + "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " + f"{cmd_channel.mention} for more information) or use a pasting service like: " + f"\n\n{URLs.site_schema}{URLs.site_paste}" + )) + def test_removing_deleted_message_logs(self): """Removing an already deleted message logs the correct message""" attachment = MockAttachment(filename="python.asdfsff") -- cgit v1.2.3 From c8bf44e30c286b27768601d5a04cd2459f170d4c Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Thu, 7 May 2020 21:29:15 +0200 Subject: AntiMalware Tests - Switched to unittest.IsolatedAsyncioTestCase --- tests/bot/cogs/test_antimalware.py | 48 +++++++++++++++----------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index eba439afb..6fb7b399e 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,4 +1,3 @@ -import asyncio import logging import unittest from os.path import splitext @@ -13,7 +12,7 @@ from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole MODULE = "bot.cogs.antimalware" -class AntiMalwareCogTests(unittest.TestCase): +class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): """Test the AntiMalware cog.""" def setUp(self): @@ -22,62 +21,56 @@ class AntiMalwareCogTests(unittest.TestCase): self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() - def test_message_with_allowed_attachment(self): + async def test_message_with_allowed_attachment(self): """Messages with allowed extensions should not be deleted""" attachment = MockAttachment(filename=f"python.{AntiMalwareConfig.whitelist[0]}") self.message.attachments = [attachment] - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.message.delete.assert_not_called() - def test_message_without_attachment(self): + async def test_message_without_attachment(self): """Messages without attachments should result in no action.""" - coroutine = self.cog.on_message(self.message) - self.assertIsNone(asyncio.run(coroutine)) + self.assertIsNone(await self.cog.on_message(self.message)) self.message.delete.assert_not_called() - def test_direct_message_with_attachment(self): + async def test_direct_message_with_attachment(self): """Direct messages should have no action taken.""" attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] self.message.guild = None - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.message.delete.assert_not_called() - def test_message_with_illegal_extension_gets_deleted(self): + async def test_message_with_illegal_extension_gets_deleted(self): """A message containing an illegal extension should send an embed.""" attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.message.delete.assert_called_once() - def test_message_send_by_staff(self): + async def test_message_send_by_staff(self): """A message send by a member of staff should be ignored.""" moderator_role = MockRole(name="Moderator", id=Roles.moderators) self.message.author.roles.append(moderator_role) attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.message.delete.assert_not_called() - def test_python_file_redirect_embed(self): + async def test_python_file_redirect_embed(self): """A message containing a .python file should result in an embed redirecting the user to our paste site""" attachment = MockAttachment(filename="python.py") self.message.attachments = [attachment] self.message.channel.send = AsyncMock() - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) args, kwargs = self.message.channel.send.call_args embed = kwargs.pop("embed") @@ -87,13 +80,12 @@ class AntiMalwareCogTests(unittest.TestCase): f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" )) - def test_txt_file_redirect_embed(self): + async def test_txt_file_redirect_embed(self): attachment = MockAttachment(filename="python.txt") self.message.attachments = [attachment] self.message.channel.send = AsyncMock() - coroutine = self.cog.on_message(self.message) - asyncio.run(coroutine) + await self.cog.on_message(self.message) args, kwargs = self.message.channel.send.call_args embed = kwargs.pop("embed") cmd_channel = self.bot.get_channel(Channels.bot_commands) @@ -109,34 +101,32 @@ class AntiMalwareCogTests(unittest.TestCase): f"\n\n{URLs.site_schema}{URLs.site_paste}" )) - def test_removing_deleted_message_logs(self): + async def test_removing_deleted_message_logs(self): """Removing an already deleted message logs the correct message""" attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) - coroutine = self.cog.on_message(self.message) logger = logging.getLogger(MODULE) with self.assertLogs(logger=logger, level="INFO") as logs: - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.assertIn( f"INFO:{MODULE}:Tried to delete message `{self.message.id}`, but message could not be found.", logs.output) - def test_message_with_illegal_attachment_logs(self): + async def test_message_with_illegal_attachment_logs(self): """Deleting a message with an illegal attachment should result in a log.""" attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] - coroutine = self.cog.on_message(self.message) file_extensions = {splitext(attachment.filename.lower())[1] for attachment in self.message.attachments} extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) blocked_extensions_str = ', '.join(extensions_blocked) logger = logging.getLogger(MODULE) with self.assertLogs(logger=logger, level="INFO") as logs: - asyncio.run(coroutine) + await self.cog.on_message(self.message) self.assertEqual( [ f"INFO:{MODULE}:" -- cgit v1.2.3 From bd9537ba85154ece1dca39ec03d36dd7d39a8388 Mon Sep 17 00:00:00 2001 From: MrGrote Date: Fri, 8 May 2020 22:11:54 +0200 Subject: Update tests/bot/cogs/test_antimalware.py Co-authored-by: Mark --- tests/bot/cogs/test_antimalware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 6fb7b399e..e0aa9d6d2 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -65,7 +65,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.message.delete.assert_not_called() async def test_python_file_redirect_embed(self): - """A message containing a .python file should result in an embed redirecting the user to our paste site""" + """A message containing a .py file should result in an embed redirecting the user to our paste site""" attachment = MockAttachment(filename="python.py") self.message.attachments = [attachment] self.message.channel.send = AsyncMock() -- cgit v1.2.3 From b06c60bc3456c36583d3d58cbf62e9ecd14e5f94 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 9 May 2020 23:19:39 -0700 Subject: Filtering: don't delete messages in DMs Bots are incapable of deleting direct messages authored by others. --- bot/cogs/filtering.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 6a703f5a1..1e21a4ce3 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -4,7 +4,7 @@ from typing import Optional, Union import discord.errors from dateutil.relativedelta import relativedelta -from discord import Colour, DMChannel, Member, Message, TextChannel +from discord import Colour, Member, Message, TextChannel from discord.ext.commands import Cog from discord.utils import escape_markdown @@ -161,8 +161,10 @@ class Filtering(Cog): match = await _filter["function"](msg) if match: - # If this is a filter (not a watchlist), we should delete the message. - if _filter["type"] == "filter": + is_private = msg.channel.type is discord.ChannelType.private + + # If this is a filter (not a watchlist) and not in a DM, delete the message. + if _filter["type"] == "filter" and not is_private: try: # Embeds (can?) trigger both the `on_message` and `on_message_edit` # event handlers, triggering filtering twice for the same message. @@ -181,7 +183,7 @@ class Filtering(Cog): if _filter["user_notification"]: await self.notify_member(msg.author, _filter["notification_msg"], msg.channel) - if isinstance(msg.channel, DMChannel): + if is_private: channel_str = "via DM" else: channel_str = f"in {msg.channel.mention}" -- cgit v1.2.3 From 55b8aed9dc8d4f38824c8b5642a74df6a9948799 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 10 May 2020 12:58:38 -0700 Subject: Filtering: don't attempt to send additional embeds for invalid invites Invalid invites won't have data available to put in the embeds. Fixes #929 Fixes BOT-3Z --- bot/cogs/filtering.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 6a703f5a1..0ad534741 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -212,7 +212,9 @@ class Filtering(Cog): additional_embeds = None additional_embeds_msg = None - if filter_name == "filter_invites": + # The function returns True for invalid invites. + # They have no data so additional embeds can't be created for them. + if filter_name == "filter_invites" and match is not True: additional_embeds = [] for invite, data in match.items(): embed = discord.Embed(description=( -- cgit v1.2.3 From 847a78a76c08a670e85d926e3afa43e1cc3180f4 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 19:41:46 +0200 Subject: AntiMalware Tests - implemented minor feedback --- tests/bot/cogs/test_antimalware.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index e0aa9d6d2..6e06df0a8 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,4 +1,3 @@ -import logging import unittest from os.path import splitext from unittest.mock import AsyncMock, Mock @@ -6,7 +5,7 @@ from unittest.mock import AsyncMock, Mock from discord import NotFound from bot.cogs import antimalware -from bot.constants import AntiMalware as AntiMalwareConfig, Channels, Roles, URLs +from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES, URLs from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole MODULE = "bot.cogs.antimalware" @@ -31,7 +30,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_message_without_attachment(self): """Messages without attachments should result in no action.""" - self.assertIsNone(await self.cog.on_message(self.message)) + await self.cog.on_message(self.message) self.message.delete.assert_not_called() async def test_direct_message_with_attachment(self): @@ -55,8 +54,8 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_message_send_by_staff(self): """A message send by a member of staff should be ignored.""" - moderator_role = MockRole(name="Moderator", id=Roles.moderators) - self.message.author.roles.append(moderator_role) + staff_role = MockRole(id=STAFF_ROLES[0]) + self.message.author.roles.append(staff_role) attachment = MockAttachment(filename="python.asdfsff") self.message.attachments = [attachment] @@ -71,6 +70,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.message.channel.send = AsyncMock() await self.cog.on_message(self.message) + self.message.channel.send.assert_called_once() args, kwargs = self.message.channel.send.call_args embed = kwargs.pop("embed") @@ -107,13 +107,13 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.message.attachments = [attachment] self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) - logger = logging.getLogger(MODULE) - - with self.assertLogs(logger=logger, level="INFO") as logs: + with self.assertLogs(logger=antimalware.log, level="INFO") as logs: await self.cog.on_message(self.message) + self.message.delete.assert_called_once() self.assertIn( f"INFO:{MODULE}:Tried to delete message `{self.message.id}`, but message could not be found.", - logs.output) + logs.output + ) async def test_message_with_illegal_attachment_logs(self): """Deleting a message with an illegal attachment should result in a log.""" @@ -123,9 +123,8 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): file_extensions = {splitext(attachment.filename.lower())[1] for attachment in self.message.attachments} extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) blocked_extensions_str = ', '.join(extensions_blocked) - logger = logging.getLogger(MODULE) - with self.assertLogs(logger=logger, level="INFO") as logs: + with self.assertLogs(logger=antimalware.log, level="INFO") as logs: await self.cog.on_message(self.message) self.assertEqual( [ @@ -133,7 +132,8 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): f"User '{self.message.author}' ({self.message.author.id}) " f"uploaded blacklisted file(s): {blocked_extensions_str}" ], - logs.output) + logs.output + ) class AntiMalwareSetupTests(unittest.TestCase): -- cgit v1.2.3 From ba71ac5b002dd3e1ee6a916ba2705a7cff697a66 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 20:24:20 +0200 Subject: AntiMalware Tests - extracted the method for determining disallowed extensions and added a test for it. --- tests/bot/cogs/test_antimalware.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 6e06df0a8..78ad996f2 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -19,10 +19,11 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.bot = MockBot() self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() + AntiMalwareConfig.whitelist = [".first", ".second", ".third"] async def test_message_with_allowed_attachment(self): """Messages with allowed extensions should not be deleted""" - attachment = MockAttachment(filename=f"python.{AntiMalwareConfig.whitelist[0]}") + attachment = MockAttachment(filename=f"python{AntiMalwareConfig.whitelist[0]}") self.message.attachments = [attachment] await self.cog.on_message(self.message) @@ -35,7 +36,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_direct_message_with_attachment(self): """Direct messages should have no action taken.""" - attachment = MockAttachment(filename="python.asdfsff") + attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] self.message.guild = None @@ -45,7 +46,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_message_with_illegal_extension_gets_deleted(self): """A message containing an illegal extension should send an embed.""" - attachment = MockAttachment(filename="python.asdfsff") + attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] await self.cog.on_message(self.message) @@ -56,7 +57,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): """A message send by a member of staff should be ignored.""" staff_role = MockRole(id=STAFF_ROLES[0]) self.message.author.roles.append(staff_role) - attachment = MockAttachment(filename="python.asdfsff") + attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] await self.cog.on_message(self.message) @@ -103,7 +104,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_removing_deleted_message_logs(self): """Removing an already deleted message logs the correct message""" - attachment = MockAttachment(filename="python.asdfsff") + attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) @@ -117,7 +118,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_message_with_illegal_attachment_logs(self): """Deleting a message with an illegal attachment should result in a log.""" - attachment = MockAttachment(filename="python.asdfsff") + attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] file_extensions = {splitext(attachment.filename.lower())[1] for attachment in self.message.attachments} @@ -135,6 +136,22 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): logs.output ) + async def test_get_disallowed_extensions(self): + """The return value should include all non-whitelisted extensions.""" + test_values = ( + (AntiMalwareConfig.whitelist, []), + ([".first"], []), + ([".first", ".disallowed"], [".disallowed"]), + ([".disallowed"], [".disallowed"]), + ([".disallowed", ".illegal"], [".disallowed", ".illegal"]), + ) + + for extensions, expected_disallowed_extensions in test_values: + with self.subTest(extensions=extensions, expected_disallowed_extensions=expected_disallowed_extensions): + self.message.attachments = [MockAttachment(filename=f"filename{extension}") for extension in extensions] + disallowed_extensions = self.cog.get_disallowed_extensions(self.message) + self.assertCountEqual(disallowed_extensions, expected_disallowed_extensions) + class AntiMalwareSetupTests(unittest.TestCase): """Tests setup of the `AntiMalware` cog.""" -- cgit v1.2.3 From 148b12603f4ad8799d135ec9956d1841cf1c7bf7 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 20:24:39 +0200 Subject: AntiMalware Tests - extracted the method for determining disallowed extensions and added a test for it. --- bot/cogs/antimalware.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bot/cogs/antimalware.py b/bot/cogs/antimalware.py index 66b5073e8..f5fd5e2d9 100644 --- a/bot/cogs/antimalware.py +++ b/bot/cogs/antimalware.py @@ -1,4 +1,5 @@ import logging +import typing as t from os.path import splitext from discord import Embed, Message, NotFound @@ -29,8 +30,7 @@ class AntiMalware(Cog): return embed = Embed() - file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} - extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) + extensions_blocked = self.get_disallowed_extensions(message) blocked_extensions_str = ', '.join(extensions_blocked) if ".py" in extensions_blocked: # Short-circuit on *.py files to provide a pastebin link @@ -73,6 +73,13 @@ class AntiMalware(Cog): except NotFound: log.info(f"Tried to delete message `{message.id}`, but message could not be found.") + @classmethod + def get_disallowed_extensions(cls, message: Message) -> t.Iterable[str]: + """Get an iterable containing all the disallowed extensions of attachments.""" + file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} + extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) + return extensions_blocked + def setup(bot: Bot) -> None: """Load the AntiMalware cog.""" -- cgit v1.2.3 From ecaddcedab6946ac4650b699a790471ef2a898c9 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 20:39:25 +0200 Subject: AntiMalware Tests - added a missing case for no extensions in test_get_disallowed_extensions --- tests/bot/cogs/test_antimalware.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 78ad996f2..7caee6f3c 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -139,6 +139,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): async def test_get_disallowed_extensions(self): """The return value should include all non-whitelisted extensions.""" test_values = ( + ([], []), (AntiMalwareConfig.whitelist, []), ([".first"], []), ([".first", ".disallowed"], [".disallowed"]), -- cgit v1.2.3 From fa467e4ef133186ff462b0178bcab08e8a3d6b2d Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 20:58:51 +0200 Subject: AntiMalware Tests - Removed exact log content checks --- tests/bot/cogs/test_antimalware.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index 7caee6f3c..a2ce9a740 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,5 +1,4 @@ import unittest -from os.path import splitext from unittest.mock import AsyncMock, Mock from discord import NotFound @@ -108,33 +107,17 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.message.attachments = [attachment] self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) - with self.assertLogs(logger=antimalware.log, level="INFO") as logs: + with self.assertLogs(logger=antimalware.log, level="INFO"): await self.cog.on_message(self.message) self.message.delete.assert_called_once() - self.assertIn( - f"INFO:{MODULE}:Tried to delete message `{self.message.id}`, but message could not be found.", - logs.output - ) async def test_message_with_illegal_attachment_logs(self): """Deleting a message with an illegal attachment should result in a log.""" attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] - file_extensions = {splitext(attachment.filename.lower())[1] for attachment in self.message.attachments} - extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) - blocked_extensions_str = ', '.join(extensions_blocked) - - with self.assertLogs(logger=antimalware.log, level="INFO") as logs: + with self.assertLogs(logger=antimalware.log, level="INFO"): await self.cog.on_message(self.message) - self.assertEqual( - [ - f"INFO:{MODULE}:" - f"User '{self.message.author}' ({self.message.author.id}) " - f"uploaded blacklisted file(s): {blocked_extensions_str}" - ], - logs.output - ) async def test_get_disallowed_extensions(self): """The return value should include all non-whitelisted extensions.""" -- cgit v1.2.3 From ddfe583d0b1e72f98855f628ff01b72c82fa491d Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 11 May 2020 21:56:39 +0200 Subject: AntiMalware Refactor - Moved embed descriptions into constants, added tests for embed descriptions --- bot/cogs/antimalware.py | 44 ++++++++++++++++++++-------------- tests/bot/cogs/test_antimalware.py | 48 ++++++++++++++++++++++++-------------- 2 files changed, 56 insertions(+), 36 deletions(-) diff --git a/bot/cogs/antimalware.py b/bot/cogs/antimalware.py index f5fd5e2d9..ea257442e 100644 --- a/bot/cogs/antimalware.py +++ b/bot/cogs/antimalware.py @@ -10,6 +10,27 @@ from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLE log = logging.getLogger(__name__) +PY_EMBED_DESCRIPTION = ( + "It looks like you tried to attach a Python file - " + f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" +) + +TXT_EMBED_DESCRIPTION = ( + "**Uh-oh!** It looks like your message got zapped by our spam filter. " + "We currently don't allow `.txt` attachments, so here are some tips to help you travel safely: \n\n" + "• If you attempted to send a message longer than 2000 characters, try shortening your message " + "to fit within the character limit or use a pasting service (see below) \n\n" + "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " + "{cmd_channel_mention} for more information) or use a pasting service like: " + f"\n\n{URLs.site_schema}{URLs.site_paste}" +) + +DISALLOWED_EMBED_DESCRIPTION = ( + "It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " + f"We currently allow the following file types: **{', '.join(AntiMalwareConfig.whitelist)}**.\n\n" + "Feel free to ask in {meta_channel_mention} if you think this is a mistake." +) + class AntiMalware(Cog): """Delete messages which contain attachments with non-whitelisted file extensions.""" @@ -34,29 +55,16 @@ class AntiMalware(Cog): blocked_extensions_str = ', '.join(extensions_blocked) if ".py" in extensions_blocked: # Short-circuit on *.py files to provide a pastebin link - embed.description = ( - "It looks like you tried to attach a Python file - " - f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" - ) + embed.description = PY_EMBED_DESCRIPTION elif ".txt" in extensions_blocked: # Work around Discord AutoConversion of messages longer than 2000 chars to .txt cmd_channel = self.bot.get_channel(Channels.bot_commands) - embed.description = ( - "**Uh-oh!** It looks like your message got zapped by our spam filter. " - "We currently don't allow `.txt` attachments, so here are some tips to help you travel safely: \n\n" - "• If you attempted to send a message longer than 2000 characters, try shortening your message " - "to fit within the character limit or use a pasting service (see below) \n\n" - "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " - f"{cmd_channel.mention} for more information) or use a pasting service like: " - f"\n\n{URLs.site_schema}{URLs.site_paste}" - ) + embed.description = TXT_EMBED_DESCRIPTION.format(cmd_channel_mention=cmd_channel.mention) elif extensions_blocked: - whitelisted_types = ', '.join(AntiMalwareConfig.whitelist) meta_channel = self.bot.get_channel(Channels.meta) - embed.description = ( - f"It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " - f"We currently allow the following file types: **{whitelisted_types}**.\n\n" - f"Feel free to ask in {meta_channel.mention} if you think this is a mistake." + embed.description = DISALLOWED_EMBED_DESCRIPTION.format( + blocked_extensions_str=blocked_extensions_str, + meta_channel_mention=meta_channel.mention, ) if embed.description: diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index a2ce9a740..fab063201 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -4,7 +4,7 @@ from unittest.mock import AsyncMock, Mock from discord import NotFound from bot.cogs import antimalware -from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES, URLs +from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole MODULE = "bot.cogs.antimalware" @@ -63,7 +63,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.message.delete.assert_not_called() - async def test_python_file_redirect_embed(self): + async def test_python_file_redirect_embed_description(self): """A message containing a .py file should result in an embed redirecting the user to our paste site""" attachment = MockAttachment(filename="python.py") self.message.attachments = [attachment] @@ -74,32 +74,44 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): args, kwargs = self.message.channel.send.call_args embed = kwargs.pop("embed") - self.assertEqual(args[0], f"Hey {self.message.author.mention}!") - self.assertEqual(embed.description, ( - "It looks like you tried to attach a Python file - " - f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" - )) + self.assertEqual(embed.description, antimalware.PY_EMBED_DESCRIPTION) - async def test_txt_file_redirect_embed(self): + async def test_txt_file_redirect_embed_description(self): + """A message containing a .txt file should result in the correct embed.""" attachment = MockAttachment(filename="python.txt") self.message.attachments = [attachment] self.message.channel.send = AsyncMock() + antimalware.TXT_EMBED_DESCRIPTION = Mock() + antimalware.TXT_EMBED_DESCRIPTION.format.return_value = "test" await self.cog.on_message(self.message) + self.message.channel.send.assert_called_once() args, kwargs = self.message.channel.send.call_args embed = kwargs.pop("embed") cmd_channel = self.bot.get_channel(Channels.bot_commands) - self.assertEqual(args[0], f"Hey {self.message.author.mention}!") - self.assertEqual(embed.description, ( - "**Uh-oh!** It looks like your message got zapped by our spam filter. " - "We currently don't allow `.txt` attachments, so here are some tips to help you travel safely: \n\n" - "• If you attempted to send a message longer than 2000 characters, try shortening your message " - "to fit within the character limit or use a pasting service (see below) \n\n" - "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " - f"{cmd_channel.mention} for more information) or use a pasting service like: " - f"\n\n{URLs.site_schema}{URLs.site_paste}" - )) + self.assertEqual(embed.description, antimalware.TXT_EMBED_DESCRIPTION.format.return_value) + antimalware.TXT_EMBED_DESCRIPTION.format.assert_called_with(cmd_channel_mention=cmd_channel.mention) + + async def test_other_disallowed_extention_embed_description(self): + """Test the description for a non .py/.txt disallowed extension.""" + attachment = MockAttachment(filename="python.disallowed") + self.message.attachments = [attachment] + self.message.channel.send = AsyncMock() + antimalware.DISALLOWED_EMBED_DESCRIPTION = Mock() + antimalware.DISALLOWED_EMBED_DESCRIPTION.format.return_value = "test" + + await self.cog.on_message(self.message) + self.message.channel.send.assert_called_once() + args, kwargs = self.message.channel.send.call_args + embed = kwargs.pop("embed") + meta_channel = self.bot.get_channel(Channels.meta) + + self.assertEqual(embed.description, antimalware.DISALLOWED_EMBED_DESCRIPTION.format.return_value) + antimalware.DISALLOWED_EMBED_DESCRIPTION.format.assert_called_with( + blocked_extensions_str=".disallowed", + meta_channel_mention=meta_channel.mention + ) async def test_removing_deleted_message_logs(self): """Removing an already deleted message logs the correct message""" -- cgit v1.2.3 From d7e6bed7b5b0f61312165e5b0b2b9291cd8df0c9 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 13 May 2020 14:17:51 +0300 Subject: Add message publishing to `Reddit` cog --- bot/cogs/reddit.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index 5a7fa100f..371b65434 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -218,7 +218,10 @@ class Reddit(Cog): for subreddit in RedditConfig.subreddits: top_posts = await self.get_top_posts(subreddit=subreddit, time="day") - await self.webhook.send(username=f"{subreddit} Top Daily Posts", embed=top_posts) + message = await self.webhook.send(username=f"{subreddit} Top Daily Posts", embed=top_posts) + + if message.channel.is_news(): + await message.publish() async def top_weekly_posts(self) -> None: """Post a summary of the top posts.""" @@ -242,6 +245,9 @@ class Reddit(Cog): await message.pin() + if message.channel.is_news(): + await message.publish() + @group(name="reddit", invoke_without_command=True) async def reddit_group(self, ctx: Context) -> None: """View the top posts from various subreddits.""" -- cgit v1.2.3 From 206aed70f6185057ccbe4f8478ed456e5ab0c197 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 14 May 2020 16:58:55 +0300 Subject: Python News: Implement stats Add stat increaser to PEP and maillist posting. --- bot/cogs/python_news.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/cogs/python_news.py b/bot/cogs/python_news.py index 57ce61638..d28af4a0b 100644 --- a/bot/cogs/python_news.py +++ b/bot/cogs/python_news.py @@ -109,6 +109,9 @@ class PythonNews(Cog): ) payload["data"]["pep"].append(pep_nr) + # Increase overall PEP new stat + self.bot.stats.incr("python_news.posted.pep") + if msg.channel.is_news(): log.trace("Publishing PEP annnouncement because it was in a news channel") await msg.publish() @@ -168,6 +171,9 @@ class PythonNews(Cog): ) payload["data"][maillist].append(thread_information["thread_id"]) + # Increase this specific maillist counter in stats + self.bot.stats.incr(f"python_news.posted.{maillist.replace('-', '_')}") + if msg.channel.is_news(): log.trace("Publishing mailing list message because it was in a news channel") await msg.publish() -- cgit v1.2.3 From d2c538e23c20c5c4b22d7b2eb2bcf03067593374 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Fri, 15 May 2020 19:33:24 +0200 Subject: Increase snekbox re eval timeout. --- bot/cogs/snekbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 8d4688114..611e80f61 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -227,7 +227,7 @@ class Snekbox(Cog): _, new_message = await self.bot.wait_for( 'message_edit', check=_predicate_eval_message_edit, - timeout=10 + timeout=30 ) await ctx.message.add_reaction(REEVAL_EMOJI) await self.bot.wait_for( -- cgit v1.2.3 From 1c06d2a9d873ced2e54bf16a96573a46c583c12f Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Fri, 15 May 2020 19:51:19 +0200 Subject: Move the re eval timeout to a module constant --- bot/cogs/snekbox.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 611e80f61..9fa75a929 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -47,6 +47,7 @@ EVAL_ROLES = (Roles.helpers, Roles.moderators, Roles.admins, Roles.owners, Roles SIGKILL = 9 REEVAL_EMOJI = '\U0001f501' # :repeat: +REEVAL_TIMEOUT = 30 class Snekbox(Cog): @@ -227,7 +228,7 @@ class Snekbox(Cog): _, new_message = await self.bot.wait_for( 'message_edit', check=_predicate_eval_message_edit, - timeout=30 + timeout=REEVAL_TIMEOUT ) await ctx.message.add_reaction(REEVAL_EMOJI) await self.bot.wait_for( -- cgit v1.2.3 From 5a48ed0d60ebc9984cae27b19953b50b52df83d9 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Fri, 15 May 2020 19:52:26 +0200 Subject: Change tests to use the new timeout constant --- tests/bot/cogs/test_snekbox.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_snekbox.py b/tests/bot/cogs/test_snekbox.py index 1dec0ccaf..ccc090f02 100644 --- a/tests/bot/cogs/test_snekbox.py +++ b/tests/bot/cogs/test_snekbox.py @@ -291,7 +291,11 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(actual, expected) self.bot.wait_for.assert_has_awaits( ( - call('message_edit', check=partial_mock(snekbox.predicate_eval_message_edit, ctx), timeout=10), + call( + 'message_edit', + check=partial_mock(snekbox.predicate_eval_message_edit, ctx), + timeout=snekbox.REEVAL_TIMEOUT, + ), call('reaction_add', check=partial_mock(snekbox.predicate_eval_emoji_reaction, ctx), timeout=10) ) ) -- cgit v1.2.3 From 3aeb9e6ac6258fe3446788fc8a731fc8bb5922d4 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 12:23:52 +0200 Subject: Adding redis to docker-compose file. This is almost hilariously easy since we can just use the official image for it. --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 11deceae8..1bcf1008e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,9 @@ services: POSTGRES_PASSWORD: pysite POSTGRES_USER: pysite + redis: + image: redis:5.0.9 + web: image: pythondiscord/site:latest command: ["run", "--debug"] @@ -41,6 +44,7 @@ services: tty: true depends_on: - web + - redis environment: BOT_TOKEN: ${BOT_TOKEN} BOT_API_KEY: badbot13m0n8f570f942013fc818f234916ca531 -- cgit v1.2.3 From 5382dd80611dcf24124477f0e09dfde668df1ace Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 12:24:34 +0200 Subject: Adding redis-py to the Pipfile This is the module we will be using to interface with Redis. --- Pipfile | 1 + Pipfile.lock | 131 ++++++++++++++++++++++++++++++----------------------------- 2 files changed, 67 insertions(+), 65 deletions(-) diff --git a/Pipfile b/Pipfile index 14c9ef926..5f85b1e51 100644 --- a/Pipfile +++ b/Pipfile @@ -23,6 +23,7 @@ colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} statsd = "~=3.3" feedparser = "~=5.2" beautifulsoup4 = "~=4.9" +redis = ">=3.5" [dev-packages] coverage = "~=5.0" diff --git a/Pipfile.lock b/Pipfile.lock index 4e7050a13..1a420182d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "64620e7e825c74fd3010821fb30843b19f5dafb2b5a1f6eafedc0a5febd99b69" + "sha256": "c6b4d38c4034e55a4bd598399f2e1f48b70a76693c986d0db0fae7442e224d41" }, "pipfile-spec": 6, "requires": { @@ -52,10 +52,10 @@ }, "aiormq": { "hashes": [ - "sha256:286e0b0772075580466e45f98f051b9728a9316b9c36f0c14c7bc1409be375b0", - "sha256:7ed7d6df6b57af7f8bce7d1ebcbdfc32b676192e46703e81e9e217316e56b5bd" + "sha256:41a9d4eb17db805f30ed172f3f609fe0c2b16657fb15b1b67df19d251dd93c0d", + "sha256:7c19477a9450824cb79f9949fd238f4148e2c0dca67756a2868863c387209f04" ], - "version": "==3.2.1" + "version": "==3.2.2" }, "alabaster": { "hashes": [ @@ -305,39 +305,39 @@ }, "more-itertools": { "hashes": [ - "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c", - "sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507" + "sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be", + "sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982" ], "index": "pypi", - "version": "==8.2.0" + "version": "==8.3.0" }, "multidict": { "hashes": [ - "sha256:317f96bc0950d249e96d8d29ab556d01dd38888fbe68324f46fd834b430169f1", - "sha256:42f56542166040b4474c0c608ed051732033cd821126493cf25b6c276df7dd35", - "sha256:4b7df040fb5fe826d689204f9b544af469593fb3ff3a069a6ad3409f742f5928", - "sha256:544fae9261232a97102e27a926019100a9db75bec7b37feedd74b3aa82f29969", - "sha256:620b37c3fea181dab09267cd5a84b0f23fa043beb8bc50d8474dd9694de1fa6e", - "sha256:6e6fef114741c4d7ca46da8449038ec8b1e880bbe68674c01ceeb1ac8a648e78", - "sha256:7774e9f6c9af3f12f296131453f7b81dabb7ebdb948483362f5afcaac8a826f1", - "sha256:85cb26c38c96f76b7ff38b86c9d560dea10cf3459bb5f4caf72fc1bb932c7136", - "sha256:a326f4240123a2ac66bb163eeba99578e9d63a8654a59f4688a79198f9aa10f8", - "sha256:ae402f43604e3b2bc41e8ea8b8526c7fa7139ed76b0d64fc48e28125925275b2", - "sha256:aee283c49601fa4c13adc64c09c978838a7e812f85377ae130a24d7198c0331e", - "sha256:b51249fdd2923739cd3efc95a3d6c363b67bbf779208e9f37fd5e68540d1a4d4", - "sha256:bb519becc46275c594410c6c28a8a0adc66fe24fef154a9addea54c1adb006f5", - "sha256:c2c37185fb0af79d5c117b8d2764f4321eeb12ba8c141a95d0aa8c2c1d0a11dd", - "sha256:dc561313279f9d05a3d0ffa89cd15ae477528ea37aa9795c4654588a3287a9ab", - "sha256:e439c9a10a95cb32abd708bb8be83b2134fa93790a4fb0535ca36db3dda94d20", - "sha256:fc3b4adc2ee8474cb3cd2a155305d5f8eda0a9c91320f83e55748e1fcb68f8e3" - ], - "version": "==4.7.5" + "sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a", + "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000", + "sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2", + "sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507", + "sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5", + "sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7", + "sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d", + "sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463", + "sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19", + "sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3", + "sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b", + "sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c", + "sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87", + "sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7", + "sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430", + "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255", + "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d" + ], + "version": "==4.7.6" }, "ordered-set": { "hashes": [ - "sha256:a7bfa858748c73b096e43db14eb23e2bc714a503f990c89fac8fab9b0ee79724" + "sha256:a31008c57f9c9776b12eb8841b1f61d1e4d70dfbbe8875ccfa2403c54af3d51b" ], - "version": "==3.1.1" + "version": "==4.0.1" }, "packaging": { "hashes": [ @@ -418,10 +418,10 @@ }, "pytz": { "hashes": [ - "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", - "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" + "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", + "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048" ], - "version": "==2019.3" + "version": "==2020.1" }, "pyyaml": { "hashes": [ @@ -440,6 +440,14 @@ "index": "pypi", "version": "==5.3.1" }, + "redis": { + "hashes": [ + "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", + "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" + ], + "index": "pypi", + "version": "==3.5.2" + }, "requests": { "hashes": [ "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", @@ -450,11 +458,11 @@ }, "sentry-sdk": { "hashes": [ - "sha256:23808d571d2461a4ce3784ec12bbee5bdb8c026c143fe79d36cef8a6d653e71f", - "sha256:bb90a4e19c7233a580715fc986cc44be2c48fc10b31e71580a2037e1c94b6950" + "sha256:0e5e947d0f7a969314aa23669a94a9712be5a688ff069ff7b9fc36c66adc160c", + "sha256:799a8bf76b012e3030a881be00e97bc0b922ce35dde699c6537122b751d80e2c" ], "index": "pypi", - "version": "==0.14.3" + "version": "==0.14.4" }, "six": { "hashes": [ @@ -595,10 +603,10 @@ "develop": { "appdirs": { "hashes": [ - "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", - "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" ], - "version": "==1.4.3" + "version": "==1.4.4" }, "attrs": { "hashes": [ @@ -657,13 +665,6 @@ ], "version": "==0.3.0" }, - "entrypoints": { - "hashes": [ - "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", - "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" - ], - "version": "==0.3" - }, "filelock": { "hashes": [ "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", @@ -673,11 +674,11 @@ }, "flake8": { "hashes": [ - "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb", - "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca" + "sha256:6c1193b0c3f853ef763969238f6c81e9e63ace9d024518edc020d5f1d6d93195", + "sha256:ea6623797bf9a52f4c9577d780da0bb17d65f870213f7b5bcc9fca82540c31d5" ], "index": "pypi", - "version": "==3.7.9" + "version": "==3.8.1" }, "flake8-annotations": { "hashes": [ @@ -743,10 +744,10 @@ }, "identify": { "hashes": [ - "sha256:2bb8760d97d8df4408f4e805883dad26a2d076f04be92a10a3e43f09c6060742", - "sha256:faffea0fd8ec86bb146ac538ac350ed0c73908326426d387eded0bcc9d077522" + "sha256:23c18d97bb50e05be1a54917ee45cc61d57cb96aedc06aabb2b02331edf0dbf0", + "sha256:88ed90632023e52a6495749c6732e61e08ec9f4f04e95484a5c37b9caf40283c" ], - "version": "==1.4.14" + "version": "==1.4.15" }, "mccabe": { "hashes": [ @@ -771,18 +772,18 @@ }, "pre-commit": { "hashes": [ - "sha256:487c675916e6f99d355ec5595ad77b325689d423ef4839db1ed2f02f639c9522", - "sha256:c0aa11bce04a7b46c5544723aedf4e81a4d5f64ad1205a30a9ea12d5e81969e1" + "sha256:5559e09afcac7808933951ffaf4ff9aac524f31efbc3f24d021540b6c579813c", + "sha256:703e2e34cbe0eedb0d319eff9f7b83e2022bb5a3ab5289a6a8841441076514d0" ], "index": "pypi", - "version": "==2.2.0" + "version": "==2.4.0" }, "pycodestyle": { "hashes": [ - "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", - "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" + "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367", + "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e" ], - "version": "==2.5.0" + "version": "==2.6.0" }, "pydocstyle": { "hashes": [ @@ -793,10 +794,10 @@ }, "pyflakes": { "hashes": [ - "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", - "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" + "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92", + "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8" ], - "version": "==2.1.1" + "version": "==2.2.0" }, "pyyaml": { "hashes": [ @@ -831,10 +832,10 @@ }, "toml": { "hashes": [ - "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", - "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" + "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", + "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88" ], - "version": "==0.10.0" + "version": "==0.10.1" }, "unittest-xml-reporting": { "hashes": [ @@ -846,10 +847,10 @@ }, "virtualenv": { "hashes": [ - "sha256:5021396e8f03d0d002a770da90e31e61159684db2859d0ba4850fbea752aa675", - "sha256:ac53ade75ca189bc97b6c1d9ec0f1a50efe33cbf178ae09452dcd9fd309013c1" + "sha256:b4c14d4d73a0c23db267095383c4276ef60e161f94fde0427f2f21a0132dde74", + "sha256:fd0e54dec8ac96c1c7c87daba85f0a59a7c37fe38748e154306ca21c73244637" ], - "version": "==20.0.18" + "version": "==20.0.20" } } } -- cgit v1.2.3 From 7a501fdecaae186590177fd4ebd6cea64119629e Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 12:40:06 +0200 Subject: Boilerplate for the RedisCacheMixin We're using __init_subclass__ to initialize our RedisDict with the subclass name as a namespace. This will be prefixed to all data that we store, so that there won't be collisions between different subclasses. --- bot/mixins/__init__.py | 3 +++ bot/mixins/redis.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 bot/mixins/__init__.py create mode 100644 bot/mixins/redis.py diff --git a/bot/mixins/__init__.py b/bot/mixins/__init__.py new file mode 100644 index 000000000..ff1f0c50d --- /dev/null +++ b/bot/mixins/__init__.py @@ -0,0 +1,3 @@ +from .redis import RedisCacheMixin + +__all__ = ['RedisCacheMixin'] diff --git a/bot/mixins/redis.py b/bot/mixins/redis.py new file mode 100644 index 000000000..f19108576 --- /dev/null +++ b/bot/mixins/redis.py @@ -0,0 +1,56 @@ +import redis as redis_py + +redis = redis_py.Redis(host="redis") + + +class RedisDict(dict): + """ + A dictionary interface for a Redis database. + + Objects created by this class should mostly behave like a normal dictionary, + but will store all the data in our Redis database for persistence between restarts. + + There are, however, a few limitations to what kinds of data types can be + stored on Redis, so this is a little bit more limited than a regular dict. + """ + + def __init__(self, namespace: str = "global"): + """Initialize the RedisDict with the right namespace.""" + # TODO: Make namespace collision impossible! + # Append a number or something if it exists already. + self.namespace = namespace + + # redis.mset({"firedog": "donkeykong"}) + # + # print(redis.get("firedog").decode("utf-8") + + +class RedisCacheMixin: + """ + A mixin which adds a cls.cache parameter which can be used for persistent caching. + + This adds a dictionary-like object called cache which can be treated like a regular dictionary, + but which can only store simple data types like ints, strings, and floats. + + To use it, simply subclass it into your class like this: + + class MyCog(Cog, RedisCacheMixin): + def some_command(self): + # You can now do this! + self.cache['some_data'] = some_data + + All the data stored in this cache will probably be available permanently, even if the bot restarts or + is updated. However, Redis is not meant to be used for reliable, permanent storage. It may be cleared + from time to time, so please only use it for caching data that you can afford to lose. + + If it's really important that your data should never disappear, please use our postgres database instead. + """ + + def __init_subclass__(cls, **kwargs): + """ + Initialize the cache when subclass is created. + + When this mixin is subclassed, we create a cache using the subclass name as the namespace. + This is to prevent collisions between subclasses. + """ + cls.cache = RedisDict(cls.__name__) -- cgit v1.2.3 From 93cce50846b1cfcf520535d69a9fe223c2cd4d7a Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 16 May 2020 16:49:48 +0300 Subject: Stats: Create guild boost stat collection Collect Guild boost amount + level and post it to StatsD every hour in task. Added starting to cog `__init__.py` and stopping to `cog_unload`. --- bot/cogs/stats.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py index d253db913..acee1f5a9 100644 --- a/bot/cogs/stats.py +++ b/bot/cogs/stats.py @@ -2,8 +2,10 @@ import string from datetime import datetime from discord import Member, Message, Status -from discord.ext.commands import Bot, Cog, Context +from discord.ext.commands import Cog, Context +from discord.ext.tasks import loop +from bot.bot import Bot from bot.constants import Channels, Guild, Stats as StatConf @@ -23,6 +25,7 @@ class Stats(Cog): def __init__(self, bot: Bot): self.bot = bot self.last_presence_update = None + self.update_guild_boost.start() @Cog.listener() async def on_message(self, message: Message) -> None: @@ -101,6 +104,18 @@ class Stats(Cog): self.bot.stats.gauge("guild.status.do_not_disturb", dnd) self.bot.stats.gauge("guild.status.offline", offline) + @loop(hours=1) + async def update_guild_boost(self) -> None: + """Update every hour guild boosts amount + level.""" + await self.bot.wait_until_guild_available() + g = self.bot.get_guild(Guild.id) + self.bot.stats.gauge("boost.amount", g.premium_subscription_count) + self.bot.stats.gauge("boost.tier", g.premium_tier) + + def cog_unload(self) -> None: + """Stop guild boost stat collecting task on Cog unload.""" + self.update_guild_boost.stop() + def setup(bot: Bot) -> None: """Load the stats cog.""" -- cgit v1.2.3 From 1a6abaac12eb2e6ab0d26065108fe1cce9a7be45 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 16 May 2020 17:00:25 +0300 Subject: Stats: Added codeblock correction stats --- bot/cogs/bot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/cogs/bot.py b/bot/cogs/bot.py index a6929b431..67ff8f95d 100644 --- a/bot/cogs/bot.py +++ b/bot/cogs/bot.py @@ -326,6 +326,8 @@ class BotCog(Cog, name="Bot"): log.trace("The code consists only of expressions, not sending instructions") if howto != "": + # Increase amount of codeblock correction in stats + self.bot.stats.incr("codeblock_corrections") howto_embed = Embed(description=howto) bot_message = await msg.channel.send(f"Hey {msg.author.mention}!", embed=howto_embed) self.codeblock_message_ids[msg.id] = bot_message.id -- cgit v1.2.3 From 158e19a6fcb2056c6bcc244a1f02d8b75d7fe503 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 16 May 2020 17:13:54 +0300 Subject: Stats: Added stats for eval successes + fails --- bot/cogs/snekbox.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 8d4688114..1c64c893b 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -205,6 +205,12 @@ class Snekbox(Cog): if paste_link: msg = f"{msg}\nFull output: {paste_link}" + # Collect stats of eval fails + successes + if icon == ":x:": + self.bot.stats.incr("evals.fail") + elif icon in (":warning:", ":white_check_mark:"): + self.bot.stats.incr("evals.success") + response = await ctx.send(msg) self.bot.loop.create_task( wait_for_deletion(response, user_ids=(ctx.author.id,), client=ctx.bot) -- cgit v1.2.3 From 5878ec93c5b883038b0f738e9bbbdee8fd1929ad Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 16 May 2020 17:25:25 +0300 Subject: Stats: Added stats for eval role uses (Helpers/Developers) --- bot/cogs/snekbox.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 1c64c893b..04c0a5ae1 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -298,6 +298,11 @@ class Snekbox(Cog): await ctx.invoke(self.bot.get_command("help"), "eval") return + if Roles.helpers in (role.id for role in ctx.author.roles): + self.bot.stats.incr("evals.roles.helpers") + else: + self.bot.stats.incr("evals.roles.developers") + log.info(f"Received code from {ctx.author} for evaluation:\n{code}") while True: -- cgit v1.2.3 From 613b00a5ec060f409d5838cb1a648d9770cecfde Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 16 May 2020 18:19:18 +0300 Subject: Stats: Added stats for eval channel using (Help/Bot commands/Topical) --- bot/cogs/snekbox.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 04c0a5ae1..1d240d8d8 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -303,6 +303,13 @@ class Snekbox(Cog): else: self.bot.stats.incr("evals.roles.developers") + if ctx.channel.category_id == Categories.help_in_use: + self.bot.stats.incr("evals.channels.help") + elif ctx.channel.id == Channels.bot_commands: + self.bot.stats.incr("evals.channels.bot_commands") + else: + self.bot.stats.incr("evals.channels.topical") + log.info(f"Received code from {ctx.author} for evaluation:\n{code}") while True: -- cgit v1.2.3 From 588521c82403f6d66693512c6d33272cc370d755 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 20:22:05 +0200 Subject: Refactor - no more mixins! It was brought to my attention that we may need several caches per Cog for some of our Cogs. This means that the original approach of having this be a mixin is a little bit problematic. Instead, RedisDict will be instantiated directly inside the class you want it in. By leveraging __set_name__, we can create a namespace containing both the class name and the variable name without the user having to provide anything. For example, if you create an attribute MyClass.cache = RedisDict(), this will be using the redis namespace 'MyClass.cache.' before anything you store in it. With this approach, it is also possible to instantiate a RedisDict with a custom namespace by simply passing it into the constructor. - RedisDict("firedog") will create items with the 'firedog.your_item' prefix. - If there are multiple RedisDicts using the same namespace, an underscore will be appended to the namespace, such that the second RedisDict("firedog") will actually create items in the 'firedog_.your_item' namespace. This is also possible to use outside of classes, so long as you provide a custom namespace when you instantiate it. Custom namespaces will always take precedence over automatic 'Class.attribute_name' ones. --- bot/mixins/__init__.py | 3 --- bot/mixins/redis.py | 56 ---------------------------------------- bot/utils/__init__.py | 4 +++ bot/utils/redis.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 59 deletions(-) delete mode 100644 bot/mixins/__init__.py delete mode 100644 bot/mixins/redis.py create mode 100644 bot/utils/redis.py diff --git a/bot/mixins/__init__.py b/bot/mixins/__init__.py deleted file mode 100644 index ff1f0c50d..000000000 --- a/bot/mixins/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .redis import RedisCacheMixin - -__all__ = ['RedisCacheMixin'] diff --git a/bot/mixins/redis.py b/bot/mixins/redis.py deleted file mode 100644 index f19108576..000000000 --- a/bot/mixins/redis.py +++ /dev/null @@ -1,56 +0,0 @@ -import redis as redis_py - -redis = redis_py.Redis(host="redis") - - -class RedisDict(dict): - """ - A dictionary interface for a Redis database. - - Objects created by this class should mostly behave like a normal dictionary, - but will store all the data in our Redis database for persistence between restarts. - - There are, however, a few limitations to what kinds of data types can be - stored on Redis, so this is a little bit more limited than a regular dict. - """ - - def __init__(self, namespace: str = "global"): - """Initialize the RedisDict with the right namespace.""" - # TODO: Make namespace collision impossible! - # Append a number or something if it exists already. - self.namespace = namespace - - # redis.mset({"firedog": "donkeykong"}) - # - # print(redis.get("firedog").decode("utf-8") - - -class RedisCacheMixin: - """ - A mixin which adds a cls.cache parameter which can be used for persistent caching. - - This adds a dictionary-like object called cache which can be treated like a regular dictionary, - but which can only store simple data types like ints, strings, and floats. - - To use it, simply subclass it into your class like this: - - class MyCog(Cog, RedisCacheMixin): - def some_command(self): - # You can now do this! - self.cache['some_data'] = some_data - - All the data stored in this cache will probably be available permanently, even if the bot restarts or - is updated. However, Redis is not meant to be used for reliable, permanent storage. It may be cleared - from time to time, so please only use it for caching data that you can afford to lose. - - If it's really important that your data should never disappear, please use our postgres database instead. - """ - - def __init_subclass__(cls, **kwargs): - """ - Initialize the cache when subclass is created. - - When this mixin is subclassed, we create a cache using the subclass name as the namespace. - This is to prevent collisions between subclasses. - """ - cls.cache = RedisDict(cls.__name__) diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 9b32e515d..7ae2db8fe 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -2,6 +2,10 @@ from abc import ABCMeta from discord.ext.commands import CogMeta +from bot.utils.redis import RedisDict + +__all__ = ['RedisDict', 'CogABCMeta'] + class CogABCMeta(CogMeta, ABCMeta): """Metaclass for ABCs meant to be implemented as Cogs.""" diff --git a/bot/utils/redis.py b/bot/utils/redis.py new file mode 100644 index 000000000..8b33e8977 --- /dev/null +++ b/bot/utils/redis.py @@ -0,0 +1,70 @@ +from collections.abc import MutableMapping +from typing import Optional + +import redis as redis_py + +redis = redis_py.Redis(host="redis") + + +class RedisDict(MutableMapping): + """ + A dictionary interface for a Redis database. + + Objects created by this class should mostly behave like a normal dictionary, + but will store all the data in our Redis database for persistence between restarts. + + Redis is limited to simple types, so to allow you to store collections like lists + and dictionaries, we JSON deserialize every value. That means that it will not be possible + to store complex objects, only stuff like strings, numbers, and collections of strings and numbers. + + TODO: Implement these: + __delitem__ + __getitem__ + __setitem__ + __iter__ + __len__ + clear (just use DEL and the hash goes) + copy (convert to dict maybe?) + pop + popitem + setdefault + update + + TODO: TEST THESE + .get + .items + .keys + .values + .__eg__ + .__ne__ + """ + + namespaces = [] + + def _set_namespace(self, namespace: str) -> None: + """Try to set the namespace, but do not permit collisions.""" + while namespace in self.namespaces: + namespace = namespace + "_" + + self.namespaces.append(namespace) + self.namespace = namespace + + def __init__(self, namespace: Optional[str] = None) -> None: + """Initialize the RedisDict with the right namespace.""" + super().__init__() + self.has_custom_namespace = namespace is not None + self._set_namespace(namespace) + + def __set_name__(self, owner: object, attribute_name: str) -> None: + """ + Set the namespace to Class.attribute_name. + + Called automatically when this class is constructed inside a class as an attribute, as long as + no custom namespace is provided to the constructor. + """ + if not self.has_custom_namespace: + self._set_namespace(f"{owner.__name__}.{attribute_name}") + + def __repr__(self) -> str: + """Return a beautiful representation of this object instance.""" + return f"RedisDict(namespace={self.namespace!r})" -- cgit v1.2.3 From ee8386e67aa7d298b4761eef79b927c3066fe037 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 23:40:21 +0200 Subject: Add basic dict methods for RedisDict. The rest of the features should be provided by the MutableMapping abc we're interfacing. Specifically, MutableMapping provides these: .pop, .popitem, .clear, .update, .setdefault, __contains__, .keys, .items, .values, .get, __eq__, and __ne__. --- bot/utils/redis.py | 80 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/bot/utils/redis.py b/bot/utils/redis.py index 8b33e8977..1e8c6422d 100644 --- a/bot/utils/redis.py +++ b/bot/utils/redis.py @@ -1,9 +1,12 @@ +import json from collections.abc import MutableMapping -from typing import Optional +from enum import Enum +from typing import Dict, List, Optional, Tuple, Union import redis as redis_py -redis = redis_py.Redis(host="redis") +ValidRedisKey = Union[str, int, float] +JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] class RedisDict(MutableMapping): @@ -16,45 +19,25 @@ class RedisDict(MutableMapping): Redis is limited to simple types, so to allow you to store collections like lists and dictionaries, we JSON deserialize every value. That means that it will not be possible to store complex objects, only stuff like strings, numbers, and collections of strings and numbers. - - TODO: Implement these: - __delitem__ - __getitem__ - __setitem__ - __iter__ - __len__ - clear (just use DEL and the hash goes) - copy (convert to dict maybe?) - pop - popitem - setdefault - update - - TODO: TEST THESE - .get - .items - .keys - .values - .__eg__ - .__ne__ """ - namespaces = [] - - def _set_namespace(self, namespace: str) -> None: - """Try to set the namespace, but do not permit collisions.""" - while namespace in self.namespaces: - namespace = namespace + "_" - - self.namespaces.append(namespace) - self.namespace = namespace + _namespaces = [] + _redis = redis_py.Redis(host="redis") # Can be overridden for testing def __init__(self, namespace: Optional[str] = None) -> None: """Initialize the RedisDict with the right namespace.""" super().__init__() - self.has_custom_namespace = namespace is not None + self._has_custom_namespace = namespace is not None self._set_namespace(namespace) + def _set_namespace(self, namespace: str) -> None: + """Try to set the namespace, but do not permit collisions.""" + while namespace in self._namespaces: + namespace = namespace + "_" + + self._namespaces.append(namespace) + self._namespace = namespace + def __set_name__(self, owner: object, attribute_name: str) -> None: """ Set the namespace to Class.attribute_name. @@ -62,9 +45,36 @@ class RedisDict(MutableMapping): Called automatically when this class is constructed inside a class as an attribute, as long as no custom namespace is provided to the constructor. """ - if not self.has_custom_namespace: + if not self._has_custom_namespace: self._set_namespace(f"{owner.__name__}.{attribute_name}") def __repr__(self) -> str: """Return a beautiful representation of this object instance.""" - return f"RedisDict(namespace={self.namespace!r})" + return f"RedisDict(namespace={self._namespace!r})" + + def __setitem__(self, key: ValidRedisKey, value: JSONSerializableType): + """Store an item in the Redis cache.""" + # JSON serialize the value before storing it. + json_value = json.dumps(value) + self._redis.hset(self._namespace, key, json_value) + + def __getitem__(self, key: ValidRedisKey): + """Get an item from the Redis cache.""" + value = self._redis.hget(self._namespace, key) + return json.loads(value) + + def __delitem__(self, key: ValidRedisKey): + """Delete an item from the Redis cache.""" + self._redis.hdel(self._namespace, key) + + def __iter__(self): + """Iterate all the items in the Redis cache.""" + return iter(self._redis.hkeys(self._namespace)) + + def __len__(self): + """Return the number of items in the Redis cache.""" + return self._redis.hlen(self._namespace) + + def copy(self) -> Dict: + """Convert to dict and return.""" + return dict(self) -- cgit v1.2.3 From 5859711a05e634d7b788944b8292071cb1e7cf72 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 16 May 2020 23:43:22 +0200 Subject: copy should dictify the .items(), not just keys. --- bot/utils/redis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/redis.py b/bot/utils/redis.py index 1e8c6422d..470de47b7 100644 --- a/bot/utils/redis.py +++ b/bot/utils/redis.py @@ -77,4 +77,4 @@ class RedisDict(MutableMapping): def copy(self) -> Dict: """Convert to dict and return.""" - return dict(self) + return dict(self.items()) -- cgit v1.2.3 From 9eeee1ce303b7ebac4fa9db37193921d052d0f8d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 00:10:29 +0200 Subject: Implements .clear with hash deletion. This would've been implemented by MutableMapping, but that implementation is O(n) instead of O(1) since it just iterates the entire hash and does HDEL. Feels wasteful. --- bot/utils/redis.py | 80 ---------------------------------------------- bot/utils/redis_dict.py | 84 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 80 deletions(-) delete mode 100644 bot/utils/redis.py create mode 100644 bot/utils/redis_dict.py diff --git a/bot/utils/redis.py b/bot/utils/redis.py deleted file mode 100644 index 470de47b7..000000000 --- a/bot/utils/redis.py +++ /dev/null @@ -1,80 +0,0 @@ -import json -from collections.abc import MutableMapping -from enum import Enum -from typing import Dict, List, Optional, Tuple, Union - -import redis as redis_py - -ValidRedisKey = Union[str, int, float] -JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] - - -class RedisDict(MutableMapping): - """ - A dictionary interface for a Redis database. - - Objects created by this class should mostly behave like a normal dictionary, - but will store all the data in our Redis database for persistence between restarts. - - Redis is limited to simple types, so to allow you to store collections like lists - and dictionaries, we JSON deserialize every value. That means that it will not be possible - to store complex objects, only stuff like strings, numbers, and collections of strings and numbers. - """ - - _namespaces = [] - _redis = redis_py.Redis(host="redis") # Can be overridden for testing - - def __init__(self, namespace: Optional[str] = None) -> None: - """Initialize the RedisDict with the right namespace.""" - super().__init__() - self._has_custom_namespace = namespace is not None - self._set_namespace(namespace) - - def _set_namespace(self, namespace: str) -> None: - """Try to set the namespace, but do not permit collisions.""" - while namespace in self._namespaces: - namespace = namespace + "_" - - self._namespaces.append(namespace) - self._namespace = namespace - - def __set_name__(self, owner: object, attribute_name: str) -> None: - """ - Set the namespace to Class.attribute_name. - - Called automatically when this class is constructed inside a class as an attribute, as long as - no custom namespace is provided to the constructor. - """ - if not self._has_custom_namespace: - self._set_namespace(f"{owner.__name__}.{attribute_name}") - - def __repr__(self) -> str: - """Return a beautiful representation of this object instance.""" - return f"RedisDict(namespace={self._namespace!r})" - - def __setitem__(self, key: ValidRedisKey, value: JSONSerializableType): - """Store an item in the Redis cache.""" - # JSON serialize the value before storing it. - json_value = json.dumps(value) - self._redis.hset(self._namespace, key, json_value) - - def __getitem__(self, key: ValidRedisKey): - """Get an item from the Redis cache.""" - value = self._redis.hget(self._namespace, key) - return json.loads(value) - - def __delitem__(self, key: ValidRedisKey): - """Delete an item from the Redis cache.""" - self._redis.hdel(self._namespace, key) - - def __iter__(self): - """Iterate all the items in the Redis cache.""" - return iter(self._redis.hkeys(self._namespace)) - - def __len__(self): - """Return the number of items in the Redis cache.""" - return self._redis.hlen(self._namespace) - - def copy(self) -> Dict: - """Convert to dict and return.""" - return dict(self.items()) diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py new file mode 100644 index 000000000..b2fd7d2e9 --- /dev/null +++ b/bot/utils/redis_dict.py @@ -0,0 +1,84 @@ +import json +from collections.abc import MutableMapping +from enum import Enum +from typing import Dict, List, Optional, Tuple, Union + +import redis as redis_py + +ValidRedisKey = Union[str, int, float] +JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] + + +class RedisDict(MutableMapping): + """ + A dictionary interface for a Redis database. + + Objects created by this class should mostly behave like a normal dictionary, + but will store all the data in our Redis database for persistence between restarts. + + Redis is limited to simple types, so to allow you to store collections like lists + and dictionaries, we JSON deserialize every value. That means that it will not be possible + to store complex objects, only stuff like strings, numbers, and collections of strings and numbers. + """ + + _namespaces = [] + _redis = redis_py.Redis(host="redis") # Can be overridden for testing + + def __init__(self, namespace: Optional[str] = None) -> None: + """Initialize the RedisDict with the right namespace.""" + super().__init__() + self._has_custom_namespace = namespace is not None + self._set_namespace(namespace) + + def _set_namespace(self, namespace: str) -> None: + """Try to set the namespace, but do not permit collisions.""" + while namespace in self._namespaces: + namespace = namespace + "_" + + self._namespaces.append(namespace) + self._namespace = namespace + + def __set_name__(self, owner: object, attribute_name: str) -> None: + """ + Set the namespace to Class.attribute_name. + + Called automatically when this class is constructed inside a class as an attribute, as long as + no custom namespace is provided to the constructor. + """ + if not self._has_custom_namespace: + self._set_namespace(f"{owner.__name__}.{attribute_name}") + + def __repr__(self) -> str: + """Return a beautiful representation of this object instance.""" + return f"RedisDict(namespace={self._namespace!r})" + + def __setitem__(self, key: ValidRedisKey, value: JSONSerializableType): + """Store an item in the Redis cache.""" + # JSON serialize the value before storing it. + json_value = json.dumps(value) + self._redis.hset(self._namespace, key, json_value) + + def __getitem__(self, key: ValidRedisKey): + """Get an item from the Redis cache.""" + value = self._redis.hget(self._namespace, key) + return json.loads(value) + + def __delitem__(self, key: ValidRedisKey): + """Delete an item from the Redis cache.""" + self._redis.hdel(self._namespace, key) + + def __iter__(self): + """Iterate all the items in the Redis cache.""" + return iter(self._redis.hkeys(self._namespace)) + + def __len__(self): + """Return the number of items in the Redis cache.""" + return self._redis.hlen(self._namespace) + + def copy(self) -> Dict: + """Convert to dict and return.""" + return dict(self.items()) + + def clear(self) -> None: + """Deletes the entire hash from the Redis cache.""" + self._redis.delete(self._namespace) -- cgit v1.2.3 From 677a7f755a15f8fdf0cd97e399c4265dd8e702d9 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 02:30:28 +0200 Subject: Implement .get, equality, and membership check This is supposed to be provided by our MutableMapping mixin, but unit tests are demonstrating that these don't really work as intended. --- bot/utils/redis_dict.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py index b2fd7d2e9..35439b2f3 100644 --- a/bot/utils/redis_dict.py +++ b/bot/utils/redis_dict.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json from collections.abc import MutableMapping from enum import Enum @@ -28,7 +30,11 @@ class RedisDict(MutableMapping): """Initialize the RedisDict with the right namespace.""" super().__init__() self._has_custom_namespace = namespace is not None - self._set_namespace(namespace) + + if self._has_custom_namespace: + self._set_namespace(namespace) + else: + self.namespace = "general" def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" @@ -52,6 +58,14 @@ class RedisDict(MutableMapping): """Return a beautiful representation of this object instance.""" return f"RedisDict(namespace={self._namespace!r})" + def __eq__(self, other: RedisDict) -> bool: + """Check equality between two RedisDicts.""" + return self.items() == other.items() and self._namespace == other._namespace + + def __ne__(self, other: RedisDict) -> bool: + """Check inequality between two RedisDicts.""" + return self.items() != other.items() or self._namespace != other._namespace + def __setitem__(self, key: ValidRedisKey, value: JSONSerializableType): """Store an item in the Redis cache.""" # JSON serialize the value before storing it. @@ -61,12 +75,18 @@ class RedisDict(MutableMapping): def __getitem__(self, key: ValidRedisKey): """Get an item from the Redis cache.""" value = self._redis.hget(self._namespace, key) - return json.loads(value) + + if value: + return json.loads(value) def __delitem__(self, key: ValidRedisKey): """Delete an item from the Redis cache.""" self._redis.hdel(self._namespace, key) + def __contains__(self, key: ValidRedisKey): + """Check if a key exists in the Redis cache.""" + return self._redis.hexists(self._namespace, key) + def __iter__(self): """Iterate all the items in the Redis cache.""" return iter(self._redis.hkeys(self._namespace)) @@ -82,3 +102,10 @@ class RedisDict(MutableMapping): def clear(self) -> None: """Deletes the entire hash from the Redis cache.""" self._redis.delete(self._namespace) + + def get(self, key: ValidRedisKey, default: Optional[str] = None) -> JSONSerializableType: + """Get the item, but provide a default if not found.""" + if key in self: + return self[key] + else: + return default -- cgit v1.2.3 From 0843728927fe73be2f2e2c37381ead1497debe11 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 02:40:30 +0200 Subject: Add fakeredis to the Pipfile --- Pipfile | 3 ++- Pipfile.lock | 25 ++++++++++++++++++++++++- bot/utils/__init__.py | 2 +- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/Pipfile b/Pipfile index 5f85b1e51..40ae52761 100644 --- a/Pipfile +++ b/Pipfile @@ -23,10 +23,11 @@ colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} statsd = "~=3.3" feedparser = "~=5.2" beautifulsoup4 = "~=4.9" -redis = ">=3.5" +redis = "~=3.5" [dev-packages] coverage = "~=5.0" +fakeredis = "~=1.4" flake8 = "~=3.7" flake8-annotations = "~=2.0" flake8-bugbear = "~=20.1" diff --git a/Pipfile.lock b/Pipfile.lock index 1a420182d..414f4a053 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c6b4d38c4034e55a4bd598399f2e1f48b70a76693c986d0db0fae7442e224d41" + "sha256": "8ec71e9c46d52bf3b8c72939519e993715c79b4bc9e6ad164c1cf88951dc48b4" }, "pipfile-spec": 6, "requires": { @@ -665,6 +665,14 @@ ], "version": "==0.3.0" }, + "fakeredis": { + "hashes": [ + "sha256:4d170886865a91dbc8b7f8cbd4e5d488f4c5f2f25dfae127f001617bbe9e8f97", + "sha256:647b2593d349d9d4e566c8dadb2e4c71ba35be5bdc4f1f7ac2d565a12a965053" + ], + "index": "pypi", + "version": "==1.4.1" + }, "filelock": { "hashes": [ "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", @@ -816,6 +824,14 @@ "index": "pypi", "version": "==5.3.1" }, + "redis": { + "hashes": [ + "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", + "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" + ], + "index": "pypi", + "version": "==3.5.2" + }, "six": { "hashes": [ "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", @@ -830,6 +846,13 @@ ], "version": "==2.0.0" }, + "sortedcontainers": { + "hashes": [ + "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", + "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" + ], + "version": "==2.1.0" + }, "toml": { "hashes": [ "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 7ae2db8fe..5ce383bf2 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -2,7 +2,7 @@ from abc import ABCMeta from discord.ext.commands import CogMeta -from bot.utils.redis import RedisDict +from bot.utils.redis_dict import RedisDict __all__ = ['RedisDict', 'CogABCMeta'] -- cgit v1.2.3 From 7cf0e83d1079ed34a3839948ce6823d95e0ebb62 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 03:16:03 +0200 Subject: Implement .pop, .popitem and .setdefault. Turns out the MutableMapping class doesn't give us servicable implementations of these, so we need to implement them ourselves. Also, let's not have keys returned as bytestrings. --- bot/utils/redis_dict.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py index 35439b2f3..c89765a24 100644 --- a/bot/utils/redis_dict.py +++ b/bot/utils/redis_dict.py @@ -89,7 +89,8 @@ class RedisDict(MutableMapping): def __iter__(self): """Iterate all the items in the Redis cache.""" - return iter(self._redis.hkeys(self._namespace)) + keys = self._redis.hkeys(self._namespace) + return iter([key.decode('utf-8') for key in keys]) def __len__(self): """Return the number of items in the Redis cache.""" @@ -103,9 +104,28 @@ class RedisDict(MutableMapping): """Deletes the entire hash from the Redis cache.""" self._redis.delete(self._namespace) - def get(self, key: ValidRedisKey, default: Optional[str] = None) -> JSONSerializableType: + def get(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: """Get the item, but provide a default if not found.""" if key in self: return self[key] else: return default + + def pop(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + """Get the item, remove it from the cache, and provide a default if not found.""" + value = self.get(key, default) + del self[key] + return value + + def popitem(self) -> JSONSerializableType: + """Get the last item added to the cache.""" + key = list(self.keys())[-1] + return self.pop(key) + + def setdefault(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + """Try to get the item. If the item does not exist, set it to `default` and return that.""" + value = self.get(key) + + if value is None: + self[key] = default + return default -- cgit v1.2.3 From bf6c113319d47594e103c71f8ff5b0ea48d15b38 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 03:18:19 +0200 Subject: Test suite for the redis dict. --- tests/bot/utils/test_redis_dict.py | 189 +++++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 tests/bot/utils/test_redis_dict.py diff --git a/tests/bot/utils/test_redis_dict.py b/tests/bot/utils/test_redis_dict.py new file mode 100644 index 000000000..f422887ce --- /dev/null +++ b/tests/bot/utils/test_redis_dict.py @@ -0,0 +1,189 @@ +import unittest + +import fakeredis +from redis import DataError + +from bot.utils import RedisDict + +redis_server = fakeredis.FakeServer() +RedisDict._redis = fakeredis.FakeStrictRedis(server=redis_server) + + +class RedisDictTests(unittest.TestCase): + """Tests the RedisDict class from utils.redis_dict.py.""" + + redis = RedisDict() + + def test_class_attribute_namespace(self): + """Test that RedisDict creates a namespace automatically for class attributes.""" + self.assertEqual(self.redis._namespace, "RedisDictTests.redis") + + def test_custom_namespace(self): + """Test that users can set a custom namespaces which never collide.""" + test_cases = ( + (RedisDict("firedog")._namespace, "firedog"), + (RedisDict("firedog")._namespace, "firedog_"), + (RedisDict("firedog")._namespace, "firedog__"), + ) + + for test_case, result in test_cases: + self.assertEqual(test_case, result) + + def test_custom_namespace_takes_precedence(self): + """Test that custom namespaces take precedence over class attribute ones.""" + class LemonJuice: + citrus = RedisDict("citrus") + watercat = RedisDict() + + test_class = LemonJuice() + self.assertEqual(test_class.citrus._namespace, "citrus") + self.assertEqual(test_class.watercat._namespace, "LemonJuice.watercat") + + def test_set_get_item(self): + """Test that users can set and get items from the RedisDict.""" + self.redis['favorite_fruit'] = 'melon' + self.redis['favorite_number'] = 86 + self.assertEqual(self.redis['favorite_fruit'], 'melon') + self.assertEqual(self.redis['favorite_number'], 86) + + def test_set_item_value_types(self): + """Test that setitem rejects values that are not JSON serializable.""" + with self.assertRaises(TypeError): + self.redis['favorite_thing'] = object + self.redis['favorite_stuff'] = RedisDict + + def test_set_item_key_types(self): + """Test that setitem rejects keys that are not strings, ints or floats.""" + fruits = ["lemon", "melon", "apple"] + + with self.assertRaises(DataError): + self.redis[fruits] = "nice" + + def test_get_method(self): + """Test that the .get method works like in a dict.""" + self.redis['favorite_movie'] = 'Code Jam Highlights' + + self.assertEqual(self.redis.get('favorite_movie'), 'Code Jam Highlights') + self.assertEqual(self.redis.get('favorite_youtuber', 'pydis'), 'pydis') + self.assertIsNone(self.redis.get('favorite_dog')) + + def test_membership(self): + """Test that we can reliably use the `in` operator with our RedisDict.""" + self.redis['favorite_country'] = "Burkina Faso" + + self.assertIn('favorite_country', self.redis) + self.assertNotIn('favorite_dentist', self.redis) + + def test_del_item(self): + """Test that users can delete items from the RedisDict.""" + self.redis['favorite_band'] = "Radiohead" + self.assertIn('favorite_band', self.redis) + + del self.redis['favorite_band'] + self.assertNotIn('favorite_band', self.redis) + + def test_iter(self): + """Test that the RedisDict can be iterated.""" + self.redis.clear() + test_cases = ( + ('favorite_turtle', 'Donatello'), + ('second_favorite_turtle', 'Leonardo'), + ('third_favorite_turtle', 'Raphael'), + ) + for key, value in test_cases: + self.redis[key] = value + + # Test regular iteration + for test_case, key in zip(test_cases, self.redis): + value = test_case[1] + self.assertEqual(self.redis[key], value) + + # Test .items iteration + for key, value in self.redis.items(): + self.assertEqual(self.redis[key], value) + + # Test .keys iteration + for test_case, key in zip(test_cases, self.redis.keys()): + value = test_case[1] + self.assertEqual(self.redis[key], value) + + def test_len(self): + """Test that we can get the correct len() from the RedisDict.""" + self.redis.clear() + self.redis['one'] = 1 + self.redis['two'] = 2 + self.redis['three'] = 3 + self.assertEqual(len(self.redis), 3) + + self.redis['four'] = 4 + self.assertEqual(len(self.redis), 4) + + def test_copy(self): + """Test that the .copy method returns a workable dictionary copy.""" + copy = self.redis.copy() + local_copy = dict(self.redis.items()) + self.assertIs(type(copy), dict) + self.assertEqual(copy, local_copy) + + def test_clear(self): + """Test that the .clear method removes the entire hash.""" + self.redis.clear() + self.redis['teddy'] = "with me" + self.redis['in my dreams'] = "you have a weird hat" + self.assertEqual(len(self.redis), 2) + + self.redis.clear() + self.assertEqual(len(self.redis), 0) + + def test_pop(self): + """Test that we can .pop an item from the RedisDict.""" + self.redis.clear() + self.redis['john'] = 'was afraid' + + self.assertEqual(self.redis.pop('john'), 'was afraid') + self.assertEqual(self.redis.pop('pete', 'breakneck'), 'breakneck') + self.assertEqual(len(self.redis), 0) + + def test_popitem(self): + """Test that we can .popitem an item from the RedisDict.""" + self.redis.clear() + self.redis['john'] = 'the revalator' + self.redis['teddy'] = 'big bear' + + self.assertEqual(len(self.redis), 2) + self.assertEqual(self.redis.popitem(), 'big bear') + self.assertEqual(len(self.redis), 1) + + def test_setdefault(self): + """Test that we can .setdefault an item from the RedisDict.""" + self.redis.clear() + self.redis.setdefault('john', 'is yellow and weak') + self.assertEqual(self.redis['john'], 'is yellow and weak') + + with self.assertRaises(TypeError): + self.redis.setdefault('geisha', object) + + def test_update(self): + """Test that we can .update the RedisDict with multiple items.""" + self.redis.clear() + self.redis["reckfried"] = "lona" + self.redis["bel air"] = "prince" + self.redis.update({ + "reckfried": "jona", + "mega": "hungry, though", + }) + + result = { + "reckfried": "jona", + "bel air": "prince", + "mega": "hungry, though", + } + self.assertEqual(self.redis.copy(), result) + + def test_equals(self): + """Test that RedisDicts can be compared with == and !=.""" + new_redis_dict = RedisDict("firedog_the_sequel") + new_new_redis_dict = new_redis_dict + + self.assertEqual(new_redis_dict, new_new_redis_dict) + self.assertNotEqual(new_redis_dict, self.redis) -- cgit v1.2.3 From 258ad9d9c5601e01d135e706c256c0cd7f7fdbe0 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 03:29:53 +0200 Subject: Make redis host and port configurable. --- bot/constants.py | 3 +++ bot/utils/redis_dict.py | 7 ++++++- config-default.yml | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/bot/constants.py b/bot/constants.py index fd280e9de..01e8ac3a3 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -199,6 +199,9 @@ class Bot(metaclass=YAMLGetter): prefix: str token: str sentry_dsn: str + redis_host: str + redis_port: int + class Filter(metaclass=YAMLGetter): section = "filter" diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py index c89765a24..dfb1c7252 100644 --- a/bot/utils/redis_dict.py +++ b/bot/utils/redis_dict.py @@ -7,6 +7,8 @@ from typing import Dict, List, Optional, Tuple, Union import redis as redis_py +from bot import constants + ValidRedisKey = Union[str, int, float] JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] @@ -24,7 +26,10 @@ class RedisDict(MutableMapping): """ _namespaces = [] - _redis = redis_py.Redis(host="redis") # Can be overridden for testing + _redis = redis_py.Redis( + host=constants.Bot.redis_host, + port=constants.Bot.redis_port, + ) # Can be overridden for testing def __init__(self, namespace: Optional[str] = None) -> None: """Initialize the RedisDict with the right namespace.""" diff --git a/config-default.yml b/config-default.yml index 83ea59016..722afa41b 100644 --- a/config-default.yml +++ b/config-default.yml @@ -2,6 +2,8 @@ bot: prefix: "!" token: !ENV "BOT_TOKEN" sentry_dsn: !ENV "BOT_SENTRY_DSN" + redis_host: "redis" + redis_port: 6379 stats: statsd_host: "graphite" -- cgit v1.2.3 From 8456023252c2d4c91c6566ee0a3f83e9033d45d9 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 03:39:52 +0200 Subject: namespace "general" -> "global" --- bot/utils/redis_dict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py index dfb1c7252..47905314a 100644 --- a/bot/utils/redis_dict.py +++ b/bot/utils/redis_dict.py @@ -39,7 +39,7 @@ class RedisDict(MutableMapping): if self._has_custom_namespace: self._set_namespace(namespace) else: - self.namespace = "general" + self.namespace = "global" def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" -- cgit v1.2.3 From e993566fe5d816fadee64aaca454ce6ba463bca2 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 11:35:34 +0200 Subject: Fix linting errors introduced by flake8 3.8 Turns out that bumping the flake8 version up to 3.8 introduces a long list of new linting errors. Since this PR is the one that bumps the version, I suppose we will also fix all the linting errors in this branch. --- bot/cogs/antispam.py | 4 ++-- bot/cogs/defcon.py | 2 +- bot/cogs/duck_pond.py | 2 +- bot/cogs/error_handler.py | 4 ++-- bot/cogs/help_channels.py | 2 +- bot/cogs/moderation/management.py | 10 +++++----- bot/cogs/moderation/scheduler.py | 10 +++++----- bot/cogs/moderation/silence.py | 2 +- bot/cogs/stats.py | 4 ++-- bot/cogs/utils.py | 4 ++-- bot/cogs/watchchannels/talentpool.py | 2 +- bot/cogs/watchchannels/watchchannel.py | 14 +++++++------- bot/decorators.py | 2 +- bot/pagination.py | 4 ++-- bot/utils/messages.py | 2 +- 15 files changed, 34 insertions(+), 34 deletions(-) diff --git a/bot/cogs/antispam.py b/bot/cogs/antispam.py index d63acbc4a..0bcca578d 100644 --- a/bot/cogs/antispam.py +++ b/bot/cogs/antispam.py @@ -94,7 +94,7 @@ class DeletionContext: await modlog.send_log_message( icon_url=Icons.filtering, colour=Colour(Colours.soft_red), - title=f"Spam detected!", + title="Spam detected!", text=mod_alert_message, thumbnail=last_message.author.avatar_url_as(static_format="png"), channel_id=Channels.mod_alerts, @@ -130,7 +130,7 @@ class AntiSpam(Cog): body += "\n\n**The cog has been unloaded.**" await self.mod_log.send_log_message( - title=f"Error: AntiSpam configuration validation failed!", + title="Error: AntiSpam configuration validation failed!", text=body, ping_everyone=True, icon_url=Icons.token_removed, diff --git a/bot/cogs/defcon.py b/bot/cogs/defcon.py index 56fca002a..f4cb0aa58 100644 --- a/bot/cogs/defcon.py +++ b/bot/cogs/defcon.py @@ -81,7 +81,7 @@ class Defcon(Cog): else: self.enabled = False self.days = timedelta(days=0) - log.info(f"DEFCON disabled") + log.info("DEFCON disabled") await self.update_channel_topic() diff --git a/bot/cogs/duck_pond.py b/bot/cogs/duck_pond.py index 1f84a0609..37d1786a2 100644 --- a/bot/cogs/duck_pond.py +++ b/bot/cogs/duck_pond.py @@ -117,7 +117,7 @@ class DuckPond(Cog): avatar_url=message.author.avatar_url ) except discord.HTTPException: - log.exception(f"Failed to send an attachment to the webhook") + log.exception("Failed to send an attachment to the webhook") await message.add_reaction("✅") diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index b2f4c59f6..16790c769 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -173,7 +173,7 @@ class ErrorHandler(Cog): await ctx.invoke(*help_command) self.bot.stats.incr("errors.missing_required_argument") elif isinstance(e, errors.TooManyArguments): - await ctx.send(f"Too many arguments provided.") + await ctx.send("Too many arguments provided.") await ctx.invoke(*help_command) self.bot.stats.incr("errors.too_many_arguments") elif isinstance(e, errors.BadArgument): @@ -213,7 +213,7 @@ class ErrorHandler(Cog): if isinstance(e, bot_missing_errors): ctx.bot.stats.incr("errors.bot_permission_error") await ctx.send( - f"Sorry, it looks like I don't have the permissions or roles I need to do that." + "Sorry, it looks like I don't have the permissions or roles I need to do that." ) elif isinstance(e, (InWhitelistCheckFailure, errors.NoPrivateMessage)): ctx.bot.stats.incr("errors.wrong_channel_or_dm_error") diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 1bd1f9d68..a20fe2b05 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -391,7 +391,7 @@ class HelpChannels(Scheduler, commands.Cog): self.in_use_category = await self.try_get_channel(constants.Categories.help_in_use) self.dormant_category = await self.try_get_channel(constants.Categories.help_dormant) except discord.HTTPException: - log.exception(f"Failed to get a category; cog will be removed") + log.exception("Failed to get a category; cog will be removed") self.bot.remove_cog(self.qualified_name) async def init_cog(self) -> None: diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index 250a24247..82ec6b0d9 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -49,8 +49,8 @@ class ModManagement(commands.Cog): async def infraction_edit( self, ctx: Context, - infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], - duration: t.Union[Expiry, allowed_strings("p", "permanent"), None], + infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], # noqa: F821 + duration: t.Union[Expiry, allowed_strings("p", "permanent"), None], # noqa: F821 *, reason: str = None ) -> None: @@ -83,14 +83,14 @@ class ModManagement(commands.Cog): "actor__id": ctx.author.id, "ordering": "-inserted_at" } - infractions = await self.bot.api_client.get(f"bot/infractions", params=params) + infractions = await self.bot.api_client.get("bot/infractions", params=params) if infractions: old_infraction = infractions[0] infraction_id = old_infraction["id"] else: await ctx.send( - f":x: Couldn't find most recent infraction; you have never given an infraction." + ":x: Couldn't find most recent infraction; you have never given an infraction." ) return else: @@ -224,7 +224,7 @@ class ModManagement(commands.Cog): ) -> None: """Send a paginated embed of infractions for the specified user.""" if not infractions: - await ctx.send(f":warning: No infractions could be found for that query.") + await ctx.send(":warning: No infractions could be found for that query.") return lines = tuple( diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index dc42bee2e..012432e60 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -91,7 +91,7 @@ class InfractionScheduler(Scheduler): log.trace(f"Applying {infr_type} infraction #{id_} to {user}.") # Default values for the confirmation message and mod log. - confirm_msg = f":ok_hand: applied" + confirm_msg = ":ok_hand: applied" # Specifying an expiry for a note or warning makes no sense. if infr_type in ("note", "warning"): @@ -154,7 +154,7 @@ class InfractionScheduler(Scheduler): self.schedule_task(infraction["id"], infraction) except discord.HTTPException as e: # Accordingly display that applying the infraction failed. - confirm_msg = f":x: failed to apply" + confirm_msg = ":x: failed to apply" expiry_msg = "" log_content = ctx.author.mention log_title = "failed to apply" @@ -281,7 +281,7 @@ class InfractionScheduler(Scheduler): log.warning(f"Failed to pardon {infr_type} infraction #{id_} for {user}.") else: - confirm_msg = f":ok_hand: pardoned" + confirm_msg = ":ok_hand: pardoned" log_title = "pardoned" log.info(f"Pardoned {infr_type} infraction #{id_} for {user}.") @@ -353,7 +353,7 @@ class InfractionScheduler(Scheduler): ) except discord.Forbidden: log.warning(f"Failed to deactivate infraction #{id_} ({type_}): bot lacks permissions.") - log_text["Failure"] = f"The bot lacks permissions to do this (role hierarchy?)" + log_text["Failure"] = "The bot lacks permissions to do this (role hierarchy?)" log_content = mod_role.mention except discord.HTTPException as e: log.exception(f"Failed to deactivate infraction #{id_} ({type_})") @@ -402,7 +402,7 @@ class InfractionScheduler(Scheduler): # Send a log message to the mod log. if send_log: - log_title = f"expiration failed" if "Failure" in log_text else "expired" + log_title = "expiration failed" if "Failure" in log_text else "expired" user = self.bot.get_user(user_id) avatar = user.avatar_url_as(static_format="png") if user else None diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index 1ef3967a9..25febfa51 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -91,7 +91,7 @@ class Silence(commands.Cog): await ctx.send(f"{Emojis.check_mark} silenced current channel for {duration} minute(s).") await asyncio.sleep(duration*60) - log.info(f"Unsilencing channel after set delay.") + log.info("Unsilencing channel after set delay.") await ctx.invoke(self.unsilence) @commands.command(aliases=("unhush",)) diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py index d253db913..e088c2b87 100644 --- a/bot/cogs/stats.py +++ b/bot/cogs/stats.py @@ -59,7 +59,7 @@ class Stats(Cog): if member.guild.id != Guild.id: return - self.bot.stats.gauge(f"guild.total_members", len(member.guild.members)) + self.bot.stats.gauge("guild.total_members", len(member.guild.members)) @Cog.listener() async def on_member_leave(self, member: Member) -> None: @@ -67,7 +67,7 @@ class Stats(Cog): if member.guild.id != Guild.id: return - self.bot.stats.gauge(f"guild.total_members", len(member.guild.members)) + self.bot.stats.gauge("guild.total_members", len(member.guild.members)) @Cog.listener() async def on_member_update(self, _before: Member, after: Member) -> None: diff --git a/bot/cogs/utils.py b/bot/cogs/utils.py index 89d556f58..f76daedac 100644 --- a/bot/cogs/utils.py +++ b/bot/cogs/utils.py @@ -253,8 +253,8 @@ class Utils(Cog): async def send_pep_zero(self, ctx: Context) -> None: """Send information about PEP 0.""" pep_embed = Embed( - title=f"**PEP 0 - Index of Python Enhancement Proposals (PEPs)**", - description=f"[Link](https://www.python.org/dev/peps/)" + title="**PEP 0 - Index of Python Enhancement Proposals (PEPs)**", + description="[Link](https://www.python.org/dev/peps/)" ) pep_embed.set_thumbnail(url=ICON_URL) pep_embed.add_field(name="Status", value="Active") diff --git a/bot/cogs/watchchannels/talentpool.py b/bot/cogs/watchchannels/talentpool.py index ad0c51fa6..68b220233 100644 --- a/bot/cogs/watchchannels/talentpool.py +++ b/bot/cogs/watchchannels/talentpool.py @@ -61,7 +61,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): return if isinstance(user, Member) and any(role.id in STAFF_ROLES for role in user.roles): - await ctx.send(f":x: Nominating staff members, eh? Here's a cookie :cookie:") + await ctx.send(":x: Nominating staff members, eh? Here's a cookie :cookie:") return if not await self.fetch_user_cache(): diff --git a/bot/cogs/watchchannels/watchchannel.py b/bot/cogs/watchchannels/watchchannel.py index 479820444..643cd46e4 100644 --- a/bot/cogs/watchchannels/watchchannel.py +++ b/bot/cogs/watchchannels/watchchannel.py @@ -82,7 +82,7 @@ class WatchChannel(metaclass=CogABCMeta): exc = self._consume_task.exception() if exc: self.log.exception( - f"The message queue consume task has failed with:", + "The message queue consume task has failed with:", exc_info=exc ) return False @@ -146,7 +146,7 @@ class WatchChannel(metaclass=CogABCMeta): try: data = await self.bot.api_client.get(self.api_endpoint, params=self.api_default_params) except ResponseCodeError as err: - self.log.exception(f"Failed to fetch the watched users from the API", exc_info=err) + self.log.exception("Failed to fetch the watched users from the API", exc_info=err) return False self.watched_users = defaultdict(dict) @@ -173,7 +173,7 @@ class WatchChannel(metaclass=CogABCMeta): self.log.trace(f"Sleeping {BigBrotherConfig.log_delay} seconds before consuming message queue") await asyncio.sleep(BigBrotherConfig.log_delay) - self.log.trace(f"Started consuming the message queue") + self.log.trace("Started consuming the message queue") # If the previous consumption Task failed, first consume the existing comsumption_queue if not self.consumption_queue: @@ -208,7 +208,7 @@ class WatchChannel(metaclass=CogABCMeta): await self.webhook.send(content=content, username=username, avatar_url=avatar_url, embed=embed) except discord.HTTPException as exc: self.log.exception( - f"Failed to send a message to the webhook", + "Failed to send a message to the webhook", exc_info=exc ) @@ -254,7 +254,7 @@ class WatchChannel(metaclass=CogABCMeta): ) except discord.HTTPException as exc: self.log.exception( - f"Failed to send an attachment to the webhook", + "Failed to send an attachment to the webhook", exc_info=exc ) @@ -326,13 +326,13 @@ class WatchChannel(metaclass=CogABCMeta): def cog_unload(self) -> None: """Takes care of unloading the cog and canceling the consumption task.""" - self.log.trace(f"Unloading the cog") + self.log.trace("Unloading the cog") if self._consume_task and not self._consume_task.done(): self._consume_task.cancel() try: self._consume_task.result() except asyncio.CancelledError as e: self.log.exception( - f"The consume task was canceled. Messages may be lost.", + "The consume task was canceled. Messages may be lost.", exc_info=e ) diff --git a/bot/decorators.py b/bot/decorators.py index 2ee5879f2..dc9c7d439 100644 --- a/bot/decorators.py +++ b/bot/decorators.py @@ -121,7 +121,7 @@ def locked() -> Callable: embed = Embed() embed.colour = Colour.red() - log.debug(f"User tried to invoke a locked command.") + log.debug("User tried to invoke a locked command.") embed.description = ( "You're already using this command. Please wait until it is done before you use it again." ) diff --git a/bot/pagination.py b/bot/pagination.py index 90c8f849c..09759d5be 100644 --- a/bot/pagination.py +++ b/bot/pagination.py @@ -147,7 +147,7 @@ class LinePaginator(Paginator): if not lines: if exception_on_empty_embed: - log.exception(f"Pagination asked for empty lines iterable") + log.exception("Pagination asked for empty lines iterable") raise EmptyPaginatorEmbed("No lines to paginate") log.debug("No lines to add to paginator, adding '(nothing to display)' message") @@ -357,7 +357,7 @@ class ImagePaginator(Paginator): if not pages: if exception_on_empty_embed: - log.exception(f"Pagination asked for empty image list") + log.exception("Pagination asked for empty image list") raise EmptyPaginatorEmbed("No images to paginate") log.debug("No images to add to paginator, adding '(no images to display)' message") diff --git a/bot/utils/messages.py b/bot/utils/messages.py index e969ee590..de8e186f3 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -100,7 +100,7 @@ async def send_attachments( log.warning(f"{failure_msg} with status {e.status}.") if link_large and large: - desc = f"\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large) + desc = "\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large) embed = Embed(description=desc) embed.set_footer(text="Attachments exceed upload size limit.") -- cgit v1.2.3 From adf50a6ddc6868dec108ad471c5f3f4033ccd69b Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 19:44:29 +0200 Subject: Changes discord-py to discord.py in Pipfile The `discord-py` package is no longer the official release, and so making this change silences some warnings about deprecation. --- Pipfile | 2 +- Pipfile.lock | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Pipfile b/Pipfile index 40ae52761..1d6cd7015 100644 --- a/Pipfile +++ b/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -discord-py = "~=1.3.2" +discord.py = "~=1.3.2" aiodns = "~=2.0" aiohttp = "~=3.5" sphinx = "~=2.2" diff --git a/Pipfile.lock b/Pipfile.lock index 414f4a053..25383b355 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8ec71e9c46d52bf3b8c72939519e993715c79b4bc9e6ad164c1cf88951dc48b4" + "sha256": "49c231092320b48c5a7618bf048a477f7e0ed33dcbfb71c6dc8f18ef819dd935" }, "pipfile-spec": 6, "requires": { @@ -166,11 +166,19 @@ "index": "pypi", "version": "==4.3.2" }, - "discord-py": { + "discord": { "hashes": [ - "sha256:406871b06d86c3dc49fba63238519f28628dac946fef8a0e22988ff58ec05580" + "sha256:9d4debb4a37845543bd4b92cb195bc53a302797333e768e70344222857ff1559", + "sha256:ff6653655e342e7721dfb3f10421345fd852c2a33f2cca912b1c39b3778a9429" ], "index": "pypi", + "version": "==1.0.1" + }, + "discord.py": { + "hashes": [ + "sha256:406871b06d86c3dc49fba63238519f28628dac946fef8a0e22988ff58ec05580", + "sha256:ad00e34c72d2faa8db2157b651d05f3c415d7d05078e7e41dc9e8dc240051beb" + ], "version": "==1.3.3" }, "docutils": { @@ -480,10 +488,10 @@ }, "soupsieve": { "hashes": [ - "sha256:e914534802d7ffd233242b785229d5ba0766a7f487385e3f714446a07bf540ae", - "sha256:fcd71e08c0aee99aca1b73f45478549ee7e7fc006d51b37bec9e9def7dc22b69" + "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", + "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232" ], - "version": "==2.0" + "version": "==2.0.1" }, "sphinx": { "hashes": [ -- cgit v1.2.3 From ecf7f24f05b9baa8705f3b2c2d044a42292fbc07 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 17 May 2020 19:50:12 +0200 Subject: Add the REDIS_PASSWORD environment variable In production, we will need this password to make a connection to Redis. --- bot/constants.py | 11 +++++++++-- bot/utils/redis_dict.py | 5 +++-- config-default.yml | 7 +++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/bot/constants.py b/bot/constants.py index 01e8ac3a3..5d854dd7a 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -199,8 +199,15 @@ class Bot(metaclass=YAMLGetter): prefix: str token: str sentry_dsn: str - redis_host: str - redis_port: int + + +class Redis(metaclass=YAMLGetter): + section = "bot" + subsection = "redis" + + host: str + port: int + password: str class Filter(metaclass=YAMLGetter): diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py index 47905314a..4a5e34249 100644 --- a/bot/utils/redis_dict.py +++ b/bot/utils/redis_dict.py @@ -27,8 +27,9 @@ class RedisDict(MutableMapping): _namespaces = [] _redis = redis_py.Redis( - host=constants.Bot.redis_host, - port=constants.Bot.redis_port, + host=constants.Redis.host, + port=constants.Redis.port, + password=constants.Redis.password, ) # Can be overridden for testing def __init__(self, namespace: Optional[str] = None) -> None: diff --git a/config-default.yml b/config-default.yml index 722afa41b..3b58d9099 100644 --- a/config-default.yml +++ b/config-default.yml @@ -2,8 +2,11 @@ bot: prefix: "!" token: !ENV "BOT_TOKEN" sentry_dsn: !ENV "BOT_SENTRY_DSN" - redis_host: "redis" - redis_port: 6379 + + redis: + host: "redis" + port: 6379 + password: !ENV "REDIS_PASSWORD" stats: statsd_host: "graphite" -- cgit v1.2.3 From bd4b439bb7f4abb6b22ad6e0d33bbe9203317475 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Tue, 19 May 2020 01:30:33 +0100 Subject: [bug] Adjustment to changes in #941, return message sent by webhook so publish can take place --- bot/cogs/reddit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index a300cfe0f..3b77538a0 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -218,7 +218,7 @@ class Reddit(Cog): for subreddit in RedditConfig.subreddits: top_posts = await self.get_top_posts(subreddit=subreddit, time="day") - message = await self.webhook.send(username=f"{subreddit} Top Daily Posts", embed=top_posts) + message = await self.webhook.send(username=f"{subreddit} Top Daily Posts", embed=top_posts, wait=True) if message.channel.is_news(): await message.publish() -- cgit v1.2.3 From 1d36ec7001c35d96ff24f7493d26997c02891e93 Mon Sep 17 00:00:00 2001 From: "S. Co1" Date: Mon, 18 May 2020 21:17:53 -0400 Subject: Add Steam gift card scam to domain blacklist --- config-default.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config-default.yml b/config-default.yml index 83ea59016..c0b5b062f 100644 --- a/config-default.yml +++ b/config-default.yml @@ -316,6 +316,8 @@ filter: - poweredbydialup.online - poweredbysecurity.org - poweredbysecurity.online + - ssteam.site + - steamwalletgift.com word_watchlist: - goo+ks* -- cgit v1.2.3 From 75385da574dfdd622ab4b0c7d5771ebd3218542d Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 19 May 2020 07:58:40 +0300 Subject: Stats: Fixed stat names Co-authored-by: Joseph Banks --- bot/cogs/snekbox.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 1d240d8d8..efff6d815 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -207,9 +207,9 @@ class Snekbox(Cog): # Collect stats of eval fails + successes if icon == ":x:": - self.bot.stats.incr("evals.fail") + self.bot.stats.incr("snekbox.python.fail") elif icon in (":warning:", ":white_check_mark:"): - self.bot.stats.incr("evals.success") + self.bot.stats.incr("snekbox.python.success") response = await ctx.send(msg) self.bot.loop.create_task( @@ -299,16 +299,16 @@ class Snekbox(Cog): return if Roles.helpers in (role.id for role in ctx.author.roles): - self.bot.stats.incr("evals.roles.helpers") + self.bot.stats.incr("snekbox_usages.roles.helpers") else: - self.bot.stats.incr("evals.roles.developers") + self.bot.stats.incr("snekbox_usages.roles.developers") if ctx.channel.category_id == Categories.help_in_use: - self.bot.stats.incr("evals.channels.help") + self.bot.stats.incr("snekbox_usages.channels.help") elif ctx.channel.id == Channels.bot_commands: - self.bot.stats.incr("evals.channels.bot_commands") + self.bot.stats.incr("snekbox_usages.channels.bot_commands") else: - self.bot.stats.incr("evals.channels.topical") + self.bot.stats.incr("snekbox_usages.channels.topical") log.info(f"Received code from {ctx.author} for evaluation:\n{code}") -- cgit v1.2.3 From 3081aad0cb360436c451b9a4515d494711adaf81 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Tue, 19 May 2020 07:59:21 +0300 Subject: Stats: Fix docstrings Co-authored-by: Joseph Banks --- bot/cogs/stats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py index acee1f5a9..9baf222e2 100644 --- a/bot/cogs/stats.py +++ b/bot/cogs/stats.py @@ -106,14 +106,14 @@ class Stats(Cog): @loop(hours=1) async def update_guild_boost(self) -> None: - """Update every hour guild boosts amount + level.""" + """Post the server boost level and tier every hour.""" await self.bot.wait_until_guild_available() g = self.bot.get_guild(Guild.id) self.bot.stats.gauge("boost.amount", g.premium_subscription_count) self.bot.stats.gauge("boost.tier", g.premium_tier) def cog_unload(self) -> None: - """Stop guild boost stat collecting task on Cog unload.""" + """Stop the boost statistic task on unload of the Cog.""" self.update_guild_boost.stop() -- cgit v1.2.3 From 21916ad9c19a326eb8406ea751e5fd9f80e9d912 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:42:11 +0300 Subject: ModLog Tests: Fix truncation tests docstring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Leon Sandøy --- tests/bot/cogs/moderation/test_modlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/moderation/test_modlog.py b/tests/bot/cogs/moderation/test_modlog.py index d60836474..b5ad21a09 100644 --- a/tests/bot/cogs/moderation/test_modlog.py +++ b/tests/bot/cogs/moderation/test_modlog.py @@ -15,7 +15,7 @@ class ModLogTests(unittest.IsolatedAsyncioTestCase): self.channel = MockTextChannel() async def test_log_entry_description_truncation(self): - """Should truncate embed description for ModLog entry.""" + """Test that embed description for ModLog entry is truncated.""" self.bot.get_channel.return_value = self.channel await self.cog.send_log_message( icon_url="foo", -- cgit v1.2.3 From 1432e5ba36fc09c7233e5be4745f540c2c4af792 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:47:59 +0300 Subject: Infraction Tests: Small fixes - Remove unnecessary space from placeholder - Rename `has_active_infraction` to `get_active_infraction` --- tests/bot/cogs/moderation/test_infractions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 51a8cc645..2b1ff5728 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -17,11 +17,11 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): self.guild = MockGuild(id=4567) self.ctx = MockContext(bot=self.bot, author=self.user, guild=self.guild) - @patch("bot.cogs.moderation.utils.has_active_infraction") + @patch("bot.cogs.moderation.utils.get_active_infraction") @patch("bot.cogs.moderation.utils.post_infraction") - async def test_apply_ban_reason_truncation(self, post_infraction_mock, has_active_mock): + async def test_apply_ban_reason_truncation(self, post_infraction_mock, get_active_mock): """Should truncate reason for `ctx.guild.ban`.""" - has_active_mock.return_value = False + get_active_mock.return_value = 'foo' post_infraction_mock.return_value = {"foo": "bar"} self.cog.apply_infraction = AsyncMock() @@ -32,7 +32,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): ban = self.cog.apply_infraction.call_args[0][3] self.assertEqual( ban.cr_frame.f_locals["kwargs"]["reason"], - textwrap.shorten("foo bar" * 3000, 512, placeholder=" ...") + textwrap.shorten("foo bar" * 3000, 512, placeholder="...") ) # Await ban to avoid warning await ban -- cgit v1.2.3 From 787c106fb4a55eacc7af04afb9bcd8f206099e81 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:50:46 +0300 Subject: Infractions: Remove space from placeholder --- bot/cogs/moderation/infractions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 01266d346..5bfaad796 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -259,7 +259,7 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_remove, user.id) - truncated_reason = textwrap.shorten(reason, width=512, placeholder=" ...") + truncated_reason = textwrap.shorten(reason, width=512, placeholder="...") action = ctx.guild.ban(user, reason=truncated_reason, delete_message_days=0) await self.apply_infraction(ctx, infraction, user, action) -- cgit v1.2.3 From caac9b92d7c3f73ca5428597606105730e56cefc Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:53:58 +0300 Subject: ModLog: Fix embed description truncation --- bot/cogs/moderation/modlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index c6497b38d..9d28030d9 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -100,7 +100,7 @@ class ModLog(Cog, name="ModLog"): """Generate log embed and send to logging channel.""" # Truncate string directly here to avoid removing newlines embed = discord.Embed( - description=text[:2046] + "..." if len(text) > 2048 else text + description=text[:2045] + "..." if len(text) > 2048 else text ) if title and icon_url: -- cgit v1.2.3 From 5989bcfefa244eb05f37b76d1e1df2f45e5782fa Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:54:49 +0300 Subject: ModLog Tests: Fix embed description truncate test --- tests/bot/cogs/moderation/test_modlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/moderation/test_modlog.py b/tests/bot/cogs/moderation/test_modlog.py index b5ad21a09..f2809f40a 100644 --- a/tests/bot/cogs/moderation/test_modlog.py +++ b/tests/bot/cogs/moderation/test_modlog.py @@ -25,5 +25,5 @@ class ModLogTests(unittest.IsolatedAsyncioTestCase): ) embed = self.channel.send.call_args[1]["embed"] self.assertEqual( - embed.description, ("foo bar" * 3000)[:2046] + "..." + embed.description, ("foo bar" * 3000)[:2045] + "..." ) -- cgit v1.2.3 From 874cb001df91ea8223385dd2b32ab4e3c280e183 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:57:07 +0300 Subject: Infr. Tests: Add more content to await comment --- tests/bot/cogs/moderation/test_infractions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 2b1ff5728..f8f340c2e 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -34,7 +34,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): ban.cr_frame.f_locals["kwargs"]["reason"], textwrap.shorten("foo bar" * 3000, 512, placeholder="...") ) - # Await ban to avoid warning + # Await ban to avoid not awaited coroutine warning await ban @patch("bot.cogs.moderation.utils.post_infraction") @@ -51,5 +51,5 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): kick.cr_frame.f_locals["kwargs"]["reason"], textwrap.shorten("foo bar" * 3000, 512, placeholder="...") ) - # Await kick to avoid warning + # Await kick to avoid not awaited coroutine warning await kick -- cgit v1.2.3 From e9bd09d90c5acf61caa955533f406851e1a65aec Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 10:59:11 +0300 Subject: Infr. Tests: Replace `str` with `dict` To allow `.get`, I had to replace `str` return value with `dict` --- tests/bot/cogs/moderation/test_infractions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index f8f340c2e..139209749 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -21,7 +21,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): @patch("bot.cogs.moderation.utils.post_infraction") async def test_apply_ban_reason_truncation(self, post_infraction_mock, get_active_mock): """Should truncate reason for `ctx.guild.ban`.""" - get_active_mock.return_value = 'foo' + get_active_mock.return_value = {"foo": "bar"} post_infraction_mock.return_value = {"foo": "bar"} self.cog.apply_infraction = AsyncMock() -- cgit v1.2.3 From d9730e41b3144862fdd9c221d160a40144a7c881 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 11:02:37 +0300 Subject: Infr. Test: Replace `get_active_mock` return value Replace `{"foo": "bar"}` with `{"id": 1}` --- tests/bot/cogs/moderation/test_infractions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 139209749..925439bf3 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -21,7 +21,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): @patch("bot.cogs.moderation.utils.post_infraction") async def test_apply_ban_reason_truncation(self, post_infraction_mock, get_active_mock): """Should truncate reason for `ctx.guild.ban`.""" - get_active_mock.return_value = {"foo": "bar"} + get_active_mock.return_value = {"id": 1} post_infraction_mock.return_value = {"foo": "bar"} self.cog.apply_infraction = AsyncMock() -- cgit v1.2.3 From a1b6d147befd4043acdddc00667d3bda94cc76ad Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Wed, 20 May 2020 11:15:09 +0300 Subject: Infr Tests: Make `get_active_infraction` return `None` --- tests/bot/cogs/moderation/test_infractions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 925439bf3..5548d9f68 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -21,7 +21,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): @patch("bot.cogs.moderation.utils.post_infraction") async def test_apply_ban_reason_truncation(self, post_infraction_mock, get_active_mock): """Should truncate reason for `ctx.guild.ban`.""" - get_active_mock.return_value = {"id": 1} + get_active_mock.return_value = None post_infraction_mock.return_value = {"foo": "bar"} self.cog.apply_infraction = AsyncMock() -- cgit v1.2.3 From 6c5979b88961cb1df3d669e07ae108d12e698119 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 21 May 2020 11:47:39 +0300 Subject: Config: Added new `HelpChannels` config `deleted_idle_minutes` This show how much minutes should this wait before making channel dormant when no messages in channel (original message deleted). --- bot/constants.py | 1 + config-default.yml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/bot/constants.py b/bot/constants.py index fd280e9de..3003c9d36 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -541,6 +541,7 @@ class HelpChannels(metaclass=YAMLGetter): claim_minutes: int cmd_whitelist: List[int] idle_minutes: int + deleted_idle_minutes: int max_available: int max_total_channels: int name_prefix: str diff --git a/config-default.yml b/config-default.yml index 83ea59016..2e8a777ba 100644 --- a/config-default.yml +++ b/config-default.yml @@ -529,6 +529,10 @@ help_channels: # Allowed duration of inactivity before making a channel dormant idle_minutes: 30 + # Allowed duration of inactivity when question message deleted + # and no one other sent before message making channel dormant. + deleted_idle_minutes: 5 + # Maximum number of channels to put in the available category max_available: 2 -- cgit v1.2.3 From 0c84302f7e3475c13924fda33c52e98566114082 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Thu, 21 May 2020 12:27:25 +0300 Subject: Help: Implemented faster close when claimant delete msg no more messages - Created function `is_empty` that check is there any message in channel after bot own available message. - `on_message_delete` that reschedule task when message is on correct channel and is empty. - In `move_idle_channel` function, implemented choosing right cooldown, based on is channel empty or not. --- bot/cogs/help_channels.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 1bd1f9d68..4415ce550 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -461,7 +461,11 @@ class HelpChannels(Scheduler, commands.Cog): """ log.trace(f"Handling in-use channel #{channel} ({channel.id}).") - idle_seconds = constants.HelpChannels.idle_minutes * 60 + if not self.is_empty(channel): + idle_seconds = constants.HelpChannels.idle_minutes * 60 + else: + idle_seconds = constants.HelpChannels.deleted_idle_minutes * 60 + time_elapsed = await self.get_idle_time(channel) if time_elapsed is None or time_elapsed >= idle_seconds: @@ -713,6 +717,32 @@ class HelpChannels(Scheduler, commands.Cog): # be put in the queue. await self.move_to_available() + @commands.Cog.listener() + async def on_message_delete(self, msg: discord.Message) -> None: + """Reschedule dormant when help channel is empty.""" + if not self.is_in_category(msg.channel, constants.Categories.help_in_use) or not self.is_empty(msg.channel): + return + + log.info(f"Claimant of #{msg.channel} ({msg.author}) deleted message, channel is empty now. Rescheduling task.") + + # Cancel existing dormant task before scheduling new. + self.cancel_task(msg.channel.id) + + task = TaskData(constants.HelpChannels.deleted_idle_minutes * 60, self.move_idle_channel(msg.channel)) + self.schedule_task(msg.channel.id, task) + + async def is_empty(self, channel: discord.TextChannel) -> bool: + """Check is last message bot sent available message.""" + msg = await self.get_last_message(channel) + if not msg or not msg.author.bot or not msg.embeds: + return False + + embed = msg.embeds[0] + if embed.description == AVAILABLE_MSG: + return True + else: + return False + async def reset_send_permissions(self) -> None: """Reset send permissions in the Available category for claimants.""" log.trace("Resetting send permissions in the Available category.") -- cgit v1.2.3 From cc3591df0f14041be683bb6716d1e427c52aa2d7 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Fri, 22 May 2020 02:34:04 +0200 Subject: Add the REDIS_PASSWORD environment variable In production, we will need this password to make a connection to Redis. --- bot/utils/redis_cache.py | 115 +++++++++++++++++++++++++++++++++++++++ bot/utils/redis_dict.py | 137 ----------------------------------------------- 2 files changed, 115 insertions(+), 137 deletions(-) create mode 100644 bot/utils/redis_cache.py delete mode 100644 bot/utils/redis_dict.py diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py new file mode 100644 index 000000000..d0a7eba4a --- /dev/null +++ b/bot/utils/redis_cache.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +from enum import Enum +from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Union + +from bot.bot import Bot + +ValidRedisKey = Union[str, int, float] +JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] + + +class RedisCache: + """ + A simplified interface for a Redis connection. + + This class must be created as a class attribute in a class. This is because it + uses __set_name__ to create a namespace like MyCog.my_class_attribute which is + used as a hash name when we store stuff in Redis, to prevent collisions. + + The class this object is instantiated in must also contains an attribute with an + instance of Bot. This is because Bot contains our redis_pool, which is how this + class communicates with the Redis server. + + We implement several convenient methods that are fairly similar to have a dict + behaves, and should be familiar to Python users. The biggest difference is that + all the public methods in this class are coroutines. + """ + + _namespaces = [] + + def __init__(self) -> None: + """Raise a NotImplementedError if `__set_name__` hasn't been run.""" + if not self._namespace: + raise NotImplementedError("RedisCache must be a class attribute.") + + def _set_namespace(self, namespace: str) -> None: + """Try to set the namespace, but do not permit collisions.""" + while namespace in self._namespaces: + namespace += "_" + + self._namespaces.append(namespace) + self._namespace = namespace + + def __set_name__(self, owner: object, attribute_name: str) -> None: + """ + Set the namespace to Class.attribute_name. + + Called automatically when this class is constructed inside a class as an attribute. + """ + if not self._has_custom_namespace: + self._set_namespace(f"{owner.__name__}.{attribute_name}") + + def __get__(self, instance: RedisCache, owner: Any) -> RedisCache: + """Fetch the Bot instance, we need it for the redis pool.""" + if self.bot: + return self + + if instance is None: + raise NotImplementedError("You must create an instance of RedisCache to use it.") + + for attribute in vars(instance).values(): + if isinstance(attribute, Bot): + self.bot = attribute + self._redis = self.bot.redis_pool + return self + else: + raise RuntimeError("Cannot initialize a RedisCache without a `Bot` instance.") + + def __repr__(self) -> str: + """Return a beautiful representation of this object instance.""" + return f"RedisCache(namespace={self._namespace!r})" + + async def set(self, key: ValidRedisKey, value: JSONSerializableType) -> None: + """Store an item in the Redis cache.""" + # await self._redis.hset(self._namespace, key, value) + + async def get(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + """Get an item from the Redis cache.""" + # value = await self._redis.hget(self._namespace, key) + + async def delete(self, key: ValidRedisKey) -> None: + """Delete an item from the Redis cache.""" + # await self._redis.hdel(self._namespace, key) + + async def contains(self, key: ValidRedisKey) -> bool: + """Check if a key exists in the Redis cache.""" + # return await self._redis.hexists(self._namespace, key) + + async def items(self) -> AsyncIterator: + """Iterate all the items in the Redis cache.""" + # data = await redis.hgetall(self.get_with_namespace(key)) + # for item in data: + # yield item + + async def length(self) -> int: + """Return the number of items in the Redis cache.""" + # return await self._redis.hlen(self._namespace) + + async def to_dict(self) -> Dict: + """Convert to dict and return.""" + # return dict(self.items()) + + async def clear(self) -> None: + """Deletes the entire hash from the Redis cache.""" + # await self._redis.delete(self._namespace) + + async def pop(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + """Get the item, remove it from the cache, and provide a default if not found.""" + value = await self.get(key, default) + await self.delete(key) + return value + + async def update(self) -> None: + """Update the Redis cache with multiple values.""" + # https://aioredis.readthedocs.io/en/v1.3.0/mixins.html#aioredis.commands.HashCommandsMixin.hmset_dict diff --git a/bot/utils/redis_dict.py b/bot/utils/redis_dict.py deleted file mode 100644 index 4a5e34249..000000000 --- a/bot/utils/redis_dict.py +++ /dev/null @@ -1,137 +0,0 @@ -from __future__ import annotations - -import json -from collections.abc import MutableMapping -from enum import Enum -from typing import Dict, List, Optional, Tuple, Union - -import redis as redis_py - -from bot import constants - -ValidRedisKey = Union[str, int, float] -JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] - - -class RedisDict(MutableMapping): - """ - A dictionary interface for a Redis database. - - Objects created by this class should mostly behave like a normal dictionary, - but will store all the data in our Redis database for persistence between restarts. - - Redis is limited to simple types, so to allow you to store collections like lists - and dictionaries, we JSON deserialize every value. That means that it will not be possible - to store complex objects, only stuff like strings, numbers, and collections of strings and numbers. - """ - - _namespaces = [] - _redis = redis_py.Redis( - host=constants.Redis.host, - port=constants.Redis.port, - password=constants.Redis.password, - ) # Can be overridden for testing - - def __init__(self, namespace: Optional[str] = None) -> None: - """Initialize the RedisDict with the right namespace.""" - super().__init__() - self._has_custom_namespace = namespace is not None - - if self._has_custom_namespace: - self._set_namespace(namespace) - else: - self.namespace = "global" - - def _set_namespace(self, namespace: str) -> None: - """Try to set the namespace, but do not permit collisions.""" - while namespace in self._namespaces: - namespace = namespace + "_" - - self._namespaces.append(namespace) - self._namespace = namespace - - def __set_name__(self, owner: object, attribute_name: str) -> None: - """ - Set the namespace to Class.attribute_name. - - Called automatically when this class is constructed inside a class as an attribute, as long as - no custom namespace is provided to the constructor. - """ - if not self._has_custom_namespace: - self._set_namespace(f"{owner.__name__}.{attribute_name}") - - def __repr__(self) -> str: - """Return a beautiful representation of this object instance.""" - return f"RedisDict(namespace={self._namespace!r})" - - def __eq__(self, other: RedisDict) -> bool: - """Check equality between two RedisDicts.""" - return self.items() == other.items() and self._namespace == other._namespace - - def __ne__(self, other: RedisDict) -> bool: - """Check inequality between two RedisDicts.""" - return self.items() != other.items() or self._namespace != other._namespace - - def __setitem__(self, key: ValidRedisKey, value: JSONSerializableType): - """Store an item in the Redis cache.""" - # JSON serialize the value before storing it. - json_value = json.dumps(value) - self._redis.hset(self._namespace, key, json_value) - - def __getitem__(self, key: ValidRedisKey): - """Get an item from the Redis cache.""" - value = self._redis.hget(self._namespace, key) - - if value: - return json.loads(value) - - def __delitem__(self, key: ValidRedisKey): - """Delete an item from the Redis cache.""" - self._redis.hdel(self._namespace, key) - - def __contains__(self, key: ValidRedisKey): - """Check if a key exists in the Redis cache.""" - return self._redis.hexists(self._namespace, key) - - def __iter__(self): - """Iterate all the items in the Redis cache.""" - keys = self._redis.hkeys(self._namespace) - return iter([key.decode('utf-8') for key in keys]) - - def __len__(self): - """Return the number of items in the Redis cache.""" - return self._redis.hlen(self._namespace) - - def copy(self) -> Dict: - """Convert to dict and return.""" - return dict(self.items()) - - def clear(self) -> None: - """Deletes the entire hash from the Redis cache.""" - self._redis.delete(self._namespace) - - def get(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: - """Get the item, but provide a default if not found.""" - if key in self: - return self[key] - else: - return default - - def pop(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: - """Get the item, remove it from the cache, and provide a default if not found.""" - value = self.get(key, default) - del self[key] - return value - - def popitem(self) -> JSONSerializableType: - """Get the last item added to the cache.""" - key = list(self.keys())[-1] - return self.pop(key) - - def setdefault(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: - """Try to get the item. If the item does not exist, set it to `default` and return that.""" - value = self.get(key) - - if value is None: - self[key] = default - return default -- cgit v1.2.3 From 23c2e7a42c13e03a1765e49f5dac3cfd4fed65b7 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Fri, 22 May 2020 02:34:51 +0200 Subject: Replace redis-py with aioredis. --- Pipfile | 2 +- Pipfile.lock | 162 +++++++++++++++++++++++++++++++++++++---------------------- 2 files changed, 104 insertions(+), 60 deletions(-) diff --git a/Pipfile b/Pipfile index 1d6cd7015..cd2f2ad7a 100644 --- a/Pipfile +++ b/Pipfile @@ -23,7 +23,7 @@ colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} statsd = "~=3.3" feedparser = "~=5.2" beautifulsoup4 = "~=4.9" -redis = "~=3.5" +aioredis = "~=1.3.1" [dev-packages] coverage = "~=5.0" diff --git a/Pipfile.lock b/Pipfile.lock index 25383b355..1941f6887 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "49c231092320b48c5a7618bf048a477f7e0ed33dcbfb71c6dc8f18ef819dd935" + "sha256": "c0b3e4d3e2c9ddb6ba28d2c09d521fe90ad4ea3df5c7ea7cd3a8b679fb3f85f9" }, "pipfile-spec": 6, "requires": { @@ -18,11 +18,11 @@ "default": { "aio-pika": { "hashes": [ - "sha256:9e4614636296e0040055bd6b304e97a38cc9796669ef391fc9b36649831d43ee", - "sha256:c9d242b3c7142d64b185feb6c5cce4154962610e89ec2e9b52bd69ef01f89b2f" + "sha256:c4cbbeb85b3c7bf81bc127371846cd949e6231717ce1e6ac7ee1dd5ede21f866", + "sha256:ec7fef24f588d90314873463ab4f2c3debce0bd8830e49e3786586be96bc2e8e" ], "index": "pypi", - "version": "==6.6.0" + "version": "==6.6.1" }, "aiodns": { "hashes": [ @@ -50,6 +50,14 @@ "index": "pypi", "version": "==3.6.2" }, + "aioredis": { + "hashes": [ + "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a", + "sha256:b61808d7e97b7cd5a92ed574937a079c9387fdadd22bfbfa7ad2fd319ecc26e3" + ], + "index": "pypi", + "version": "==1.3.1" + }, "aiormq": { "hashes": [ "sha256:41a9d4eb17db805f30ed172f3f609fe0c2b16657fb15b1b67df19d251dd93c0d", @@ -87,12 +95,12 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:594ca51a10d2b3443cbac41214e12dbb2a1cd57e1a7344659849e2e20ba6a8d8", - "sha256:a4bbe77fd30670455c5296242967a123ec28c37e9702a8a81bd2f20a4baf0368", - "sha256:d4e96ac9b0c3a6d3f0caae2e4124e6055c5dcafde8e2f831ff194c104f0775a0" + "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7", + "sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8", + "sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c" ], "index": "pypi", - "version": "==4.9.0" + "version": "==4.9.1" }, "certifi": { "hashes": [ @@ -205,6 +213,51 @@ "index": "pypi", "version": "==0.18.0" }, + "hiredis": { + "hashes": [ + "sha256:01b577f84c20ecc9c07fc4c184231b08e3c3942de096fa99978e053de231c423", + "sha256:01ff0900134166961c9e339df77c33b72f7edc5cb41739f0babcd9faa345926e", + "sha256:03ed34a13316d0c34213c4fd46e0fa3a5299073f4d4f08e93fed8c2108b399b3", + "sha256:040436e91df5143aff9e0debb49530d0b17a6bd52200ce568621c31ef581b10d", + "sha256:091eb38fbf968d1c5b703e412bbbd25f43a7967d8400842cee33a5a07b33c27b", + "sha256:102f9b9dc6ed57feb3a7c9bdf7e71cb7c278fe8df1edfcfe896bc3e0c2be9447", + "sha256:2b4b392c7e3082860c8371fab3ae762139090f9115819e12d9f56060f9ede05d", + "sha256:2c9cc0b986397b833073f466e6b9e9c70d1d4dc2c2c1b3e9cae3a23102ff296c", + "sha256:2fa65a9df683bca72073cd77709ddeb289ea2b114d3775d225fbbcc5faf808c5", + "sha256:38437a681f17c975fd22349e72c29bc643f8e7eb2d6dc5df419eac59afa4d7ce", + "sha256:3b3428fa3cf1ee178807b52c9bee8950ab94cd4eaa9bfae8c1bbae3c49501d34", + "sha256:3dd8c2fae7f5494978facb0e93297dd627b1a3f536f3b070cf0a7d9157a07dcb", + "sha256:4414a96c212e732723b5c3d7c04d386ebbb2ec359e1de646322cbc3f875cbd0d", + "sha256:48c627581ad4ef60adbac980981407939acf13a0e18f093502c7b542223c4f19", + "sha256:4a60e71625a2d78d8ab84dfb2fa2cfd9458c964b6e6c04fea76d9ade153fb371", + "sha256:585ace09f434e43d8a8dbeb366865b1a044d7c06319b3c7372a0a00e63b860f4", + "sha256:74b364b3f06c9cf0a53f7df611045bc9437ed972a283fa1f0b12537236d23ddc", + "sha256:75c65c3850e89e9daa68d1b9bedd5806f177d60aa5a7b0953b4829481cfc1f72", + "sha256:7f052de8bf744730a9120dbdc67bfeb7605a01f69fb8e7ba5c475af33c24e145", + "sha256:8113a7d5e87ecf57cd4ae263cc9e429adb9a3e59f5a7768da5d3312a8d0a051a", + "sha256:84857ce239eb8ed191ac78e77ff65d52902f00f30f4ee83bf80eb71da73b70e6", + "sha256:8644a48ddc4a40b3e3a6b9443f396c2ee353afb2d45656c4fc68d04a82e8e3f7", + "sha256:936aa565e673536e8a211e43ec43197406f24cd1f290138bd143765079c8ba00", + "sha256:9afeb88c67bbc663b9f27385c496da056d06ad87f55df6e393e1516cfecb0461", + "sha256:9d62cc7880110e4f83b0a51d218f465d3095e2751fbddd34e553dbd106a929ff", + "sha256:a1fadd062fc8d647ff39220c57ea2b48c99bb73f18223828ec97f88fc27e7898", + "sha256:a7754a783b1e5d6f627c19d099b178059c62f782ab62b4d8ba165b9fbc2ee34c", + "sha256:aa59dd63bb3f736de4fc2d080114429d5d369dfb3265f771778e8349d67a97a4", + "sha256:ae2ee0992f8de249715435942137843a93db204dd7db1e7cc9bdc5a8436443e8", + "sha256:b36842d7cf32929d568f37ec5b3173b72b2ec6572dec4d6be6ce774762215aee", + "sha256:bcbf9379c553b5facc6c04c1e5569b44b38ff16bcbf354676287698d61ee0c92", + "sha256:cbccbda6f1c62ab460449d9c85fdf24d0d32a6bf45176581151e53cc26a5d910", + "sha256:d0caf98dfb8af395d6732bd16561c0a2458851bea522e39f12f04802dbf6f502", + "sha256:d6456afeddba036def1a36d8a2758eca53202308d83db20ab5d0b66590919627", + "sha256:dbaef9a21a4f10bc281684ee4124f169e62bb533c2a92b55f8c06f64f9af7b8f", + "sha256:dce84916c09aaece006272b37234ae84a8ed13abb3a4d341a23933b8701abfb5", + "sha256:eb8c9c8b9869539d58d60ff4a28373a22514d40495911451343971cb4835b7a9", + "sha256:efc98b14ee3a8595e40b1425e8d42f5fd26f11a7b215a81ef9259068931754f4", + "sha256:fa2dc05b87d97acc1c6ae63f3e0f39eae5246565232484b08db6bf2dc1580678", + "sha256:fe7d6ce9f6a5fbe24f09d95ea93e9c7271abc4e1565da511e1449b107b4d7848" + ], + "version": "==1.0.1" + }, "humanfriendly": { "hashes": [ "sha256:bf52ec91244819c780341a3438d5d7b09f431d3f113a475147ac9b7b167a3d12", @@ -235,36 +288,36 @@ }, "lxml": { "hashes": [ - "sha256:06d4e0bbb1d62e38ae6118406d7cdb4693a3fa34ee3762238bcb96c9e36a93cd", - "sha256:0701f7965903a1c3f6f09328c1278ac0eee8f56f244e66af79cb224b7ef3801c", - "sha256:1f2c4ec372bf1c4a2c7e4bb20845e8bcf8050365189d86806bad1e3ae473d081", - "sha256:4235bc124fdcf611d02047d7034164897ade13046bda967768836629bc62784f", - "sha256:5828c7f3e615f3975d48f40d4fe66e8a7b25f16b5e5705ffe1d22e43fb1f6261", - "sha256:585c0869f75577ac7a8ff38d08f7aac9033da2c41c11352ebf86a04652758b7a", - "sha256:5d467ce9c5d35b3bcc7172c06320dddb275fea6ac2037f72f0a4d7472035cea9", - "sha256:63dbc21efd7e822c11d5ddbedbbb08cd11a41e0032e382a0fd59b0b08e405a3a", - "sha256:7bc1b221e7867f2e7ff1933165c0cec7153dce93d0cdba6554b42a8beb687bdb", - "sha256:8620ce80f50d023d414183bf90cc2576c2837b88e00bea3f33ad2630133bbb60", - "sha256:8a0ebda56ebca1a83eb2d1ac266649b80af8dd4b4a3502b2c1e09ac2f88fe128", - "sha256:90ed0e36455a81b25b7034038e40880189169c308a3df360861ad74da7b68c1a", - "sha256:95e67224815ef86924fbc2b71a9dbd1f7262384bca4bc4793645794ac4200717", - "sha256:afdb34b715daf814d1abea0317b6d672476b498472f1e5aacbadc34ebbc26e89", - "sha256:b4b2c63cc7963aedd08a5f5a454c9f67251b1ac9e22fd9d72836206c42dc2a72", - "sha256:d068f55bda3c2c3fcaec24bd083d9e2eede32c583faf084d6e4b9daaea77dde8", - "sha256:d5b3c4b7edd2e770375a01139be11307f04341ec709cf724e0f26ebb1eef12c3", - "sha256:deadf4df349d1dcd7b2853a2c8796593cc346600726eff680ed8ed11812382a7", - "sha256:df533af6f88080419c5a604d0d63b2c33b1c0c4409aba7d0cb6de305147ea8c8", - "sha256:e4aa948eb15018a657702fee0b9db47e908491c64d36b4a90f59a64741516e77", - "sha256:e5d842c73e4ef6ed8c1bd77806bf84a7cb535f9c0cf9b2c74d02ebda310070e1", - "sha256:ebec08091a22c2be870890913bdadd86fcd8e9f0f22bcb398abd3af914690c15", - "sha256:edc15fcfd77395e24543be48871c251f38132bb834d9fdfdad756adb6ea37679", - "sha256:f2b74784ed7e0bc2d02bd53e48ad6ba523c9b36c194260b7a5045071abbb1012", - "sha256:fa071559f14bd1e92077b1b5f6c22cf09756c6de7139370249eb372854ce51e6", - "sha256:fd52e796fee7171c4361d441796b64df1acfceb51f29e545e812f16d023c4bbc", - "sha256:fe976a0f1ef09b3638778024ab9fb8cde3118f203364212c198f71341c0715ca" - ], - "index": "pypi", - "version": "==4.5.0" + "sha256:06748c7192eab0f48e3d35a7adae609a329c6257495d5e53878003660dc0fec6", + "sha256:0790ddca3f825dd914978c94c2545dbea5f56f008b050e835403714babe62a5f", + "sha256:1aa7a6197c1cdd65d974f3e4953764eee3d9c7b67e3966616b41fab7f8f516b7", + "sha256:22c6d34fdb0e65d5f782a4d1a1edb52e0a8365858dafb1c08cb1d16546cf0786", + "sha256:2754d4406438c83144f9ffd3628bbe2dcc6d62b20dbc5c1ec4bc4385e5d44b42", + "sha256:27ee0faf8077c7c1a589573b1450743011117f1aa1a91d5ae776bbc5ca6070f2", + "sha256:2b02c106709466a93ed424454ce4c970791c486d5fcdf52b0d822a7e29789626", + "sha256:2d1ddce96cf15f1254a68dba6935e6e0f1fe39247de631c115e84dd404a6f031", + "sha256:4f282737d187ae723b2633856085c31ae5d4d432968b7f3f478a48a54835f5c4", + "sha256:51bb4edeb36d24ec97eb3e6a6007be128b720114f9a875d6b370317d62ac80b9", + "sha256:7eee37c1b9815e6505847aa5e68f192e8a1b730c5c7ead39ff317fde9ce29448", + "sha256:7fd88cb91a470b383aafad554c3fe1ccf6dfb2456ff0e84b95335d582a799804", + "sha256:9144ce36ca0824b29ebc2e02ca186e54040ebb224292072250467190fb613b96", + "sha256:925baf6ff1ef2c45169f548cc85204433e061360bfa7d01e1be7ae38bef73194", + "sha256:a636346c6c0e1092ffc202d97ec1843a75937d8c98aaf6771348ad6422e44bb0", + "sha256:a87dbee7ad9dce3aaefada2081843caf08a44a8f52e03e0a4cc5819f8398f2f4", + "sha256:a9e3b8011388e7e373565daa5e92f6c9cb844790dc18e43073212bb3e76f7007", + "sha256:afb53edf1046599991fb4a7d03e601ab5f5422a5435c47ee6ba91ec3b61416a6", + "sha256:b26719890c79a1dae7d53acac5f089d66fd8cc68a81f4e4bd355e45470dc25e1", + "sha256:b7462cdab6fffcda853338e1741ce99706cdf880d921b5a769202ea7b94e8528", + "sha256:b77975465234ff49fdad871c08aa747aae06f5e5be62866595057c43f8d2f62c", + "sha256:c47a8a5d00060122ca5908909478abce7bbf62d812e3fc35c6c802df8fb01fe7", + "sha256:c79e5debbe092e3c93ca4aee44c9a7631bdd407b2871cb541b979fd350bbbc29", + "sha256:d8d40e0121ca1606aa9e78c28a3a7d88a05c06b3ca61630242cded87d8ce55fa", + "sha256:ee2be8b8f72a2772e72ab926a3bccebf47bb727bda41ae070dc91d1fb759b726", + "sha256:f95d28193c3863132b1f55c1056036bf580b5a488d908f7d22a04ace8935a3a9", + "sha256:fadd2a63a2bfd7fb604508e553d1cf68eca250b2fbdbd81213b5f6f2fbf23529" + ], + "index": "pypi", + "version": "==4.5.1" }, "markdownify": { "hashes": [ @@ -349,10 +402,10 @@ }, "packaging": { "hashes": [ - "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3", - "sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752" + "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", + "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" ], - "version": "==20.3" + "version": "==20.4" }, "pamqp": { "hashes": [ @@ -448,14 +501,6 @@ "index": "pypi", "version": "==5.3.1" }, - "redis": { - "hashes": [ - "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", - "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" - ], - "index": "pypi", - "version": "==3.5.2" - }, "requests": { "hashes": [ "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", @@ -474,10 +519,10 @@ }, "six": { "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "version": "==1.14.0" + "version": "==1.15.0" }, "snowballstemmer": { "hashes": [ @@ -837,15 +882,14 @@ "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" ], - "index": "pypi", "version": "==3.5.2" }, "six": { "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "version": "==1.14.0" + "version": "==1.15.0" }, "snowballstemmer": { "hashes": [ @@ -878,10 +922,10 @@ }, "virtualenv": { "hashes": [ - "sha256:b4c14d4d73a0c23db267095383c4276ef60e161f94fde0427f2f21a0132dde74", - "sha256:fd0e54dec8ac96c1c7c87daba85f0a59a7c37fe38748e154306ca21c73244637" + "sha256:a116629d4e7f4d03433b8afa27f43deba09d48bc48f5ecefa4f015a178efb6cf", + "sha256:a730548b27366c5e6cbdf6f97406d861cccece2e22275e8e1a757aeff5e00c70" ], - "version": "==20.0.20" + "version": "==20.0.21" } } } -- cgit v1.2.3 From 3f596d5245403b75759a9c73029768d9e4510303 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Fri, 22 May 2020 02:38:55 +0200 Subject: Opens a Redis connection in the Bot class. This global connection is the one we will be using in RedisCache to power all our commands. This also ensures that connection is closed when the bot starts its shutdown process. --- bot/bot.py | 17 +++++++++++++++++ bot/utils/__init__.py | 4 ++-- bot/utils/redis_cache.py | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/bot/bot.py b/bot/bot.py index a85a22aa9..f55eec5bb 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -5,6 +5,7 @@ import warnings from typing import Optional import aiohttp +import aioredis import discord from discord.ext import commands from sentry_sdk import push_scope @@ -28,11 +29,13 @@ class Bot(commands.Bot): super().__init__(*args, **kwargs) self.http_session: Optional[aiohttp.ClientSession] = None + self.redis_session: Optional[aioredis.Redis] = None self.api_client = api.APIClient(loop=self.loop) self._connector = None self._resolver = None self._guild_available = asyncio.Event() + self._redis_ready = asyncio.Event() statsd_url = constants.Stats.statsd_host @@ -42,8 +45,18 @@ class Bot(commands.Bot): # will effectively disable stats. statsd_url = "127.0.0.1" + asyncio.create_task(self._create_redis_session()) + self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") + async def _create_redis_session(self) -> None: + """Create the Redis connection pool, and then open the redis event gate.""" + self.redis_session = await aioredis.create_redis_pool( + address=(constants.Redis.host, constants.Redis.port), + password=constants.Redis.password, + ) + self._redis_ready.set() + def add_cog(self, cog: commands.Cog) -> None: """Adds a "cog" to the bot and logs the operation.""" super().add_cog(cog) @@ -78,6 +91,10 @@ class Bot(commands.Bot): if self.stats._transport: self.stats._transport.close() + if self.redis_session: + self.redis_session.close() + await self.redis_session.wait_closed() + async def login(self, *args, **kwargs) -> None: """Re-create the connector and set up sessions before logging into Discord.""" self._recreate() diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 5ce383bf2..c5a12d5e3 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -2,9 +2,9 @@ from abc import ABCMeta from discord.ext.commands import CogMeta -from bot.utils.redis_dict import RedisDict +from bot.utils.redis_cache import RedisCache -__all__ = ['RedisDict', 'CogABCMeta'] +__all__ = ['RedisCache', 'CogABCMeta'] class CogABCMeta(CogMeta, ABCMeta): diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index d0a7eba4a..467f16767 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -61,7 +61,7 @@ class RedisCache: for attribute in vars(instance).values(): if isinstance(attribute, Bot): self.bot = attribute - self._redis = self.bot.redis_pool + self._redis = self.bot.redis_session return self else: raise RuntimeError("Cannot initialize a RedisCache without a `Bot` instance.") -- cgit v1.2.3 From a46eff8d976cf65155f27ed75f49bd3e58155c84 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 22 May 2020 08:12:08 +0300 Subject: Help: Fix docstrings of `is_empty` and `on_message_delete` Co-authored-by: Mark --- bot/cogs/help_channels.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 4415ce550..ed1f7c55e 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -719,7 +719,11 @@ class HelpChannels(Scheduler, commands.Cog): @commands.Cog.listener() async def on_message_delete(self, msg: discord.Message) -> None: - """Reschedule dormant when help channel is empty.""" + """ + Reschedule an in-use channel to become dormant sooner if the channel is empty. + + The new time for the dormant task is configured with `HelpChannels.deleted_idle_minutes`. + """ if not self.is_in_category(msg.channel, constants.Categories.help_in_use) or not self.is_empty(msg.channel): return @@ -732,7 +736,7 @@ class HelpChannels(Scheduler, commands.Cog): self.schedule_task(msg.channel.id, task) async def is_empty(self, channel: discord.TextChannel) -> bool: - """Check is last message bot sent available message.""" + """Return True if the most recent message in `channel` is the bot's `AVAILABLE_MSG`.""" msg = await self.get_last_message(channel) if not msg or not msg.author.bot or not msg.embeds: return False -- cgit v1.2.3 From 841ce9ba155d2aea3011500f5129d6b3dd309b99 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 22 May 2020 08:29:41 +0300 Subject: Help: Create `embed_description_match` - Created function `embed_description_match`. - Implemented this to `is_empty` - Implemented this to `is_dormant_message` --- bot/cogs/help_channels.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index ed1f7c55e..554fdc55e 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -440,11 +440,18 @@ class HelpChannels(Scheduler, commands.Cog): def is_dormant_message(self, message: t.Optional[discord.Message]) -> bool: """Return True if the contents of the `message` match `DORMANT_MSG`.""" - if not message or not message.embeds: + if not message: + return False + + return self.embed_description_match(message, DORMANT_MSG) + + def embed_description_match(self, message: discord.Message, text: str) -> bool: + """Return `True` if `message` embed description match with `text`.""" + if not message.embeds: return False embed = message.embeds[0] - return message.author == self.bot.user and embed.description.strip() == DORMANT_MSG.strip() + return message.author == self.bot.user and embed.description.strip() == text.strip() @staticmethod def is_in_category(channel: discord.TextChannel, category_id: int) -> bool: @@ -722,7 +729,7 @@ class HelpChannels(Scheduler, commands.Cog): """ Reschedule an in-use channel to become dormant sooner if the channel is empty. - The new time for the dormant task is configured with `HelpChannels.deleted_idle_minutes`. + The new time for the dormant task is configured with `HelpChannels.deleted_idle_minutes`. """ if not self.is_in_category(msg.channel, constants.Categories.help_in_use) or not self.is_empty(msg.channel): return @@ -738,14 +745,10 @@ class HelpChannels(Scheduler, commands.Cog): async def is_empty(self, channel: discord.TextChannel) -> bool: """Return True if the most recent message in `channel` is the bot's `AVAILABLE_MSG`.""" msg = await self.get_last_message(channel) - if not msg or not msg.author.bot or not msg.embeds: + if not msg: return False - embed = msg.embeds[0] - if embed.description == AVAILABLE_MSG: - return True - else: - return False + return self.embed_description_match(msg, AVAILABLE_MSG) async def reset_send_permissions(self) -> None: """Reset send permissions in the Available category for claimants.""" -- cgit v1.2.3 From b3619949a17ba40a3b1f6364cf83464275717283 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Fri, 22 May 2020 08:35:03 +0300 Subject: Eval Stats: Replaced `elif` with `else` on icon check Co-authored-by: Mark --- bot/cogs/snekbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index efff6d815..e2e55e7ca 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -208,7 +208,7 @@ class Snekbox(Cog): # Collect stats of eval fails + successes if icon == ":x:": self.bot.stats.incr("snekbox.python.fail") - elif icon in (":warning:", ":white_check_mark:"): + else: self.bot.stats.incr("snekbox.python.success") response = await ctx.send(msg) -- cgit v1.2.3 From d3550ce1138e1b00e64ba355a09a08b480b077e6 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 11:11:43 -0700 Subject: HelpChannels: fix `is_empty` not being awaited --- bot/cogs/help_channels.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 554fdc55e..2aec22be4 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -468,7 +468,7 @@ class HelpChannels(Scheduler, commands.Cog): """ log.trace(f"Handling in-use channel #{channel} ({channel.id}).") - if not self.is_empty(channel): + if not await self.is_empty(channel): idle_seconds = constants.HelpChannels.idle_minutes * 60 else: idle_seconds = constants.HelpChannels.deleted_idle_minutes * 60 @@ -731,7 +731,10 @@ class HelpChannels(Scheduler, commands.Cog): The new time for the dormant task is configured with `HelpChannels.deleted_idle_minutes`. """ - if not self.is_in_category(msg.channel, constants.Categories.help_in_use) or not self.is_empty(msg.channel): + if not self.is_in_category(msg.channel, constants.Categories.help_in_use): + return + + if not await self.is_empty(msg.channel): return log.info(f"Claimant of #{msg.channel} ({msg.author}) deleted message, channel is empty now. Rescheduling task.") -- cgit v1.2.3 From 8deeeca83c6c2c3de3b856ea8d6f94f8b5db3526 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 11:16:21 -0700 Subject: HelpChannels: rename `embed_description_match` --- bot/cogs/help_channels.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 2aec22be4..b9b577256 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -443,15 +443,15 @@ class HelpChannels(Scheduler, commands.Cog): if not message: return False - return self.embed_description_match(message, DORMANT_MSG) + return self.match_bot_embed(message, DORMANT_MSG) - def embed_description_match(self, message: discord.Message, text: str) -> bool: - """Return `True` if `message` embed description match with `text`.""" + def match_bot_embed(self, message: discord.Message, description: str) -> bool: + """Return `True` if the bot's `message`'s embed description matches `description`.""" if not message.embeds: return False embed = message.embeds[0] - return message.author == self.bot.user and embed.description.strip() == text.strip() + return message.author == self.bot.user and embed.description.strip() == description.strip() @staticmethod def is_in_category(channel: discord.TextChannel, category_id: int) -> bool: @@ -751,7 +751,7 @@ class HelpChannels(Scheduler, commands.Cog): if not msg: return False - return self.embed_description_match(msg, AVAILABLE_MSG) + return self.match_bot_embed(msg, AVAILABLE_MSG) async def reset_send_permissions(self) -> None: """Reset send permissions in the Available category for claimants.""" -- cgit v1.2.3 From e8266b8e1029f31dea3ad6ecbe36b7df56b8acdc Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 11:20:46 -0700 Subject: HelpChannels: move message None check inside `match_bot_embed` It was being done repeatedly outside the function so let's move it in to reduce redundancy. --- bot/cogs/help_channels.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index b9b577256..07acff34d 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -440,14 +440,11 @@ class HelpChannels(Scheduler, commands.Cog): def is_dormant_message(self, message: t.Optional[discord.Message]) -> bool: """Return True if the contents of the `message` match `DORMANT_MSG`.""" - if not message: - return False - return self.match_bot_embed(message, DORMANT_MSG) - def match_bot_embed(self, message: discord.Message, description: str) -> bool: + def match_bot_embed(self, message: t.Optional[discord.Message], description: str) -> bool: """Return `True` if the bot's `message`'s embed description matches `description`.""" - if not message.embeds: + if not message or not message.embeds: return False embed = message.embeds[0] @@ -748,9 +745,6 @@ class HelpChannels(Scheduler, commands.Cog): async def is_empty(self, channel: discord.TextChannel) -> bool: """Return True if the most recent message in `channel` is the bot's `AVAILABLE_MSG`.""" msg = await self.get_last_message(channel) - if not msg: - return False - return self.match_bot_embed(msg, AVAILABLE_MSG) async def reset_send_permissions(self) -> None: -- cgit v1.2.3 From 278ae309be27058920424c4049272bd5171bc158 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 11:22:27 -0700 Subject: HelpChannels: remove `is_dormant_message` At this point, it's just a thin wrapper to call another function. It's redundant. --- bot/cogs/help_channels.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 07acff34d..f0e6746f0 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -438,10 +438,6 @@ class HelpChannels(Scheduler, commands.Cog): """Return True if `member` has the 'Help Cooldown' role.""" return any(constants.Roles.help_cooldown == role.id for role in member.roles) - def is_dormant_message(self, message: t.Optional[discord.Message]) -> bool: - """Return True if the contents of the `message` match `DORMANT_MSG`.""" - return self.match_bot_embed(message, DORMANT_MSG) - def match_bot_embed(self, message: t.Optional[discord.Message], description: str) -> bool: """Return `True` if the bot's `message`'s embed description matches `description`.""" if not message or not message.embeds: @@ -822,7 +818,7 @@ class HelpChannels(Scheduler, commands.Cog): embed = discord.Embed(description=AVAILABLE_MSG) msg = await self.get_last_message(channel) - if self.is_dormant_message(msg): + if self.match_bot_embed(msg, DORMANT_MSG): log.trace(f"Found dormant message {msg.id} in {channel_info}; editing it.") await msg.edit(embed=embed) else: -- cgit v1.2.3 From 57fe4bf893e94289b5b6f7158ff2d6b92b1e3fae Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Fri, 22 May 2020 22:56:25 +0200 Subject: Set up async testbed --- bot/bot.py | 3 +- bot/utils/redis_cache.py | 13 ++- tests/bot/utils/test_redis_cache.py | 128 ++++++++++++++++++++++++ tests/bot/utils/test_redis_dict.py | 189 ------------------------------------ 4 files changed, 135 insertions(+), 198 deletions(-) create mode 100644 tests/bot/utils/test_redis_cache.py delete mode 100644 tests/bot/utils/test_redis_dict.py diff --git a/bot/bot.py b/bot/bot.py index f55eec5bb..8a3805989 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -45,8 +45,7 @@ class Bot(commands.Bot): # will effectively disable stats. statsd_url = "127.0.0.1" - asyncio.create_task(self._create_redis_session()) - + self.loop.create_task(self._create_redis_session()) self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") async def _create_redis_session(self) -> None: diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 467f16767..483bbc2cd 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -30,8 +30,8 @@ class RedisCache: def __init__(self) -> None: """Raise a NotImplementedError if `__set_name__` hasn't been run.""" - if not self._namespace: - raise NotImplementedError("RedisCache must be a class attribute.") + self._namespace = None + self.bot = None def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" @@ -47,8 +47,7 @@ class RedisCache: Called automatically when this class is constructed inside a class as an attribute. """ - if not self._has_custom_namespace: - self._set_namespace(f"{owner.__name__}.{attribute_name}") + self._set_namespace(f"{owner.__name__}.{attribute_name}") def __get__(self, instance: RedisCache, owner: Any) -> RedisCache: """Fetch the Bot instance, we need it for the redis pool.""" @@ -106,9 +105,9 @@ class RedisCache: async def pop(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: """Get the item, remove it from the cache, and provide a default if not found.""" - value = await self.get(key, default) - await self.delete(key) - return value + # value = await self.get(key, default) + # await self.delete(key) + # return value async def update(self) -> None: """Update the Redis cache with multiple values.""" diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py new file mode 100644 index 000000000..f6344803f --- /dev/null +++ b/tests/bot/utils/test_redis_cache.py @@ -0,0 +1,128 @@ +import asyncio +import unittest +from unittest.mock import MagicMock + +import fakeredis.aioredis + +from bot.bot import Bot +from bot.utils import RedisCache + + +class RedisCacheTests(unittest.IsolatedAsyncioTestCase): + """Tests the RedisDict class from utils.redis_dict.py.""" + + redis = RedisCache() + + async def asyncSetUp(self): # noqa: N802 - this special method can't be all lowercase + """Sets up the objects that only have to be initialized once.""" + self.bot = MagicMock( + spec=Bot, + redis_session=await fakeredis.aioredis.create_redis_pool(), + _redis_ready=asyncio.Event(), + ) + self.bot._redis_ready.set() + + def test_class_attribute_namespace(self): + """Test that RedisDict creates a namespace automatically for class attributes.""" + self.assertEqual(self.redis._namespace, "RedisCacheTests.redis") + # Test that errors are raised when this isn't true. + + # def test_set_get_item(self): + # """Test that users can set and get items from the RedisDict.""" + # self.redis['favorite_fruit'] = 'melon' + # self.redis['favorite_number'] = 86 + # self.assertEqual(self.redis['favorite_fruit'], 'melon') + # self.assertEqual(self.redis['favorite_number'], 86) + # + # def test_set_item_types(self): + # """Test that setitem rejects keys and values that are not strings, ints or floats.""" + # fruits = ["lemon", "melon", "apple"] + # + # with self.assertRaises(DataError): + # self.redis[fruits] = "nice" + # + # def test_contains(self): + # """Test that we can reliably use the `in` operator with our RedisDict.""" + # self.redis['favorite_country'] = "Burkina Faso" + # + # self.assertIn('favorite_country', self.redis) + # self.assertNotIn('favorite_dentist', self.redis) + # + # def test_items(self): + # """Test that the RedisDict can be iterated.""" + # self.redis.clear() + # test_cases = ( + # ('favorite_turtle', 'Donatello'), + # ('second_favorite_turtle', 'Leonardo'), + # ('third_favorite_turtle', 'Raphael'), + # ) + # for key, value in test_cases: + # self.redis[key] = value + # + # # Test regular iteration + # for test_case, key in zip(test_cases, self.redis): + # value = test_case[1] + # self.assertEqual(self.redis[key], value) + # + # # Test .items iteration + # for key, value in self.redis.items(): + # self.assertEqual(self.redis[key], value) + # + # # Test .keys iteration + # for test_case, key in zip(test_cases, self.redis.keys()): + # value = test_case[1] + # self.assertEqual(self.redis[key], value) + # + # def test_length(self): + # """Test that we can get the correct len() from the RedisDict.""" + # self.redis.clear() + # self.redis['one'] = 1 + # self.redis['two'] = 2 + # self.redis['three'] = 3 + # self.assertEqual(len(self.redis), 3) + # + # self.redis['four'] = 4 + # self.assertEqual(len(self.redis), 4) + # + # def test_to_dict(self): + # """Test that the .copy method returns a workable dictionary copy.""" + # copy = self.redis.copy() + # local_copy = dict(self.redis.items()) + # self.assertIs(type(copy), dict) + # self.assertEqual(copy, local_copy) + # + # def test_clear(self): + # """Test that the .clear method removes the entire hash.""" + # self.redis.clear() + # self.redis['teddy'] = "with me" + # self.redis['in my dreams'] = "you have a weird hat" + # self.assertEqual(len(self.redis), 2) + # + # self.redis.clear() + # self.assertEqual(len(self.redis), 0) + # + # def test_pop(self): + # """Test that we can .pop an item from the RedisDict.""" + # self.redis.clear() + # self.redis['john'] = 'was afraid' + # + # self.assertEqual(self.redis.pop('john'), 'was afraid') + # self.assertEqual(self.redis.pop('pete', 'breakneck'), 'breakneck') + # self.assertEqual(len(self.redis), 0) + # + # def test_update(self): + # """Test that we can .update the RedisDict with multiple items.""" + # self.redis.clear() + # self.redis["reckfried"] = "lona" + # self.redis["bel air"] = "prince" + # self.redis.update({ + # "reckfried": "jona", + # "mega": "hungry, though", + # }) + # + # result = { + # "reckfried": "jona", + # "bel air": "prince", + # "mega": "hungry, though", + # } + # self.assertEqual(self.redis.copy(), result) diff --git a/tests/bot/utils/test_redis_dict.py b/tests/bot/utils/test_redis_dict.py deleted file mode 100644 index f422887ce..000000000 --- a/tests/bot/utils/test_redis_dict.py +++ /dev/null @@ -1,189 +0,0 @@ -import unittest - -import fakeredis -from redis import DataError - -from bot.utils import RedisDict - -redis_server = fakeredis.FakeServer() -RedisDict._redis = fakeredis.FakeStrictRedis(server=redis_server) - - -class RedisDictTests(unittest.TestCase): - """Tests the RedisDict class from utils.redis_dict.py.""" - - redis = RedisDict() - - def test_class_attribute_namespace(self): - """Test that RedisDict creates a namespace automatically for class attributes.""" - self.assertEqual(self.redis._namespace, "RedisDictTests.redis") - - def test_custom_namespace(self): - """Test that users can set a custom namespaces which never collide.""" - test_cases = ( - (RedisDict("firedog")._namespace, "firedog"), - (RedisDict("firedog")._namespace, "firedog_"), - (RedisDict("firedog")._namespace, "firedog__"), - ) - - for test_case, result in test_cases: - self.assertEqual(test_case, result) - - def test_custom_namespace_takes_precedence(self): - """Test that custom namespaces take precedence over class attribute ones.""" - class LemonJuice: - citrus = RedisDict("citrus") - watercat = RedisDict() - - test_class = LemonJuice() - self.assertEqual(test_class.citrus._namespace, "citrus") - self.assertEqual(test_class.watercat._namespace, "LemonJuice.watercat") - - def test_set_get_item(self): - """Test that users can set and get items from the RedisDict.""" - self.redis['favorite_fruit'] = 'melon' - self.redis['favorite_number'] = 86 - self.assertEqual(self.redis['favorite_fruit'], 'melon') - self.assertEqual(self.redis['favorite_number'], 86) - - def test_set_item_value_types(self): - """Test that setitem rejects values that are not JSON serializable.""" - with self.assertRaises(TypeError): - self.redis['favorite_thing'] = object - self.redis['favorite_stuff'] = RedisDict - - def test_set_item_key_types(self): - """Test that setitem rejects keys that are not strings, ints or floats.""" - fruits = ["lemon", "melon", "apple"] - - with self.assertRaises(DataError): - self.redis[fruits] = "nice" - - def test_get_method(self): - """Test that the .get method works like in a dict.""" - self.redis['favorite_movie'] = 'Code Jam Highlights' - - self.assertEqual(self.redis.get('favorite_movie'), 'Code Jam Highlights') - self.assertEqual(self.redis.get('favorite_youtuber', 'pydis'), 'pydis') - self.assertIsNone(self.redis.get('favorite_dog')) - - def test_membership(self): - """Test that we can reliably use the `in` operator with our RedisDict.""" - self.redis['favorite_country'] = "Burkina Faso" - - self.assertIn('favorite_country', self.redis) - self.assertNotIn('favorite_dentist', self.redis) - - def test_del_item(self): - """Test that users can delete items from the RedisDict.""" - self.redis['favorite_band'] = "Radiohead" - self.assertIn('favorite_band', self.redis) - - del self.redis['favorite_band'] - self.assertNotIn('favorite_band', self.redis) - - def test_iter(self): - """Test that the RedisDict can be iterated.""" - self.redis.clear() - test_cases = ( - ('favorite_turtle', 'Donatello'), - ('second_favorite_turtle', 'Leonardo'), - ('third_favorite_turtle', 'Raphael'), - ) - for key, value in test_cases: - self.redis[key] = value - - # Test regular iteration - for test_case, key in zip(test_cases, self.redis): - value = test_case[1] - self.assertEqual(self.redis[key], value) - - # Test .items iteration - for key, value in self.redis.items(): - self.assertEqual(self.redis[key], value) - - # Test .keys iteration - for test_case, key in zip(test_cases, self.redis.keys()): - value = test_case[1] - self.assertEqual(self.redis[key], value) - - def test_len(self): - """Test that we can get the correct len() from the RedisDict.""" - self.redis.clear() - self.redis['one'] = 1 - self.redis['two'] = 2 - self.redis['three'] = 3 - self.assertEqual(len(self.redis), 3) - - self.redis['four'] = 4 - self.assertEqual(len(self.redis), 4) - - def test_copy(self): - """Test that the .copy method returns a workable dictionary copy.""" - copy = self.redis.copy() - local_copy = dict(self.redis.items()) - self.assertIs(type(copy), dict) - self.assertEqual(copy, local_copy) - - def test_clear(self): - """Test that the .clear method removes the entire hash.""" - self.redis.clear() - self.redis['teddy'] = "with me" - self.redis['in my dreams'] = "you have a weird hat" - self.assertEqual(len(self.redis), 2) - - self.redis.clear() - self.assertEqual(len(self.redis), 0) - - def test_pop(self): - """Test that we can .pop an item from the RedisDict.""" - self.redis.clear() - self.redis['john'] = 'was afraid' - - self.assertEqual(self.redis.pop('john'), 'was afraid') - self.assertEqual(self.redis.pop('pete', 'breakneck'), 'breakneck') - self.assertEqual(len(self.redis), 0) - - def test_popitem(self): - """Test that we can .popitem an item from the RedisDict.""" - self.redis.clear() - self.redis['john'] = 'the revalator' - self.redis['teddy'] = 'big bear' - - self.assertEqual(len(self.redis), 2) - self.assertEqual(self.redis.popitem(), 'big bear') - self.assertEqual(len(self.redis), 1) - - def test_setdefault(self): - """Test that we can .setdefault an item from the RedisDict.""" - self.redis.clear() - self.redis.setdefault('john', 'is yellow and weak') - self.assertEqual(self.redis['john'], 'is yellow and weak') - - with self.assertRaises(TypeError): - self.redis.setdefault('geisha', object) - - def test_update(self): - """Test that we can .update the RedisDict with multiple items.""" - self.redis.clear() - self.redis["reckfried"] = "lona" - self.redis["bel air"] = "prince" - self.redis.update({ - "reckfried": "jona", - "mega": "hungry, though", - }) - - result = { - "reckfried": "jona", - "bel air": "prince", - "mega": "hungry, though", - } - self.assertEqual(self.redis.copy(), result) - - def test_equals(self): - """Test that RedisDicts can be compared with == and !=.""" - new_redis_dict = RedisDict("firedog_the_sequel") - new_new_redis_dict = new_redis_dict - - self.assertEqual(new_redis_dict, new_new_redis_dict) - self.assertNotEqual(new_redis_dict, self.redis) -- cgit v1.2.3 From fd6f3d30b4c67f9a81346bb142d4696948fa2812 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 15:40:50 -0700 Subject: Fix assertion for `create_task` in duck pond tests The assertion wasn't using the assertion method. Furthermore, it was testing a non-existent function `create_loop` rather than `create_task`. --- tests/bot/cogs/test_duck_pond.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/cogs/test_duck_pond.py b/tests/bot/cogs/test_duck_pond.py index 7e6bfc748..a8c0107c6 100644 --- a/tests/bot/cogs/test_duck_pond.py +++ b/tests/bot/cogs/test_duck_pond.py @@ -45,7 +45,7 @@ class DuckPondTests(base.LoggingTestsMixin, unittest.IsolatedAsyncioTestCase): self.assertEqual(cog.bot, bot) self.assertEqual(cog.webhook_id, constants.Webhooks.duck_pond) - bot.loop.create_loop.called_once_with(cog.fetch_webhook()) + bot.loop.create_task.assert_called_once_with(cog.fetch_webhook()) def test_fetch_webhook_succeeds_without_connectivity_issues(self): """The `fetch_webhook` method waits until `READY` event and sets the `webhook` attribute.""" -- cgit v1.2.3 From 45e6f8dba869a367b01d99a596bd3355802d1fbe Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 15:44:04 -0700 Subject: Improve aiohttp context manager mocking in snekbox tests I'm not sure how it even managed to work before. It was calling the `post` coroutine (without specifying a URL) and then changing `__aenter__`. Now, a separate mock is created for the context manager and the `post` simply returns that mocked context manager. --- tests/bot/cogs/test_snekbox.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/bot/cogs/test_snekbox.py b/tests/bot/cogs/test_snekbox.py index 1dec0ccaf..84b273a7d 100644 --- a/tests/bot/cogs/test_snekbox.py +++ b/tests/bot/cogs/test_snekbox.py @@ -21,7 +21,10 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): """Post the eval code to the URLs.snekbox_eval_api endpoint.""" resp = MagicMock() resp.json = AsyncMock(return_value="return") - self.bot.http_session.post().__aenter__.return_value = resp + + context_manager = MagicMock() + context_manager.__aenter__.return_value = resp + self.bot.http_session.post.return_value = context_manager self.assertEqual(await self.cog.post_eval("import random"), "return") self.bot.http_session.post.assert_called_with( @@ -41,7 +44,10 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): key = "MarkDiamond" resp = MagicMock() resp.json = AsyncMock(return_value={"key": key}) - self.bot.http_session.post().__aenter__.return_value = resp + + context_manager = MagicMock() + context_manager.__aenter__.return_value = resp + self.bot.http_session.post.return_value = context_manager self.assertEqual( await self.cog.upload_output("My awesome output"), @@ -57,7 +63,10 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): """Output upload gracefully fallback if the upload fail.""" resp = MagicMock() resp.json = AsyncMock(side_effect=Exception) - self.bot.http_session.post().__aenter__.return_value = resp + + context_manager = MagicMock() + context_manager.__aenter__.return_value = resp + self.bot.http_session.post.return_value = context_manager log = logging.getLogger("bot.cogs.snekbox") with self.assertLogs(logger=log, level='ERROR'): -- cgit v1.2.3 From 6aed2f6b69b79b5a7e5f327819d026e7a63a7dab Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 16:15:23 -0700 Subject: Fix unawaited coro warning when instantiating Bot for MockBot's spec The fix is to mock the loop and pass it to the Bot. It will then set it as `self.loop` rather than trying to get an event loop from asyncio. The `create_task` patch has been moved to this loop mock rather than being done in MockBot to ensure that it applies to anything calling it when instantiating the Bot. --- tests/helpers.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 2b79a6c2a..2efeff7db 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -4,6 +4,7 @@ import collections import itertools import logging import unittest.mock +from asyncio import AbstractEventLoop from typing import Iterable, Optional import discord @@ -264,10 +265,16 @@ class MockAPIClient(CustomMockMixin, unittest.mock.MagicMock): spec_set = APIClient -# Create a Bot instance to get a realistic MagicMock of `discord.ext.commands.Bot` -bot_instance = Bot(command_prefix=unittest.mock.MagicMock()) -bot_instance.http_session = None -bot_instance.api_client = None +def _get_mock_loop() -> unittest.mock.Mock: + """Return a mocked asyncio.AbstractEventLoop.""" + loop = unittest.mock.create_autospec(spec=AbstractEventLoop, spec_set=True) + + # Since calling `create_task` on our MockBot does not actually schedule the coroutine object + # as a task in the asyncio loop, this `side_effect` calls `close()` on the coroutine object + # to prevent "has not been awaited"-warnings. + loop.create_task.side_effect = lambda coroutine: coroutine.close() + + return loop class MockBot(CustomMockMixin, unittest.mock.MagicMock): @@ -277,17 +284,14 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): Instances of this class will follow the specifications of `discord.ext.commands.Bot` instances. For more information, see the `MockGuild` docstring. """ - spec_set = bot_instance + spec_set = Bot(command_prefix=unittest.mock.MagicMock(), loop=_get_mock_loop()) additional_spec_asyncs = ("wait_for",) def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - self.api_client = MockAPIClient() - # Since calling `create_task` on our MockBot does not actually schedule the coroutine object - # as a task in the asyncio loop, this `side_effect` calls `close()` on the coroutine object - # to prevent "has not been awaited"-warnings. - self.loop.create_task.side_effect = lambda coroutine: coroutine.close() + self.loop = _get_mock_loop() + self.api_client = MockAPIClient(loop=self.loop) # Create a TextChannel instance to get a realistic MagicMock of `discord.TextChannel` -- cgit v1.2.3 From 1ad7833d800918efca06e5d6b2fbafdb0d757009 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 16:23:12 -0700 Subject: Properly mock the redis pool in MockBot Because some of the redis pool/connection methods return futures rather than being coroutines, the redis pool had to be mocked using the CustomMockMixin so it could take advantage of `additional_spec_asyncs` to use AsyncMocks for these future-returning methods. --- tests/bot/utils/test_redis_cache.py | 12 +++--------- tests/helpers.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index f6344803f..991225481 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -1,11 +1,9 @@ -import asyncio import unittest -from unittest.mock import MagicMock import fakeredis.aioredis -from bot.bot import Bot from bot.utils import RedisCache +from tests import helpers class RedisCacheTests(unittest.IsolatedAsyncioTestCase): @@ -15,12 +13,8 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): # noqa: N802 - this special method can't be all lowercase """Sets up the objects that only have to be initialized once.""" - self.bot = MagicMock( - spec=Bot, - redis_session=await fakeredis.aioredis.create_redis_pool(), - _redis_ready=asyncio.Event(), - ) - self.bot._redis_ready.set() + self.bot = helpers.MockBot() + self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() def test_class_attribute_namespace(self): """Test that RedisDict creates a namespace automatically for class attributes.""" diff --git a/tests/helpers.py b/tests/helpers.py index 2efeff7db..33d4f787c 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -7,6 +7,7 @@ import unittest.mock from asyncio import AbstractEventLoop from typing import Iterable, Optional +import aioredis.abc import discord from discord.ext.commands import Context @@ -265,6 +266,17 @@ class MockAPIClient(CustomMockMixin, unittest.mock.MagicMock): spec_set = APIClient +class MockRedisPool(CustomMockMixin, unittest.mock.MagicMock): + """ + A MagicMock subclass to mock an aioredis connection pool. + + Instances of this class will follow the specifications of `aioredis.abc.AbcPool` instances. + For more information, see the `MockGuild` docstring. + """ + spec_set = aioredis.abc.AbcPool + additional_spec_asyncs = ("execute", "execute_pubsub") + + def _get_mock_loop() -> unittest.mock.Mock: """Return a mocked asyncio.AbstractEventLoop.""" loop = unittest.mock.create_autospec(spec=AbstractEventLoop, spec_set=True) @@ -293,6 +305,10 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): self.loop = _get_mock_loop() self.api_client = MockAPIClient(loop=self.loop) + # fakeredis can't be used cause it'd require awaiting a coroutine to create the pool, + # which cannot be done here in __init__. + self.redis_session = MockRedisPool() + # Create a TextChannel instance to get a realistic MagicMock of `discord.TextChannel` channel_data = { -- cgit v1.2.3 From d8f1634ab68b2cd480d57c8b9da8834866b5c9cc Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 22 May 2020 16:25:10 -0700 Subject: Use autospecced mocks in MockBot for the stats and aiohttp session This will help catch anything that tries to get/set an attribute/method which doesn't exist. It'll also catch missing/too many parameters being passed to methods. --- tests/helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/helpers.py b/tests/helpers.py index 33d4f787c..d226be3f0 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -9,9 +9,11 @@ from typing import Iterable, Optional import aioredis.abc import discord +from aiohttp import ClientSession from discord.ext.commands import Context from bot.api import APIClient +from bot.async_stats import AsyncStatsClient from bot.bot import Bot @@ -304,6 +306,8 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): self.loop = _get_mock_loop() self.api_client = MockAPIClient(loop=self.loop) + self.http_session = unittest.mock.create_autospec(spec=ClientSession, spec_set=True) + self.stats = unittest.mock.create_autospec(spec=AsyncStatsClient, spec_set=True) # fakeredis can't be used cause it'd require awaiting a coroutine to create the pool, # which cannot be done here in __init__. -- cgit v1.2.3 From eb63fb02a49bf1979afd04a1350304edf00d3a56 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 02:06:27 +0200 Subject: Finish .set and .get, and add tests. The .set and .get will accept ints, floats, and strings. These will be converted into "typestrings", which is basically just a simple format that's been invented for this object. For example, an int looks like `b"i|2423"`. Note how it is still stored as a bytestring (like everything in Redis), but because of this prefix we are able to coerce it into the type we want on the way out of the db. --- bot/utils/redis_cache.py | 72 ++++++++++++++++++++++++++++++------- tests/bot/utils/test_redis_cache.py | 36 +++++++++++++------ tests/helpers.py | 2 +- 3 files changed, 85 insertions(+), 25 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 483bbc2cd..24f2f2e03 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -1,12 +1,10 @@ from __future__ import annotations -from enum import Enum -from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Union +from typing import Any, AsyncIterator, Dict, Optional, Union from bot.bot import Bot -ValidRedisKey = Union[str, int, float] -JSONSerializableType = Optional[Union[str, float, bool, Dict, List, Tuple, Enum]] +ValidRedisType = Union[str, int, float] class RedisCache: @@ -41,7 +39,39 @@ class RedisCache: self._namespaces.append(namespace) self._namespace = namespace - def __set_name__(self, owner: object, attribute_name: str) -> None: + @staticmethod + def _to_typestring(value: ValidRedisType) -> str: + """Turn a valid Redis type into a typestring.""" + if isinstance(value, float): + return f"f|{value}" + elif isinstance(value, int): + return f"i|{value}" + elif isinstance(value, str): + return f"s|{value}" + + @staticmethod + def _from_typestring(value: str) -> ValidRedisType: + """Turn a valid Redis type into a typestring.""" + if value.startswith("f|"): + return float(value[2:]) + if value.startswith("i|"): + return int(value[2:]) + if value.startswith("s|"): + return value[2:] + + async def _validate_cache(self) -> None: + """Validate that the RedisCache is ready to be used.""" + if self.bot is None: + raise RuntimeError("Critical error: RedisCache has no `Bot` instance.") + + if self._namespace is None: + raise RuntimeError( + "Critical error: RedisCache has no namespace. " + "Did you initialize this object as a class attribute?" + ) + await self.bot._redis_ready.wait() + + def __set_name__(self, owner: Any, attribute_name: str) -> None: """ Set the namespace to Class.attribute_name. @@ -54,8 +84,11 @@ class RedisCache: if self.bot: return self + if self._namespace is None: + raise RuntimeError("RedisCache must be a class attribute.") + if instance is None: - raise NotImplementedError("You must create an instance of RedisCache to use it.") + raise RuntimeError("You must create an instance of RedisCache to use it.") for attribute in vars(instance).values(): if isinstance(attribute, Bot): @@ -69,19 +102,32 @@ class RedisCache: """Return a beautiful representation of this object instance.""" return f"RedisCache(namespace={self._namespace!r})" - async def set(self, key: ValidRedisKey, value: JSONSerializableType) -> None: + async def set(self, key: ValidRedisType, value: ValidRedisType) -> None: """Store an item in the Redis cache.""" - # await self._redis.hset(self._namespace, key, value) + await self._validate_cache() + + # Convert to a typestring and then set it + key = self._to_typestring(key) + value = self._to_typestring(value) + await self._redis.hset(self._namespace, key, value) - async def get(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + async def get(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: """Get an item from the Redis cache.""" - # value = await self._redis.hget(self._namespace, key) + await self._validate_cache() + key = self._to_typestring(key) + value = await self._redis.hget(self._namespace, key) + + if value is None: + return default + else: + value = self._from_typestring(value.decode("utf-8")) + return value - async def delete(self, key: ValidRedisKey) -> None: + async def delete(self, key: ValidRedisType) -> None: """Delete an item from the Redis cache.""" # await self._redis.hdel(self._namespace, key) - async def contains(self, key: ValidRedisKey) -> bool: + async def contains(self, key: ValidRedisType) -> bool: """Check if a key exists in the Redis cache.""" # return await self._redis.hexists(self._namespace, key) @@ -103,7 +149,7 @@ class RedisCache: """Deletes the entire hash from the Redis cache.""" # await self._redis.delete(self._namespace) - async def pop(self, key: ValidRedisKey, default: Optional[JSONSerializableType] = None) -> JSONSerializableType: + async def pop(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: """Get the item, remove it from the cache, and provide a default if not found.""" # value = await self.get(key, default) # await self.delete(key) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 991225481..ad38bfde0 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -7,27 +7,41 @@ from tests import helpers class RedisCacheTests(unittest.IsolatedAsyncioTestCase): - """Tests the RedisDict class from utils.redis_dict.py.""" + """Tests the RedisCache class from utils.redis_dict.py.""" redis = RedisCache() - async def asyncSetUp(self): # noqa: N802 - this special method can't be all lowercase + async def asyncSetUp(self): # noqa: N802 """Sets up the objects that only have to be initialized once.""" self.bot = helpers.MockBot() self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() - def test_class_attribute_namespace(self): + async def test_class_attribute_namespace(self): """Test that RedisDict creates a namespace automatically for class attributes.""" self.assertEqual(self.redis._namespace, "RedisCacheTests.redis") - # Test that errors are raised when this isn't true. - # def test_set_get_item(self): - # """Test that users can set and get items from the RedisDict.""" - # self.redis['favorite_fruit'] = 'melon' - # self.redis['favorite_number'] = 86 - # self.assertEqual(self.redis['favorite_fruit'], 'melon') - # self.assertEqual(self.redis['favorite_number'], 86) - # + # Test that errors are raised when not assigned as a class attribute + bad_cache = RedisCache() + + with self.assertRaises(RuntimeError): + await bad_cache.set("test", "me_up_deadman") + + async def test_set_get_item(self): + """Test that users can set and get items from the RedisDict.""" + test_cases = ( + ('favorite_fruit', 'melon'), + ('favorite_number', 86), + ('favorite_fraction', 86.54) + ) + + # Test that we can get and set different types. + for test in test_cases: + await self.redis.set(*test) + self.assertEqual(await self.redis.get(test[0]), test[1]) + + # Test that .get allows a default value + self.assertEqual(await self.redis.get('favorite_nothing', "bearclaw"), "bearclaw") + # def test_set_item_types(self): # """Test that setitem rejects keys and values that are not strings, ints or floats.""" # fruits = ["lemon", "melon", "apple"] diff --git a/tests/helpers.py b/tests/helpers.py index d226be3f0..2b176db79 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -299,7 +299,7 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): For more information, see the `MockGuild` docstring. """ spec_set = Bot(command_prefix=unittest.mock.MagicMock(), loop=_get_mock_loop()) - additional_spec_asyncs = ("wait_for",) + additional_spec_asyncs = ("wait_for", "_redis_ready") def __init__(self, **kwargs) -> None: super().__init__(**kwargs) -- cgit v1.2.3 From c8a9a7713c4394556faadb432d1ed3b7ba5c103a Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 11:35:19 +0200 Subject: Finish asyncifying RedisCache methods - All methods will now do a validation check - Complete interface spec added to class: - .update - .clear - .pop - .to_dict - .length - .contains - .delete - .get - .set --- bot/utils/redis_cache.py | 50 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 24f2f2e03..bd14fc239 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -50,8 +50,10 @@ class RedisCache: return f"s|{value}" @staticmethod - def _from_typestring(value: str) -> ValidRedisType: - """Turn a valid Redis type into a typestring.""" + def _from_typestring(value: Union[bytes, str]) -> ValidRedisType: + """Turn a typestring into a valid Redis type.""" + if isinstance(value, bytes): + value = value.decode('utf-8') if value.startswith("f|"): return float(value[2:]) if value.startswith("i|"): @@ -59,6 +61,14 @@ class RedisCache: if value.startswith("s|"): return value[2:] + def _dict_from_typestring(self, dictionary: Dict) -> Dict: + """Turns all contents of a dict into valid Redis types.""" + return {self._from_typestring(key): self._from_typestring(value) for key, value in dictionary.items()} + + def _dict_to_typestring(self, dictionary: Dict) -> Dict: + """Turns all contents of a dict into typestrings.""" + return {self._to_typestring(key): self._to_typestring(value) for key, value in dictionary.items()} + async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" if self.bot is None: @@ -120,41 +130,49 @@ class RedisCache: if value is None: return default else: - value = self._from_typestring(value.decode("utf-8")) + value = self._from_typestring(value) return value async def delete(self, key: ValidRedisType) -> None: """Delete an item from the Redis cache.""" - # await self._redis.hdel(self._namespace, key) + await self._validate_cache() + key = self._to_typestring(key) + return await self._redis.hdel(self._namespace, key) async def contains(self, key: ValidRedisType) -> bool: """Check if a key exists in the Redis cache.""" - # return await self._redis.hexists(self._namespace, key) + await self._validate_cache() + key = self._to_typestring(key) + return await self._redis.hexists(self._namespace, key) async def items(self) -> AsyncIterator: """Iterate all the items in the Redis cache.""" - # data = await redis.hgetall(self.get_with_namespace(key)) - # for item in data: - # yield item + await self._validate_cache() + data = await self._redis.hgetall(self._namespace) # Get all the keys + for key, value in self._dict_from_typestring(data).items(): + yield key, value async def length(self) -> int: """Return the number of items in the Redis cache.""" - # return await self._redis.hlen(self._namespace) + await self._validate_cache() + return await self._redis.hlen(self._namespace) async def to_dict(self) -> Dict: """Convert to dict and return.""" - # return dict(self.items()) + return {key: value async for key, value in self.items()} async def clear(self) -> None: """Deletes the entire hash from the Redis cache.""" - # await self._redis.delete(self._namespace) + await self._validate_cache() + await self._redis.delete(self._namespace) async def pop(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: """Get the item, remove it from the cache, and provide a default if not found.""" - # value = await self.get(key, default) - # await self.delete(key) - # return value + value = await self.get(key, default) + await self.delete(key) + return value - async def update(self) -> None: + async def update(self, items: Dict) -> None: """Update the Redis cache with multiple values.""" - # https://aioredis.readthedocs.io/en/v1.3.0/mixins.html#aioredis.commands.HashCommandsMixin.hmset_dict + await self._validate_cache() + await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) -- cgit v1.2.3 From 387bf5c6b6a21e25c4fc690fb992b6b3e4c165a6 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 11:36:12 +0200 Subject: Complete asyncified test suite for RedisCache This commit just alters existing code to work with the new interface, and with async. All tests are passing successfully. --- tests/bot/utils/test_redis_cache.py | 206 ++++++++++++++++++++---------------- 1 file changed, 112 insertions(+), 94 deletions(-) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index ad38bfde0..d257e91d9 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -16,16 +16,24 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): self.bot = helpers.MockBot() self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() - async def test_class_attribute_namespace(self): + def test_class_attribute_namespace(self): """Test that RedisDict creates a namespace automatically for class attributes.""" self.assertEqual(self.redis._namespace, "RedisCacheTests.redis") - # Test that errors are raised when not assigned as a class attribute + async def test_class_attribute_required(self): + """Test that errors are raised when not assigned as a class attribute.""" bad_cache = RedisCache() + self.assertIs(bad_cache._namespace, None) with self.assertRaises(RuntimeError): await bad_cache.set("test", "me_up_deadman") + def test_namespace_collision(self): + """Test that we prevent colliding namespaces.""" + bad_cache = RedisCache() + bad_cache._set_namespace("RedisCacheTests.redis") + self.assertEqual(bad_cache._namespace, "RedisCacheTests.redis_") + async def test_set_get_item(self): """Test that users can set and get items from the RedisDict.""" test_cases = ( @@ -42,95 +50,105 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): # Test that .get allows a default value self.assertEqual(await self.redis.get('favorite_nothing', "bearclaw"), "bearclaw") - # def test_set_item_types(self): - # """Test that setitem rejects keys and values that are not strings, ints or floats.""" - # fruits = ["lemon", "melon", "apple"] - # - # with self.assertRaises(DataError): - # self.redis[fruits] = "nice" - # - # def test_contains(self): - # """Test that we can reliably use the `in` operator with our RedisDict.""" - # self.redis['favorite_country'] = "Burkina Faso" - # - # self.assertIn('favorite_country', self.redis) - # self.assertNotIn('favorite_dentist', self.redis) - # - # def test_items(self): - # """Test that the RedisDict can be iterated.""" - # self.redis.clear() - # test_cases = ( - # ('favorite_turtle', 'Donatello'), - # ('second_favorite_turtle', 'Leonardo'), - # ('third_favorite_turtle', 'Raphael'), - # ) - # for key, value in test_cases: - # self.redis[key] = value - # - # # Test regular iteration - # for test_case, key in zip(test_cases, self.redis): - # value = test_case[1] - # self.assertEqual(self.redis[key], value) - # - # # Test .items iteration - # for key, value in self.redis.items(): - # self.assertEqual(self.redis[key], value) - # - # # Test .keys iteration - # for test_case, key in zip(test_cases, self.redis.keys()): - # value = test_case[1] - # self.assertEqual(self.redis[key], value) - # - # def test_length(self): - # """Test that we can get the correct len() from the RedisDict.""" - # self.redis.clear() - # self.redis['one'] = 1 - # self.redis['two'] = 2 - # self.redis['three'] = 3 - # self.assertEqual(len(self.redis), 3) - # - # self.redis['four'] = 4 - # self.assertEqual(len(self.redis), 4) - # - # def test_to_dict(self): - # """Test that the .copy method returns a workable dictionary copy.""" - # copy = self.redis.copy() - # local_copy = dict(self.redis.items()) - # self.assertIs(type(copy), dict) - # self.assertEqual(copy, local_copy) - # - # def test_clear(self): - # """Test that the .clear method removes the entire hash.""" - # self.redis.clear() - # self.redis['teddy'] = "with me" - # self.redis['in my dreams'] = "you have a weird hat" - # self.assertEqual(len(self.redis), 2) - # - # self.redis.clear() - # self.assertEqual(len(self.redis), 0) - # - # def test_pop(self): - # """Test that we can .pop an item from the RedisDict.""" - # self.redis.clear() - # self.redis['john'] = 'was afraid' - # - # self.assertEqual(self.redis.pop('john'), 'was afraid') - # self.assertEqual(self.redis.pop('pete', 'breakneck'), 'breakneck') - # self.assertEqual(len(self.redis), 0) - # - # def test_update(self): - # """Test that we can .update the RedisDict with multiple items.""" - # self.redis.clear() - # self.redis["reckfried"] = "lona" - # self.redis["bel air"] = "prince" - # self.redis.update({ - # "reckfried": "jona", - # "mega": "hungry, though", - # }) - # - # result = { - # "reckfried": "jona", - # "bel air": "prince", - # "mega": "hungry, though", - # } - # self.assertEqual(self.redis.copy(), result) + async def test_set_item_type(self): + """Test that .set rejects keys and values that are not strings, ints or floats.""" + fruits = ["lemon", "melon", "apple"] + + with self.assertRaises(TypeError): + await self.redis.set(fruits, "nice") + + async def test_delete_item(self): + """Test that .delete allows us to delete stuff from the RedisCache.""" + # Add an item and verify that it gets added + await self.redis.set("internet", "firetruck") + self.assertEqual(await self.redis.get("internet"), "firetruck") + + # Delete that item and verify that it gets deleted + await self.redis.delete("internet") + self.assertIs(await self.redis.get("internet"), None) + + async def test_contains(self): + """Test that we can check membership with .contains.""" + await self.redis.set('favorite_country', "Burkina Faso") + + self.assertIs(await self.redis.contains('favorite_country'), True) + self.assertIs(await self.redis.contains('favorite_dentist'), False) + + async def test_items(self): + """Test that the RedisDict can be iterated.""" + await self.redis.clear() + + # Set up our test cases in the Redis cache + test_cases = [ + ('favorite_turtle', 'Donatello'), + ('second_favorite_turtle', 'Leonardo'), + ('third_favorite_turtle', 'Raphael'), + ] + for key, value in test_cases: + await self.redis.set(key, value) + + # Consume the AsyncIterator into a regular list, easier to compare that way. + redis_items = [item async for item in self.redis.items()] + + # These sequences are probably in the same order now, but probably + # isn't good enough for tests. Let's not rely on .hgetall always + # returning things in sequence, and just sort both lists to be safe. + redis_items = sorted(redis_items) + test_cases = sorted(test_cases) + + # If these are equal now, everything works fine. + self.assertSequenceEqual(test_cases, redis_items) + + async def test_length(self): + """Test that we can get the correct .length from the RedisDict.""" + await self.redis.clear() + await self.redis.set('one', 1) + await self.redis.set('two', 2) + await self.redis.set('three', 3) + self.assertEqual(await self.redis.length(), 3) + + await self.redis.set('four', 4) + self.assertEqual(await self.redis.length(), 4) + + async def test_to_dict(self): + """Test that the .copy method returns a workable dictionary copy.""" + copy = await self.redis.to_dict() + local_copy = {key: value async for key, value in self.redis.items()} + self.assertIs(type(copy), dict) + self.assertDictEqual(copy, local_copy) + + async def test_clear(self): + """Test that the .clear method removes the entire hash.""" + await self.redis.clear() + await self.redis.set('teddy', 'with me') + await self.redis.set('in my dreams', 'you have a weird hat') + self.assertEqual(await self.redis.length(), 2) + + await self.redis.clear() + self.assertEqual(await self.redis.length(), 0) + + async def test_pop(self): + """Test that we can .pop an item from the RedisDict.""" + await self.redis.clear() + await self.redis.set('john', 'was afraid') + + self.assertEqual(await self.redis.pop('john'), 'was afraid') + self.assertEqual(await self.redis.pop('pete', 'breakneck'), 'breakneck') + self.assertEqual(await self.redis.length(), 0) + + async def test_update(self): + """Test that we can .update the RedisDict with multiple items.""" + await self.redis.clear() + await self.redis.set("reckfried", "lona") + await self.redis.set("bel air", "prince") + await self.redis.update({ + "reckfried": "jona", + "mega": "hungry, though", + }) + + result = { + "reckfried": "jona", + "bel air": "prince", + "mega": "hungry, though", + } + self.assertDictEqual(await self.redis.to_dict(), result) -- cgit v1.2.3 From 5bd8e13088822cfa4b189a30a7d745de61984dc7 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 11:45:01 +0200 Subject: Better docstring for RedisCache --- bot/utils/redis_cache.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index bd14fc239..26a100ef0 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -21,7 +21,37 @@ class RedisCache: We implement several convenient methods that are fairly similar to have a dict behaves, and should be familiar to Python users. The biggest difference is that - all the public methods in this class are coroutines. + all the public methods in this class are coroutines, and must be awaited. + + Because of limitations in Redis, this cache will only accept strings, integers and + floats both for keys and values. + + Simple example for how to use this: + + class SomeCog(Cog): + # To initialize a valid RedisCache, just add it as a class attribute here. + # Do not add it to the __init__ method or anywhere else, it MUST be a class + # attribute. Do not pass any parameters. + cache = RedisCache() + + async def my_method(self): + # Now we can store some stuff in the cache just by doing this. + # This data will persist through restarts! + await self.cache.set("key", "value") + + # To get the data, simply do this. + value = await self.cache.get("key") + + # Other methods work more or less like a dictionary. + # Checking if something is in the cache + await self.cache.contains("key") + + # iterating the cache + async for key, value in self.cache.items(): + print(value) + + # We can even iterate in a comprehension! + consumed = [value async for key, value in self.cache.items()] """ _namespaces = [] -- cgit v1.2.3 From db75440a273277111e7140b1c226630b865d154b Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 11:55:10 +0200 Subject: Better docstring for RedisCache.contains --- bot/utils/redis_cache.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 26a100ef0..2b60ae0c3 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -170,7 +170,11 @@ class RedisCache: return await self._redis.hdel(self._namespace, key) async def contains(self, key: ValidRedisType) -> bool: - """Check if a key exists in the Redis cache.""" + """ + Check if a key exists in the Redis cache. + + Return True if the key exists, otherwise False. + """ await self._validate_cache() key = self._to_typestring(key) return await self._redis.hexists(self._namespace, key) -- cgit v1.2.3 From fc1999ea80df2ebc904260ff0e6f56d9b36bc6c5 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 12:17:39 +0200 Subject: Unbreak the error_handler --- bot/cogs/error_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index e635bd46f..77d16c051 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -167,7 +167,7 @@ class ErrorHandler(Cog): self.bot.stats.incr("errors.missing_required_argument") elif isinstance(e, errors.TooManyArguments): await ctx.send("Too many arguments provided.") - await ctx.invoke(*help_command) + await prepared_help_command self.bot.stats.incr("errors.too_many_arguments") elif isinstance(e, errors.BadArgument): await ctx.send(f"Bad argument: {e}\n") -- cgit v1.2.3 From 49492ffd5e5c87d347048dc370085be12215ed7f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 14:12:13 +0200 Subject: Moving the Redis session creation to Bot._recreate --- bot/bot.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bot/bot.py b/bot/bot.py index 8a3805989..224f5f4e4 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -45,7 +45,6 @@ class Bot(commands.Bot): # will effectively disable stats. statsd_url = "127.0.0.1" - self.loop.create_task(self._create_redis_session()) self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") async def _create_redis_session(self) -> None: @@ -91,6 +90,7 @@ class Bot(commands.Bot): self.stats._transport.close() if self.redis_session: + self._redis_ready.clear() self.redis_session.close() await self.redis_session.wait_closed() @@ -101,7 +101,7 @@ class Bot(commands.Bot): await super().login(*args, **kwargs) def _recreate(self) -> None: - """Re-create the connector, aiohttp session, and the APIClient.""" + """Re-create the connector, aiohttp session, the APIClient and the Redis session.""" # Use asyncio for DNS resolution instead of threads so threads aren't spammed. # Doesn't seem to have any state with regards to being closed, so no need to worry? self._resolver = aiohttp.AsyncResolver() @@ -112,6 +112,9 @@ class Bot(commands.Bot): "The previous connector was not closed; it will remain open and be overwritten" ) + # Create the redis session + self.loop.create_task(self._create_redis_session()) + # Use AF_INET as its socket family to prevent HTTPS related problems both locally # and in production. self._connector = aiohttp.TCPConnector( -- cgit v1.2.3 From 489f9405a77eb88baa0d77a88fd04bf13cbbde1f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 14:21:39 +0200 Subject: CI needs REDIS_PASSWORD to pass tests --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d56675029..4500cb6e8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,6 +22,7 @@ jobs: REDDIT_CLIENT_ID: spam REDDIT_SECRET: ham WOLFRAM_API_KEY: baz + REDIS_PASSWORD: '' steps: - task: UsePythonVersion@0 -- cgit v1.2.3 From 1ea471865fa68a001e25980d50e71333752add6d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 14:27:11 +0200 Subject: Update exception message This was incorrectly suggesting the user needed to create an instance of RedisCache, when in fact it is the parent that needs to be instantiated. Co-authored-by: Sebastiaan Zeeff <33516116+SebastiaanZ@users.noreply.github.com> --- bot/utils/redis_cache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 2b60ae0c3..f9d9e571f 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -128,7 +128,10 @@ class RedisCache: raise RuntimeError("RedisCache must be a class attribute.") if instance is None: - raise RuntimeError("You must create an instance of RedisCache to use it.") + raise RuntimeError( + "You must access the RedisCache instance through the cog instance " + "before accessing it using the cog's class object." + ) for attribute in vars(instance).values(): if isinstance(attribute, Bot): -- cgit v1.2.3 From aa0bb028ed889d93376981213673053a540e137c Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 14:30:56 +0200 Subject: Fix typo in test_to_dict docstring --- tests/bot/utils/test_redis_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index d257e91d9..2ce57499a 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -111,7 +111,7 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(await self.redis.length(), 4) async def test_to_dict(self): - """Test that the .copy method returns a workable dictionary copy.""" + """Test that the .to_dict method returns a workable dictionary copy.""" copy = await self.redis.to_dict() local_copy = {key: value async for key, value in self.redis.items()} self.assertIs(type(copy), dict) -- cgit v1.2.3 From 2fb86258471626863c2214cabc2529e78c77729a Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 14:41:16 +0200 Subject: Don't rely on HDEL ignoring keys for .pop Previously we would try to .delete keys that did not exist if a default was provided when calling .pop. This is okay to do (because HDEL will just ignore any attempts to delete non-existing calls), but it does add an additional pointless API call to Redis, so I've added some validation as a small optimization. This also adds a few additional lines of documentation as requested by @SebastiaanZ in their review. --- bot/utils/redis_cache.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index f9d9e571f..6831be157 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -35,6 +35,14 @@ class RedisCache: cache = RedisCache() async def my_method(self): + + # Now we're ready to use the RedisCache. + # One thing to note here is that this will not work unless + # we access self.cache through an _instance_ of this class. + # + # For example, attempting to use SomeCog.cache will _not_ work, + # you _must_ instantiate the class first and use that instance. + # # Now we can store some stuff in the cache just by doing this. # This data will persist through restarts! await self.cache.set("key", "value") @@ -167,7 +175,13 @@ class RedisCache: return value async def delete(self, key: ValidRedisType) -> None: - """Delete an item from the Redis cache.""" + """ + Delete an item from the Redis cache. + + If we try to delete a key that does not exist, it will simply be ignored. + + See https://redis.io/commands/hdel for more info on how this works. + """ await self._validate_cache() key = self._to_typestring(key) return await self._redis.hdel(self._namespace, key) @@ -206,7 +220,12 @@ class RedisCache: async def pop(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: """Get the item, remove it from the cache, and provide a default if not found.""" value = await self.get(key, default) - await self.delete(key) + + # No need to try to delete something that doesn't exist, + # that's just a superfluous API call. + if value != default: + await self.delete(key) + return value async def update(self, items: Dict) -> None: -- cgit v1.2.3 From 5120717a47c07812d1631cf0905ff3062e139487 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 23 May 2020 15:17:57 +0200 Subject: DRY approach to typestring prefix resolution Thanks to @kwzrd for this idea, basically we're making a constant with the typestring prefixes and iterating that in all our converters. These converter functions will also now raise TypeErrors if we try to convert something that isn't in this constants list. I've also added a new test that tests this functionality. --- bot/utils/redis_cache.py | 54 ++++++++++++++++++++++++++----------- tests/bot/utils/test_redis_cache.py | 21 +++++++++++++++ 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 6831be157..1ec1b9fea 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -4,6 +4,11 @@ from typing import Any, AsyncIterator, Dict, Optional, Union from bot.bot import Bot +TYPESTRING_PREFIXES = ( + ("f|", float), + ("i|", int), + ("s|", str), +) ValidRedisType = Union[str, int, float] @@ -78,26 +83,45 @@ class RedisCache: self._namespace = namespace @staticmethod - def _to_typestring(value: ValidRedisType) -> str: - """Turn a valid Redis type into a typestring.""" - if isinstance(value, float): - return f"f|{value}" - elif isinstance(value, int): - return f"i|{value}" - elif isinstance(value, str): - return f"s|{value}" + def _valid_typestring_types() -> str: + """ + Creates a nice, readable list of valid types for typestrings, useful for error messages. + + This will be dynamically updated if we change the TYPESTRING_PREFIXES constant up top. + """ + valid_types = ", ".join([str(_type).split("'")[1] for _, _type in TYPESTRING_PREFIXES]) + valid_types = ", and ".join(valid_types.rsplit(", ", 1)) + return valid_types @staticmethod - def _from_typestring(value: Union[bytes, str]) -> ValidRedisType: + def _valid_typestring_prefixes() -> str: + """ + Creates a nice, readable list of valid prefixes for typestrings, useful for error messages. + + This will be dynamically updated if we change the TYPESTRING_PREFIXES constant up top. + """ + valid_prefixes = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_PREFIXES]) + valid_prefixes = ", and ".join(valid_prefixes.rsplit(", ", 1)) + return valid_prefixes + + def _to_typestring(self, value: ValidRedisType) -> str: + """Turn a valid Redis type into a typestring.""" + for prefix, _type in TYPESTRING_PREFIXES: + if isinstance(value, _type): + return f"{prefix}{value}" + raise TypeError(f"RedisCache._from_typestring only supports the types {self._valid_typestring_types()}.") + + def _from_typestring(self, value: Union[bytes, str]) -> ValidRedisType: """Turn a typestring into a valid Redis type.""" + # Stuff that comes out of Redis will be bytestrings, so let's decode those. if isinstance(value, bytes): value = value.decode('utf-8') - if value.startswith("f|"): - return float(value[2:]) - if value.startswith("i|"): - return int(value[2:]) - if value.startswith("s|"): - return value[2:] + + # Now we convert our unicode string back into the type it originally was. + for prefix, _type in TYPESTRING_PREFIXES: + if value.startswith(prefix): + return _type(value[2:]) + raise TypeError(f"RedisCache._to_typestring only supports the prefixes {self._valid_typestring_prefixes()}.") def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 2ce57499a..150195726 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -152,3 +152,24 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): "mega": "hungry, though", } self.assertDictEqual(await self.redis.to_dict(), result) + + def test_typestring_conversion(self): + """Test the typestring-related helper functions.""" + conversion_tests = ( + (12, "i|12"), + (12.4, "f|12.4"), + ("cowabunga", "s|cowabunga"), + ) + + # Test conversion to typestring + for _input, expected in conversion_tests: + self.assertEqual(self.redis._to_typestring(_input), expected) + + # Test conversion from typestrings + for _input, expected in conversion_tests: + self.assertEqual(self.redis._from_typestring(expected), _input) + + # Test that exceptions are raised on invalid input + with self.assertRaises(TypeError): + self.redis._to_typestring(["internet"]) + self.redis._from_typestring("o|firedog") -- cgit v1.2.3 From a52a13020f3468c671cb549052a9c8e303ae9d8c Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 23 May 2020 10:27:32 -0700 Subject: Remove redis session mock from MockBot It's not feasible to mock it because all the commands return futures rather than being coroutines, so they cannot automatically be turned into AsyncMocks. Furthermore, no code should ever use the redis session directly besides RedisCache. Since the tests for RedisCache already use fakeredis, there's no use in trying to mock redis in MockBot. --- tests/helpers.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 2b176db79..5ad826156 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -7,7 +7,6 @@ import unittest.mock from asyncio import AbstractEventLoop from typing import Iterable, Optional -import aioredis.abc import discord from aiohttp import ClientSession from discord.ext.commands import Context @@ -268,17 +267,6 @@ class MockAPIClient(CustomMockMixin, unittest.mock.MagicMock): spec_set = APIClient -class MockRedisPool(CustomMockMixin, unittest.mock.MagicMock): - """ - A MagicMock subclass to mock an aioredis connection pool. - - Instances of this class will follow the specifications of `aioredis.abc.AbcPool` instances. - For more information, see the `MockGuild` docstring. - """ - spec_set = aioredis.abc.AbcPool - additional_spec_asyncs = ("execute", "execute_pubsub") - - def _get_mock_loop() -> unittest.mock.Mock: """Return a mocked asyncio.AbstractEventLoop.""" loop = unittest.mock.create_autospec(spec=AbstractEventLoop, spec_set=True) @@ -309,10 +297,6 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): self.http_session = unittest.mock.create_autospec(spec=ClientSession, spec_set=True) self.stats = unittest.mock.create_autospec(spec=AsyncStatsClient, spec_set=True) - # fakeredis can't be used cause it'd require awaiting a coroutine to create the pool, - # which cannot be done here in __init__. - self.redis_session = MockRedisPool() - # Create a TextChannel instance to get a realistic MagicMock of `discord.TextChannel` channel_data = { -- cgit v1.2.3 From 923d03a8040251ae7766b9655a3d0ff9f8413c8b Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 23 May 2020 11:14:13 -0700 Subject: Show a warning when redis pool isn't closed --- bot/bot.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bot/bot.py b/bot/bot.py index 224f5f4e4..bf7f9c9df 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -112,6 +112,11 @@ class Bot(commands.Bot): "The previous connector was not closed; it will remain open and be overwritten" ) + if self.redis_session and not self.redis_session.closed: + log.warning( + "The previous redis pool was not closed; it will remain open and be overwritten" + ) + # Create the redis session self.loop.create_task(self._create_redis_session()) -- cgit v1.2.3 From 3f8dce7502e3afb2d119979cfc455efcde7ad9db Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 10:18:39 +0200 Subject: use __name__ for type list Instead of relying on __str__ representation, we'll use the __name__ dunder. Co-authored-by: Mark --- bot/utils/redis_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 1ec1b9fea..fadbca673 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -89,7 +89,7 @@ class RedisCache: This will be dynamically updated if we change the TYPESTRING_PREFIXES constant up top. """ - valid_types = ", ".join([str(_type).split("'")[1] for _, _type in TYPESTRING_PREFIXES]) + valid_types = ", ".join(str(_type.__name__) for _, _type in TYPESTRING_PREFIXES) valid_types = ", and ".join(valid_types.rsplit(", ", 1)) return valid_types -- cgit v1.2.3 From ed12a2fa303b7faebeb773dac096bd2b0b8ec23d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 10:25:17 +0200 Subject: len(prefix) instead of hardcoding 2 Co-authored-by: Mark --- bot/utils/redis_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index fadbca673..13e88e8e1 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -120,7 +120,7 @@ class RedisCache: # Now we convert our unicode string back into the type it originally was. for prefix, _type in TYPESTRING_PREFIXES: if value.startswith(prefix): - return _type(value[2:]) + return _type(value[len(prefix):]) raise TypeError(f"RedisCache._to_typestring only supports the prefixes {self._valid_typestring_prefixes()}.") def _dict_from_typestring(self, dictionary: Dict) -> Dict: -- cgit v1.2.3 From 98d8bb3e841ba52fe036b36492029a9fdeb36518 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 10:53:46 +0200 Subject: Refactor the nice prefix/type strings to constants It's leaner to just move that code out of the class and up to the module level as constants. This commit also renames ValidRedisType to RedisType. --- bot/utils/redis_cache.py | 50 ++++++++++++++++++------------------------------ 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 13e88e8e1..d5563c079 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -4,12 +4,20 @@ from typing import Any, AsyncIterator, Dict, Optional, Union from bot.bot import Bot +RedisType = Union[str, int, float] TYPESTRING_PREFIXES = ( ("f|", float), ("i|", int), ("s|", str), ) -ValidRedisType = Union[str, int, float] + +# Makes a nice list like "float, int, and str" +NICE_TYPE_LIST = ", ".join(str(_type.__name__) for _, _type in TYPESTRING_PREFIXES) +NICE_TYPE_LIST = ", and ".join(NICE_TYPE_LIST.rsplit(", ", 1)) + +# Makes a list like "'f|', 'i|', and 's|'" +NICE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_PREFIXES]) +NICE_PREFIX_LIST = ", and ".join(NICE_PREFIX_LIST.rsplit(", ", 1)) class RedisCache: @@ -83,35 +91,15 @@ class RedisCache: self._namespace = namespace @staticmethod - def _valid_typestring_types() -> str: - """ - Creates a nice, readable list of valid types for typestrings, useful for error messages. - - This will be dynamically updated if we change the TYPESTRING_PREFIXES constant up top. - """ - valid_types = ", ".join(str(_type.__name__) for _, _type in TYPESTRING_PREFIXES) - valid_types = ", and ".join(valid_types.rsplit(", ", 1)) - return valid_types - - @staticmethod - def _valid_typestring_prefixes() -> str: - """ - Creates a nice, readable list of valid prefixes for typestrings, useful for error messages. - - This will be dynamically updated if we change the TYPESTRING_PREFIXES constant up top. - """ - valid_prefixes = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_PREFIXES]) - valid_prefixes = ", and ".join(valid_prefixes.rsplit(", ", 1)) - return valid_prefixes - - def _to_typestring(self, value: ValidRedisType) -> str: + def _to_typestring(value: RedisType) -> str: """Turn a valid Redis type into a typestring.""" for prefix, _type in TYPESTRING_PREFIXES: if isinstance(value, _type): return f"{prefix}{value}" - raise TypeError(f"RedisCache._from_typestring only supports the types {self._valid_typestring_types()}.") + raise TypeError(f"RedisCache._from_typestring only supports the types {NICE_TYPE_LIST}.") - def _from_typestring(self, value: Union[bytes, str]) -> ValidRedisType: + @staticmethod + def _from_typestring(value: Union[bytes, str]) -> RedisType: """Turn a typestring into a valid Redis type.""" # Stuff that comes out of Redis will be bytestrings, so let's decode those. if isinstance(value, bytes): @@ -121,7 +109,7 @@ class RedisCache: for prefix, _type in TYPESTRING_PREFIXES: if value.startswith(prefix): return _type(value[len(prefix):]) - raise TypeError(f"RedisCache._to_typestring only supports the prefixes {self._valid_typestring_prefixes()}.") + raise TypeError(f"RedisCache._to_typestring only supports the prefixes {NICE_PREFIX_LIST}.") def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" @@ -177,7 +165,7 @@ class RedisCache: """Return a beautiful representation of this object instance.""" return f"RedisCache(namespace={self._namespace!r})" - async def set(self, key: ValidRedisType, value: ValidRedisType) -> None: + async def set(self, key: RedisType, value: RedisType) -> None: """Store an item in the Redis cache.""" await self._validate_cache() @@ -186,7 +174,7 @@ class RedisCache: value = self._to_typestring(value) await self._redis.hset(self._namespace, key, value) - async def get(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: + async def get(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: """Get an item from the Redis cache.""" await self._validate_cache() key = self._to_typestring(key) @@ -198,7 +186,7 @@ class RedisCache: value = self._from_typestring(value) return value - async def delete(self, key: ValidRedisType) -> None: + async def delete(self, key: RedisType) -> None: """ Delete an item from the Redis cache. @@ -210,7 +198,7 @@ class RedisCache: key = self._to_typestring(key) return await self._redis.hdel(self._namespace, key) - async def contains(self, key: ValidRedisType) -> bool: + async def contains(self, key: RedisType) -> bool: """ Check if a key exists in the Redis cache. @@ -241,7 +229,7 @@ class RedisCache: await self._validate_cache() await self._redis.delete(self._namespace) - async def pop(self, key: ValidRedisType, default: Optional[ValidRedisType] = None) -> ValidRedisType: + async def pop(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: """Get the item, remove it from the cache, and provide a default if not found.""" value = await self.get(key, default) -- cgit v1.2.3 From 1d05a4d409cd0652cec36128114739ede2f529cf Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 10:54:34 +0200 Subject: Improves various docstrings and comments. Thanks to @MarkKoz for suggesting most of these in their code review. --- bot/utils/redis_cache.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index d5563c079..e4dce7526 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -24,14 +24,6 @@ class RedisCache: """ A simplified interface for a Redis connection. - This class must be created as a class attribute in a class. This is because it - uses __set_name__ to create a namespace like MyCog.my_class_attribute which is - used as a hash name when we store stuff in Redis, to prevent collisions. - - The class this object is instantiated in must also contains an attribute with an - instance of Bot. This is because Bot contains our redis_pool, which is how this - class communicates with the Redis server. - We implement several convenient methods that are fairly similar to have a dict behaves, and should be familiar to Python users. The biggest difference is that all the public methods in this class are coroutines, and must be awaited. @@ -39,6 +31,10 @@ class RedisCache: Because of limitations in Redis, this cache will only accept strings, integers and floats both for keys and values. + Please note that this class MUST be created as a class attribute, and that that class + must also contain an attribute with an instance of our Bot. See `__get__` and `__set_name__` + for more information about how this works. + Simple example for how to use this: class SomeCog(Cog): @@ -78,12 +74,18 @@ class RedisCache: _namespaces = [] def __init__(self) -> None: - """Raise a NotImplementedError if `__set_name__` hasn't been run.""" + """Initialize the RedisCache.""" self._namespace = None self.bot = None def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" + # We need a unique namespace, to prevent collisions. This loop + # will try appending underscores to the end of the namespace until + # it finds one that is unique. + # + # For example, if `john` and `john_` are both taken, the namespace will + # be `john__` at the end of this loop. while namespace in self._namespaces: namespace += "_" @@ -136,11 +138,26 @@ class RedisCache: Set the namespace to Class.attribute_name. Called automatically when this class is constructed inside a class as an attribute. + + This class MUST be created as a class attribute in a class, otherwise it will raise + exceptions whenever a method is used. This is because it uses this method to create + a namespace like `MyCog.my_class_attribute` which is used as a hash name when we store + stuff in Redis, to prevent collisions. """ self._set_namespace(f"{owner.__name__}.{attribute_name}") def __get__(self, instance: RedisCache, owner: Any) -> RedisCache: - """Fetch the Bot instance, we need it for the redis pool.""" + """ + This is called if the RedisCache is a class attribute, and is accessed. + + The class this object is instantiated in must contain an attribute with an + instance of Bot. This is because Bot contains our redis_session, which is + the mechanism by which we will communicate with the Redis server. + + Any attempt to use RedisCache in a class that does not have a Bot instance + will fail. It is mostly intended to be used inside of a Cog, although theoretically + it should work in any class that has a Bot instance. + """ if self.bot: return self -- cgit v1.2.3 From f05fefb4a51c6653f7f93805489838259782c376 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 11:26:04 +0200 Subject: Better RuntimeErrors. We provide suggestions for how to solve these problems now. --- bot/utils/redis_cache.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index e4dce7526..558ab33a7 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -124,7 +124,13 @@ class RedisCache: async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" if self.bot is None: - raise RuntimeError("Critical error: RedisCache has no `Bot` instance.") + raise RuntimeError( + "Critical error: RedisCache has no `Bot` instance. " + "This happens when the class RedisCache was created in doesn't " + "have a Bot instance. Please make sure that you're instantiating " + "the RedisCache inside a class that has a Bot instance " + "class attribute." + ) if self._namespace is None: raise RuntimeError( @@ -176,7 +182,13 @@ class RedisCache: self._redis = self.bot.redis_session return self else: - raise RuntimeError("Cannot initialize a RedisCache without a `Bot` instance.") + raise RuntimeError( + "Critical error: RedisCache has no `Bot` instance. " + "This happens when the class RedisCache was created in doesn't " + "have a Bot instance. Please make sure that you're instantiating " + "the RedisCache inside a class that has a Bot instance " + "class attribute." + ) def __repr__(self) -> str: """Return a beautiful representation of this object instance.""" -- cgit v1.2.3 From b2009d5304beba4829b7727ca154bb6a0d1cd50a Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 11:42:33 +0200 Subject: Make .items return ItemsView instead of AsyncIter There really was no compelling reason why this method should return an AsyncIterator or than that `async for items in cache.items()` has nice readability, but there were a few concerns. One is a concern about race conditions raised by @SebastiaanZ, and @MarkKoz raised a concern that it was misleading to have an AsyncIterator that only "pretended" to be lazy. To address these concerns, I've refactored it to return a regular ItemsView instead. I also improved the docstring, and fixed the relevant tests. --- bot/utils/redis_cache.py | 28 +++++++++++++++++++++------- tests/bot/utils/test_redis_cache.py | 4 ++-- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 558ab33a7..fb9a534bd 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, AsyncIterator, Dict, Optional, Union +from typing import Any, Dict, ItemsView, Optional, Union from bot.bot import Bot @@ -237,12 +237,26 @@ class RedisCache: key = self._to_typestring(key) return await self._redis.hexists(self._namespace, key) - async def items(self) -> AsyncIterator: - """Iterate all the items in the Redis cache.""" + async def items(self) -> ItemsView: + """ + Fetch all the key/value pairs in the cache. + + Returns a normal ItemsView, like you would get from dict.items(). + + Keep in mind that these items are just a _copy_ of the data in the + RedisCache - any changes you make to them will not be reflected + into the RedisCache itself. If you want to change these, you need + to make a .set call. + + Example: + items = await my_cache.items() + for key, value in items: + # Iterate like a normal dictionary + """ await self._validate_cache() - data = await self._redis.hgetall(self._namespace) # Get all the keys - for key, value in self._dict_from_typestring(data).items(): - yield key, value + return self._dict_from_typestring( + await self._redis.hgetall(self._namespace) + ).items() async def length(self) -> int: """Return the number of items in the Redis cache.""" @@ -251,7 +265,7 @@ class RedisCache: async def to_dict(self) -> Dict: """Convert to dict and return.""" - return {key: value async for key, value in self.items()} + return {key: value for key, value in await self.items()} async def clear(self) -> None: """Deletes the entire hash from the Redis cache.""" diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 150195726..6e12002ed 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -88,7 +88,7 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): await self.redis.set(key, value) # Consume the AsyncIterator into a regular list, easier to compare that way. - redis_items = [item async for item in self.redis.items()] + redis_items = [item for item in await self.redis.items()] # These sequences are probably in the same order now, but probably # isn't good enough for tests. Let's not rely on .hgetall always @@ -113,7 +113,7 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_to_dict(self): """Test that the .to_dict method returns a workable dictionary copy.""" copy = await self.redis.to_dict() - local_copy = {key: value async for key, value in self.redis.items()} + local_copy = {key: value for key, value in await self.redis.items()} self.assertIs(type(copy), dict) self.assertDictEqual(copy, local_copy) -- cgit v1.2.3 From 01bedcadf762262eef0a2b406faf66cdc16a5c85 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 13:04:41 +0200 Subject: Add .increment and .decrement methods. Sometimes, we just want to store a counter in the cache. In this case, it is convenient to have a single method that will allow us to increment or decrement this counter. These methods allow you to decrement or increment floats and integers by an specified amount. By default, it'll increment or decrement by 1. Since this involves several API requests, we create an asyncio.Lock so that we don't end up with race conditions. --- bot/utils/redis_cache.py | 35 +++++++++++++++++++++++++++++++++++ tests/bot/utils/test_redis_cache.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index fb9a534bd..290fae1a0 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio from typing import Any, Dict, ItemsView, Optional, Union from bot.bot import Bot @@ -77,6 +78,7 @@ class RedisCache: """Initialize the RedisCache.""" self._namespace = None self.bot = None + self.increment_lock = asyncio.Lock() def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" @@ -287,3 +289,36 @@ class RedisCache: """Update the Redis cache with multiple values.""" await self._validate_cache() await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) + + async def increment(self, key: RedisType, amount: Optional[int, float] = 1) -> None: + """ + Increment the value by `amount`. + + This works for both floats and ints, but will raise a TypeError + if you try to do it for any other type of value. + + This also supports negative amounts, although it would provide better + readability to use .decrement() for that. + """ + # Since this has several API calls, we need a lock to prevent race conditions + async with self.increment_lock: + value = await self.get(key) + + # Can't increment a non-existing value + if value is None: + raise RuntimeError("The provided key does not exist!") + + # If it does exist, and it's an int or a float, increment and set it. + if isinstance(value, int) or isinstance(value, float): + value += amount + await self.set(key, value) + else: + raise TypeError("You may only increment or decrement values that are integers or floats.") + + async def decrement(self, key: RedisType, amount: Optional[int, float] = 1) -> None: + """ + Decrement the value by `amount`. + + Basically just does the opposite of .increment. + """ + await self.increment(key, -amount) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 6e12002ed..dbbaef018 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -173,3 +173,37 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): with self.assertRaises(TypeError): self.redis._to_typestring(["internet"]) self.redis._from_typestring("o|firedog") + + async def test_increment_decrement(self): + """Test .increment and .decrement methods.""" + await self.redis.set("entropic", 5) + await self.redis.set("disentropic", 12.5) + + # Test default increment + await self.redis.increment("entropic") + self.assertEqual(await self.redis.get("entropic"), 6) + + # Test default decrement + await self.redis.decrement("entropic") + self.assertEqual(await self.redis.get("entropic"), 5) + + # Test float increment with float + await self.redis.increment("disentropic", 2.0) + self.assertEqual(await self.redis.get("disentropic"), 14.5) + + # Test float increment with int + await self.redis.increment("disentropic", 2) + self.assertEqual(await self.redis.get("disentropic"), 16.5) + + # Test negative increments, because why not. + await self.redis.increment("entropic", -5) + self.assertEqual(await self.redis.get("entropic"), 0) + + # Negative decrements? Sure. + await self.redis.decrement("entropic", -5) + self.assertEqual(await self.redis.get("entropic"), 5) + + # What about if we use a negative float to decrement an int? + # This should convert the type into a float. + await self.redis.decrement("entropic", -2.5) + self.assertEqual(await self.redis.get("entropic"), 7.5) -- cgit v1.2.3 From f80ce10aee4a46ab4fc4a2d249fe182fb812a826 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 13:13:03 +0200 Subject: Rename Bot._redis_ready to Bot.redis_ready It's a public attribute, we're accessing it from RedisCache. --- bot/bot.py | 6 +++--- bot/utils/redis_cache.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/bot.py b/bot/bot.py index bf7f9c9df..0d423201b 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -30,12 +30,12 @@ class Bot(commands.Bot): self.http_session: Optional[aiohttp.ClientSession] = None self.redis_session: Optional[aioredis.Redis] = None + self.redis_ready = asyncio.Event() self.api_client = api.APIClient(loop=self.loop) self._connector = None self._resolver = None self._guild_available = asyncio.Event() - self._redis_ready = asyncio.Event() statsd_url = constants.Stats.statsd_host @@ -53,7 +53,7 @@ class Bot(commands.Bot): address=(constants.Redis.host, constants.Redis.port), password=constants.Redis.password, ) - self._redis_ready.set() + self.redis_ready.set() def add_cog(self, cog: commands.Cog) -> None: """Adds a "cog" to the bot and logs the operation.""" @@ -90,7 +90,7 @@ class Bot(commands.Bot): self.stats._transport.close() if self.redis_session: - self._redis_ready.clear() + self.redis_ready.clear() self.redis_session.close() await self.redis_session.wait_closed() diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 290fae1a0..bd885c22c 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -139,7 +139,7 @@ class RedisCache: "Critical error: RedisCache has no namespace. " "Did you initialize this object as a class attribute?" ) - await self.bot._redis_ready.wait() + await self.bot.redis_ready.wait() def __set_name__(self, owner: Any, attribute_name: str) -> None: """ -- cgit v1.2.3 From 361b740f27a579a085e93ebfdd06df10f386cca1 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 13:44:18 +0200 Subject: Add logging to the RedisCache. Mostly trace and exception logging. --- bot/utils/redis_cache.py | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index bd885c22c..5fc34d464 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -1,10 +1,13 @@ from __future__ import annotations import asyncio +import logging from typing import Any, Dict, ItemsView, Optional, Union from bot.bot import Bot +log = logging.getLogger(__name__) + RedisType = Union[str, int, float] TYPESTRING_PREFIXES = ( ("f|", float), @@ -91,6 +94,7 @@ class RedisCache: while namespace in self._namespaces: namespace += "_" + log.trace(f"RedisCache setting namespace to {self._namespace}") self._namespaces.append(namespace) self._namespace = namespace @@ -126,6 +130,7 @@ class RedisCache: async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" if self.bot is None: + log.exception("Attempt to use RedisCache with no `Bot` instance.") raise RuntimeError( "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " @@ -135,6 +140,7 @@ class RedisCache: ) if self._namespace is None: + log.exception("Attempt to use RedisCache with no namespace.") raise RuntimeError( "Critical error: RedisCache has no namespace. " "Did you initialize this object as a class attribute?" @@ -170,9 +176,14 @@ class RedisCache: return self if self._namespace is None: + log.exception("RedisCache must be a class attribute.") raise RuntimeError("RedisCache must be a class attribute.") if instance is None: + log.exception( + "Attempt to access RedisCache instance through the cog's class object " + "before accessing it through the cog instance." + ) raise RuntimeError( "You must access the RedisCache instance through the cog instance " "before accessing it using the cog's class object." @@ -184,6 +195,7 @@ class RedisCache: self._redis = self.bot.redis_session return self else: + log.exception("Attempt to use RedisCache with no `Bot` instance.") raise RuntimeError( "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " @@ -203,18 +215,24 @@ class RedisCache: # Convert to a typestring and then set it key = self._to_typestring(key) value = self._to_typestring(value) + + log.trace(f"Setting {key} to {value}.") await self._redis.hset(self._namespace, key, value) async def get(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: """Get an item from the Redis cache.""" await self._validate_cache() key = self._to_typestring(key) + + log.trace(f"Attempting to retrieve {key}.") value = await self._redis.hget(self._namespace, key) if value is None: + log.trace(f"Value not found, returning default value {default}") return default else: value = self._from_typestring(value) + log.trace(f"Value found, returning value {value}") return value async def delete(self, key: RedisType) -> None: @@ -227,6 +245,8 @@ class RedisCache: """ await self._validate_cache() key = self._to_typestring(key) + + log.trace(f"Attempting to delete {key}.") return await self._redis.hdel(self._namespace, key) async def contains(self, key: RedisType) -> bool: @@ -237,7 +257,10 @@ class RedisCache: """ await self._validate_cache() key = self._to_typestring(key) - return await self._redis.hexists(self._namespace, key) + exists = await self._redis.hexists(self._namespace, key) + + log.trace(f"Testing if {key} exists in the RedisCache - Result is {exists}") + return exists async def items(self) -> ItemsView: """ @@ -256,14 +279,19 @@ class RedisCache: # Iterate like a normal dictionary """ await self._validate_cache() - return self._dict_from_typestring( + items = self._dict_from_typestring( await self._redis.hgetall(self._namespace) ).items() + log.trace(f"Retrieving all key/value pairs from cache, total of {len(items)} items.") + return items + async def length(self) -> int: """Return the number of items in the Redis cache.""" await self._validate_cache() - return await self._redis.hlen(self._namespace) + number_of_items = await self._redis.hlen(self._namespace) + log.trace(f"Returning length. Result is {number_of_items}.") + return number_of_items async def to_dict(self) -> Dict: """Convert to dict and return.""" @@ -272,15 +300,18 @@ class RedisCache: async def clear(self) -> None: """Deletes the entire hash from the Redis cache.""" await self._validate_cache() + log.trace("Clearing the cache of all key/value pairs.") await self._redis.delete(self._namespace) async def pop(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: """Get the item, remove it from the cache, and provide a default if not found.""" + log.trace(f"Attempting to pop {key}.") value = await self.get(key, default) # No need to try to delete something that doesn't exist, # that's just a superfluous API call. if value != default: + log.trace(f"Key {key} exists, deleting it from the cache.") await self.delete(key) return value @@ -288,6 +319,7 @@ class RedisCache: async def update(self, items: Dict) -> None: """Update the Redis cache with multiple values.""" await self._validate_cache() + log.trace(f"Updating the cache with the following items:\n{items}") await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) async def increment(self, key: RedisType, amount: Optional[int, float] = 1) -> None: @@ -300,12 +332,15 @@ class RedisCache: This also supports negative amounts, although it would provide better readability to use .decrement() for that. """ + log.trace(f"Attempting to increment/decrement the value with the key {key} by {amount}.") + # Since this has several API calls, we need a lock to prevent race conditions async with self.increment_lock: value = await self.get(key) # Can't increment a non-existing value if value is None: + log.exception("Attempt to increment/decrement value for non-existent key.") raise RuntimeError("The provided key does not exist!") # If it does exist, and it's an int or a float, increment and set it. @@ -313,6 +348,7 @@ class RedisCache: value += amount await self.set(key, value) else: + log.exception("Attempt to increment/decrement non-numerical value.") raise TypeError("You may only increment or decrement values that are integers or floats.") async def decrement(self, key: RedisType, amount: Optional[int, float] = 1) -> None: -- cgit v1.2.3 From c5e6e8f796265ee6faebdd3d02c839972cd028a9 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 13:49:20 +0200 Subject: MockBot needs to be aware of redis_ready Forgot to update the additional_spec_asyncs when changing the name of this Bot attribute to be public. --- tests/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers.py b/tests/helpers.py index 5ad826156..13283339b 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -287,7 +287,7 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): For more information, see the `MockGuild` docstring. """ spec_set = Bot(command_prefix=unittest.mock.MagicMock(), loop=_get_mock_loop()) - additional_spec_asyncs = ("wait_for", "_redis_ready") + additional_spec_asyncs = ("wait_for", "redis_ready") def __init__(self, **kwargs) -> None: super().__init__(**kwargs) -- cgit v1.2.3 From 66d273f0b8f1de6850760f4d561db446c027fdfe Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 15:21:35 +0200 Subject: Add an option to use fakeredis in Bot. Without this option, all contributors would need to set up a Redis server in order to run the bot. But with use_fakeredis set to True, this is no longer necessary because it will just set up a fakeredis redis pool instead of trying to contact an actual server. This is more than good enough for most local testing purposes, since data persistence across restarts isn't really relevant for them. This also means we need to move fakeredis into our real dependency list instead of having it as a dev dependency, so there's a minor change for that as well. I also made a small kaizen change to sort all the dependencies in the Pipfile alphabetically. --- Pipfile | 28 +++++++++++++------------- Pipfile.lock | 58 +++++++++++++++++++++++++++--------------------------- bot/bot.py | 26 +++++++++++++++++++----- bot/constants.py | 1 + config-default.yml | 1 + 5 files changed, 66 insertions(+), 48 deletions(-) diff --git a/Pipfile b/Pipfile index cd2f2ad7a..b42ca6d58 100644 --- a/Pipfile +++ b/Pipfile @@ -4,30 +4,30 @@ verify_ssl = true name = "pypi" [packages] -discord.py = "~=1.3.2" +aio-pika = "~=6.1" aiodns = "~=2.0" aiohttp = "~=3.5" -sphinx = "~=2.2" -markdownify = "~=0.4" -lxml = "~=4.4" -pyyaml = "~=5.1" +aioredis = "~=1.3.1" +beautifulsoup4 = "~=4.9" +colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} +coloredlogs = "~=14.0" +deepdiff = "~=4.0" +discord.py = "~=1.3.2" +fakeredis = "~=1.4" +feedparser = "~=5.2" fuzzywuzzy = "~=0.17" -aio-pika = "~=6.1" +lxml = "~=4.4" +markdownify = "~=0.4" +more_itertools = "~=8.2" python-dateutil = "~=2.8" -deepdiff = "~=4.0" +pyyaml = "~=5.1" requests = "~=2.22" -more_itertools = "~=8.2" sentry-sdk = "~=0.14" -coloredlogs = "~=14.0" -colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} +sphinx = "~=2.2" statsd = "~=3.3" -feedparser = "~=5.2" -beautifulsoup4 = "~=4.9" -aioredis = "~=1.3.1" [dev-packages] coverage = "~=5.0" -fakeredis = "~=1.4" flake8 = "~=3.7" flake8-annotations = "~=2.0" flake8-bugbear = "~=20.1" diff --git a/Pipfile.lock b/Pipfile.lock index 1941f6887..0e591710c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c0b3e4d3e2c9ddb6ba28d2c09d521fe90ad4ea3df5c7ea7cd3a8b679fb3f85f9" + "sha256": "0297accc3d614d3da8080b89d56ef7fe489c28a0ada8102df396a604af7ee330" }, "pipfile-spec": 6, "requires": { @@ -196,6 +196,14 @@ ], "version": "==0.16" }, + "fakeredis": { + "hashes": [ + "sha256:4d170886865a91dbc8b7f8cbd4e5d488f4c5f2f25dfae127f001617bbe9e8f97", + "sha256:647b2593d349d9d4e566c8dadb2e4c71ba35be5bdc4f1f7ac2d565a12a965053" + ], + "index": "pypi", + "version": "==1.4.1" + }, "feedparser": { "hashes": [ "sha256:bd030652c2d08532c034c27fcd7c85868e7fa3cb2b17f230a44a6bbc92519bf9", @@ -501,6 +509,13 @@ "index": "pypi", "version": "==5.3.1" }, + "redis": { + "hashes": [ + "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", + "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" + ], + "version": "==3.5.2" + }, "requests": { "hashes": [ "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", @@ -531,6 +546,13 @@ ], "version": "==2.0.0" }, + "sortedcontainers": { + "hashes": [ + "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", + "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" + ], + "version": "==2.1.0" + }, "soupsieve": { "hashes": [ "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", @@ -718,14 +740,6 @@ ], "version": "==0.3.0" }, - "fakeredis": { - "hashes": [ - "sha256:4d170886865a91dbc8b7f8cbd4e5d488f4c5f2f25dfae127f001617bbe9e8f97", - "sha256:647b2593d349d9d4e566c8dadb2e4c71ba35be5bdc4f1f7ac2d565a12a965053" - ], - "index": "pypi", - "version": "==1.4.1" - }, "filelock": { "hashes": [ "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", @@ -735,11 +749,11 @@ }, "flake8": { "hashes": [ - "sha256:6c1193b0c3f853ef763969238f6c81e9e63ace9d024518edc020d5f1d6d93195", - "sha256:ea6623797bf9a52f4c9577d780da0bb17d65f870213f7b5bcc9fca82540c31d5" + "sha256:c69ac1668e434d37a2d2880b3ca9aafd54b3a10a3ac1ab101d22f29e29cf8634", + "sha256:ccaa799ef9893cebe69fdfefed76865aeaefbb94cb8545617b2298786a4de9a5" ], "index": "pypi", - "version": "==3.8.1" + "version": "==3.8.2" }, "flake8-annotations": { "hashes": [ @@ -805,10 +819,10 @@ }, "identify": { "hashes": [ - "sha256:23c18d97bb50e05be1a54917ee45cc61d57cb96aedc06aabb2b02331edf0dbf0", - "sha256:88ed90632023e52a6495749c6732e61e08ec9f4f04e95484a5c37b9caf40283c" + "sha256:0f3c3aac62b51b86fea6ff52fe8ff9e06f57f10411502443809064d23e16f1c2", + "sha256:f9ad3d41f01e98eb066b6e05c5b184fd1e925fadec48eb165b4e01c72a1ef3a7" ], - "version": "==1.4.15" + "version": "==1.4.16" }, "mccabe": { "hashes": [ @@ -877,13 +891,6 @@ "index": "pypi", "version": "==5.3.1" }, - "redis": { - "hashes": [ - "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", - "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" - ], - "version": "==3.5.2" - }, "six": { "hashes": [ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", @@ -898,13 +905,6 @@ ], "version": "==2.0.0" }, - "sortedcontainers": { - "hashes": [ - "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", - "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" - ], - "version": "==2.1.0" - }, "toml": { "hashes": [ "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", diff --git a/bot/bot.py b/bot/bot.py index 0d423201b..f1365d532 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -7,6 +7,7 @@ from typing import Optional import aiohttp import aioredis import discord +import fakeredis.aioredis from discord.ext import commands from sentry_sdk import push_scope @@ -48,11 +49,26 @@ class Bot(commands.Bot): self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") async def _create_redis_session(self) -> None: - """Create the Redis connection pool, and then open the redis event gate.""" - self.redis_session = await aioredis.create_redis_pool( - address=(constants.Redis.host, constants.Redis.port), - password=constants.Redis.password, - ) + """ + Create the Redis connection pool, and then open the redis event gate. + + If constants.Redis.use_fakeredis is True, we'll set up a fake redis pool instead + of attempting to communicate with a real Redis server. This is useful because it + means contributors don't necessarily need to get Redis running locally just + to run the bot. + + The fakeredis cache won't have persistence across restarts, but that + usually won't matter for local bot testing. + """ + if constants.Redis.use_fakeredis: + log.info("Using fakeredis instead of communicating with a real Redis server.") + self.redis_session = await fakeredis.aioredis.create_redis_pool() + else: + self.redis_session = await aioredis.create_redis_pool( + address=(constants.Redis.host, constants.Redis.port), + password=constants.Redis.password, + ) + self.redis_ready.set() def add_cog(self, cog: commands.Cog) -> None: diff --git a/bot/constants.py b/bot/constants.py index 5d854dd7a..75d394b6a 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -208,6 +208,7 @@ class Redis(metaclass=YAMLGetter): host: str port: int password: str + use_fakeredis: bool # If this is True, Bot will use fakeredis.aioredis class Filter(metaclass=YAMLGetter): diff --git a/config-default.yml b/config-default.yml index 5be393463..cee955f20 100644 --- a/config-default.yml +++ b/config-default.yml @@ -7,6 +7,7 @@ bot: host: "redis" port: 6379 password: !ENV "REDIS_PASSWORD" + use_fakeredis: false stats: statsd_host: "graphite" -- cgit v1.2.3 From ad8b1fa455e141074daec5047682e82ed96db1f5 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sun, 24 May 2020 19:09:45 +0200 Subject: Improve error and error testing for increment Changed a RuntimeError to a KeyError (thanks @MarkKoz), and also added some tests to ensure that the right errors are raised whenever this method is used incorrectly. --- bot/utils/redis_cache.py | 2 +- tests/bot/utils/test_redis_cache.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 5fc34d464..b91d663f3 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -341,7 +341,7 @@ class RedisCache: # Can't increment a non-existing value if value is None: log.exception("Attempt to increment/decrement value for non-existent key.") - raise RuntimeError("The provided key does not exist!") + raise KeyError("The provided key does not exist!") # If it does exist, and it's an int or a float, increment and set it. if isinstance(value, int) or isinstance(value, float): diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index dbbaef018..7405487ed 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -207,3 +207,11 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): # This should convert the type into a float. await self.redis.decrement("entropic", -2.5) self.assertEqual(await self.redis.get("entropic"), 7.5) + + # Let's test that they raise the right errors + with self.assertRaises(KeyError): + await self.redis.increment("doesn't_exist!") + + await self.redis.set("stringthing", "stringthing") + with self.assertRaises(TypeError): + await self.redis.increment("stringthing") -- cgit v1.2.3 From 185c9e84b5fde13ab21de614564eee94963d05b5 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Sun, 24 May 2020 22:46:48 +0100 Subject: Add discord.gift to URL blacklist, closes #958 --- config-default.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config-default.yml b/config-default.yml index 2e98186f1..9a28be700 100644 --- a/config-default.yml +++ b/config-default.yml @@ -318,6 +318,7 @@ filter: - poweredbysecurity.online - ssteam.site - steamwalletgift.com + - discord.gift word_watchlist: - goo+ks* -- cgit v1.2.3 From 856cecbd2354d4cbdbace5a39b7eb9e3d3bf23c7 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sun, 24 May 2020 19:29:13 -0700 Subject: Add support for Union type annotations for constants Note that `Optional[x]` is just an alias for `Union[None, x]` so this effectively supports `Optional` too. This was especially troublesome because the redis password must be unset/None in order to avoid authentication, but the test would complain that `None` isn't a `str`. Setting to an empty string would pass the test but then make redis authenticate and fail. --- bot/constants.py | 14 +++++++------- tests/bot/test_constants.py | 17 ++++++++++++----- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/bot/constants.py b/bot/constants.py index 75d394b6a..145ae54db 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -15,7 +15,7 @@ import os from collections.abc import Mapping from enum import Enum from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional import yaml @@ -198,7 +198,7 @@ class Bot(metaclass=YAMLGetter): prefix: str token: str - sentry_dsn: str + sentry_dsn: Optional[str] class Redis(metaclass=YAMLGetter): @@ -207,7 +207,7 @@ class Redis(metaclass=YAMLGetter): host: str port: int - password: str + password: Optional[str] use_fakeredis: bool # If this is True, Bot will use fakeredis.aioredis @@ -459,7 +459,7 @@ class Guild(metaclass=YAMLGetter): class Keys(metaclass=YAMLGetter): section = "keys" - site_api: str + site_api: Optional[str] class URLs(metaclass=YAMLGetter): @@ -502,8 +502,8 @@ class Reddit(metaclass=YAMLGetter): section = "reddit" subreddits: list - client_id: str - secret: str + client_id: Optional[str] + secret: Optional[str] class Wolfram(metaclass=YAMLGetter): @@ -511,7 +511,7 @@ class Wolfram(metaclass=YAMLGetter): user_limit_day: int guild_limit_day: int - key: str + key: Optional[str] class AntiSpam(metaclass=YAMLGetter): diff --git a/tests/bot/test_constants.py b/tests/bot/test_constants.py index dae7c066c..db9a9bcb0 100644 --- a/tests/bot/test_constants.py +++ b/tests/bot/test_constants.py @@ -1,4 +1,5 @@ import inspect +import typing import unittest from bot import constants @@ -8,7 +9,7 @@ class ConstantsTests(unittest.TestCase): """Tests for our constants.""" def test_section_configuration_matches_type_specification(self): - """The section annotations should match the actual types of the sections.""" + """"The section annotations should match the actual types of the sections.""" sections = ( cls @@ -19,8 +20,14 @@ class ConstantsTests(unittest.TestCase): for name, annotation in section.__annotations__.items(): with self.subTest(section=section, name=name, annotation=annotation): value = getattr(section, name) + annotation_args = typing.get_args(annotation) - if getattr(annotation, '_name', None) in ('Dict', 'List'): - self.skipTest("Cannot validate containers yet.") - - self.assertIsInstance(value, annotation) + if not annotation_args: + self.assertIsInstance(value, annotation) + else: + origin = typing.get_origin(annotation) + if origin is typing.Union: + is_instance = any(isinstance(value, arg) for arg in annotation_args) + self.assertTrue(is_instance) + else: + self.skipTest(f"Validating type {annotation} is unsupported.") -- cgit v1.2.3 From 0ede719d7beb36f476ac26f948ab940882978476 Mon Sep 17 00:00:00 2001 From: Jannes Jonkers Date: Mon, 25 May 2020 20:44:35 +0200 Subject: AntiMalware tests - Switched from monkeypatch to unittest.patch --- tests/bot/cogs/test_antimalware.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index fab063201..f219fc1ba 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,5 +1,5 @@ import unittest -from unittest.mock import AsyncMock, Mock +from unittest.mock import AsyncMock, Mock, patch from discord import NotFound @@ -10,6 +10,7 @@ from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole MODULE = "bot.cogs.antimalware" +@patch(f"{MODULE}.AntiMalwareConfig.whitelist", new=[".first", ".second", ".third"]) class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): """Test the AntiMalware cog.""" @@ -18,7 +19,6 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.bot = MockBot() self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() - AntiMalwareConfig.whitelist = [".first", ".second", ".third"] async def test_message_with_allowed_attachment(self): """Messages with allowed extensions should not be deleted""" -- cgit v1.2.3 From 9b9aa9b2adbdcd0e0b8c4f4ad38f112a9566fa2f Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 12:03:09 -0700 Subject: Support validating collection types for constants This is a simple validation that only check the type of the collection. It does not validate the types inside the collection because that has proven to be quite complex. --- tests/bot/test_constants.py | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/tests/bot/test_constants.py b/tests/bot/test_constants.py index db9a9bcb0..2937b6189 100644 --- a/tests/bot/test_constants.py +++ b/tests/bot/test_constants.py @@ -5,6 +5,31 @@ import unittest from bot import constants +def is_annotation_instance(value: typing.Any, annotation: typing.Any) -> bool: + """ + Return True if `value` is an instance of the type represented by `annotation`. + + This doesn't account for things like Unions or checking for homogenous types in collections. + """ + origin = typing.get_origin(annotation) + + # This is done in case a bare e.g. `typing.List` is used. + # In such case, for the assertion to pass, the type needs to be normalised to e.g. `list`. + # `get_origin()` does this normalisation for us. + type_ = annotation if origin is None else origin + + return isinstance(value, type_) + + +def is_any_instance(value: typing.Any, types: typing.Collection) -> bool: + """Return True if `value` is an instance of any type in `types`.""" + for type_ in types: + if is_annotation_instance(value, type_): + return True + + return False + + class ConstantsTests(unittest.TestCase): """Tests for our constants.""" @@ -20,14 +45,13 @@ class ConstantsTests(unittest.TestCase): for name, annotation in section.__annotations__.items(): with self.subTest(section=section, name=name, annotation=annotation): value = getattr(section, name) + origin = typing.get_origin(annotation) annotation_args = typing.get_args(annotation) + failure_msg = f"{value} is not an instance of {annotation}" - if not annotation_args: - self.assertIsInstance(value, annotation) + if origin is typing.Union: + is_instance = is_any_instance(value, annotation_args) + self.assertTrue(is_instance, failure_msg) else: - origin = typing.get_origin(annotation) - if origin is typing.Union: - is_instance = any(isinstance(value, arg) for arg in annotation_args) - self.assertTrue(is_instance) - else: - self.skipTest(f"Validating type {annotation} is unsupported.") + is_instance = is_annotation_instance(value, annotation) + self.assertTrue(is_instance, failure_msg) -- cgit v1.2.3 From 87d42add019e8ef1bad5d9593f6ed5a803e4d153 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 12:04:50 -0700 Subject: Improve output of section name in config validation subtests --- tests/bot/test_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bot/test_constants.py b/tests/bot/test_constants.py index 2937b6189..f10d6fbe8 100644 --- a/tests/bot/test_constants.py +++ b/tests/bot/test_constants.py @@ -43,7 +43,7 @@ class ConstantsTests(unittest.TestCase): ) for section in sections: for name, annotation in section.__annotations__.items(): - with self.subTest(section=section, name=name, annotation=annotation): + with self.subTest(section=section.__name__, name=name, annotation=annotation): value = getattr(section, name) origin = typing.get_origin(annotation) annotation_args = typing.get_args(annotation) -- cgit v1.2.3 From 8b5c1aabf58eb3c794cc61173bd7500a696a8376 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Mon, 25 May 2020 12:12:24 -0700 Subject: Expose the redis port to the host Useful for those that run redis with docker-compose but not the bot. The bot on the host won't have access to the Docker network in such case so the port must be exposed. --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 1bcf1008e..9884e35f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,8 @@ services: redis: image: redis:5.0.9 + ports: + - "127.0.0.1:6379:6379" web: image: pythondiscord/site:latest -- cgit v1.2.3 From 6cedfdc0b24ea44b86fca039c9d7335072abede6 Mon Sep 17 00:00:00 2001 From: Joseph Banks Date: Tue, 26 May 2020 02:21:58 +0100 Subject: [stats] Do not report modmail channels to stats --- bot/cogs/stats.py | 8 +++++++- bot/constants.py | 2 ++ config-default.yml | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py index 9baf222e2..14409ecb0 100644 --- a/bot/cogs/stats.py +++ b/bot/cogs/stats.py @@ -6,7 +6,7 @@ from discord.ext.commands import Cog, Context from discord.ext.tasks import loop from bot.bot import Bot -from bot.constants import Channels, Guild, Stats as StatConf +from bot.constants import Categories, Channels, Guild, Stats as StatConf CHANNEL_NAME_OVERRIDES = { @@ -36,6 +36,12 @@ class Stats(Cog): if message.guild.id != Guild.id: return + if message.channel.category.id == Categories.modmail: + if message.channel.id != Channels.incidents: + # Do not report modmail channels to stats, there are too many + # of them for interesting statistics to be drawn out of this. + return + reformatted_name = message.channel.name.replace('-', '_') if CHANNEL_NAME_OVERRIDES.get(message.channel.id): diff --git a/bot/constants.py b/bot/constants.py index 3003c9d36..39de2ee41 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -365,6 +365,7 @@ class Categories(metaclass=YAMLGetter): help_available: int help_in_use: int help_dormant: int + modmail: int class Channels(metaclass=YAMLGetter): @@ -384,6 +385,7 @@ class Channels(metaclass=YAMLGetter): esoteric: int helpers: int how_to_get_help: int + incidents: int message_log: int meta: int mod_alerts: int diff --git a/config-default.yml b/config-default.yml index 9a28be700..c7d25894c 100644 --- a/config-default.yml +++ b/config-default.yml @@ -118,6 +118,7 @@ guild: help_available: 691405807388196926 help_in_use: 696958401460043776 help_dormant: 691405908919451718 + modmail: 714494672835444826 channels: announcements: 354619224620138496 @@ -164,6 +165,7 @@ guild: mod_spam: &MOD_SPAM 620607373828030464 organisation: &ORGANISATION 551789653284356126 staff_lounge: &STAFF_LOUNGE 464905259261755392 + incidents: 714214212200562749 # Voice admins_voice: &ADMINS_VOICE 500734494840717332 -- cgit v1.2.3 From 9b882c768344cc866d366dc595fbfc19bc2cb6de Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:38:00 +0200 Subject: Turn log.exception into log.error Also, refactor error messages to be consistent and DRY throughout the file. --- bot/utils/redis_cache.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index b91d663f3..da78f1431 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -130,21 +130,24 @@ class RedisCache: async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" if self.bot is None: - log.exception("Attempt to use RedisCache with no `Bot` instance.") - raise RuntimeError( + error_message = ( "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " "have a Bot instance. Please make sure that you're instantiating " "the RedisCache inside a class that has a Bot instance " "class attribute." ) + log.error(error_message) + raise RuntimeError(error_message) if self._namespace is None: - log.exception("Attempt to use RedisCache with no namespace.") - raise RuntimeError( + error_message = ( "Critical error: RedisCache has no namespace. " "Did you initialize this object as a class attribute?" ) + log.error(error_message) + raise RuntimeError(error_message) + await self.bot.redis_ready.wait() def __set_name__(self, owner: Any, attribute_name: str) -> None: @@ -176,18 +179,17 @@ class RedisCache: return self if self._namespace is None: - log.exception("RedisCache must be a class attribute.") - raise RuntimeError("RedisCache must be a class attribute.") + error_message = "RedisCache must be a class attribute." + log.error(error_message) + raise RuntimeError(error_message) if instance is None: - log.exception( - "Attempt to access RedisCache instance through the cog's class object " - "before accessing it through the cog instance." - ) - raise RuntimeError( + error_message = ( "You must access the RedisCache instance through the cog instance " "before accessing it using the cog's class object." ) + log.error(error_message) + raise RuntimeError(error_message) for attribute in vars(instance).values(): if isinstance(attribute, Bot): @@ -195,14 +197,15 @@ class RedisCache: self._redis = self.bot.redis_session return self else: - log.exception("Attempt to use RedisCache with no `Bot` instance.") - raise RuntimeError( + error_message = ( "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " "have a Bot instance. Please make sure that you're instantiating " "the RedisCache inside a class that has a Bot instance " "class attribute." ) + log.error(error_message) + raise RuntimeError(error_message) def __repr__(self) -> str: """Return a beautiful representation of this object instance.""" @@ -340,16 +343,18 @@ class RedisCache: # Can't increment a non-existing value if value is None: - log.exception("Attempt to increment/decrement value for non-existent key.") - raise KeyError("The provided key does not exist!") + error_message = "The provided key does not exist!" + log.error(error_message) + raise KeyError(error_message) # If it does exist, and it's an int or a float, increment and set it. if isinstance(value, int) or isinstance(value, float): value += amount await self.set(key, value) else: - log.exception("Attempt to increment/decrement non-numerical value.") - raise TypeError("You may only increment or decrement values that are integers or floats.") + error_message = "You may only increment or decrement values that are integers or floats." + log.error(error_message) + raise TypeError(error_message) async def decrement(self, key: RedisType, amount: Optional[int, float] = 1) -> None: """ -- cgit v1.2.3 From 723c1d3337b0a59401f1d3fc50a123f0314a5d3e Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:47:16 +0200 Subject: Fix edge case where pop might not delete. If you passed a key for a value that was the same as your optional, it would just return it but not delete it. This edge case isn't worth it, so I'm just removing that condition and letting the extra API call fly. --- bot/utils/redis_cache.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index da78f1431..dd20b5842 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -222,7 +222,7 @@ class RedisCache: log.trace(f"Setting {key} to {value}.") await self._redis.hset(self._namespace, key, value) - async def get(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: + async def get(self, key: RedisType, default: Optional[RedisType] = None) -> Optional[RedisType]: """Get an item from the Redis cache.""" await self._validate_cache() key = self._to_typestring(key) @@ -311,11 +311,11 @@ class RedisCache: log.trace(f"Attempting to pop {key}.") value = await self.get(key, default) - # No need to try to delete something that doesn't exist, - # that's just a superfluous API call. - if value != default: - log.trace(f"Key {key} exists, deleting it from the cache.") - await self.delete(key) + log.trace( + f"Attempting to delete item with key '{key}' from the cache. " + "If this key doesn't exist, nothing will happen." + ) + await self.delete(key) return value -- cgit v1.2.3 From b630aceb1adb624f45465aeb698e844f9ea340c8 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:51:53 +0200 Subject: Add better docstring for RedisCache.update --- bot/utils/redis_cache.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index dd20b5842..b77ec47a2 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -319,8 +319,18 @@ class RedisCache: return value - async def update(self, items: Dict) -> None: - """Update the Redis cache with multiple values.""" + async def update(self, items: Dict[RedisType, RedisType]) -> None: + """ + Update the Redis cache with multiple values. + + This works exactly like dict.update from a normal dictionary. You pass + a dictionary with one or more key/value pairs into this method. If the keys + do not exist in the RedisCache, they are created. If they do exist, the values + are updated with the new ones from `items`. + + Please note that both the keys and the values in the `items` dictionary + must consist of valid RedisTypes - ints, floats, or strings. + """ await self._validate_cache() log.trace(f"Updating the cache with the following items:\n{items}") await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) -- cgit v1.2.3 From 19206734df651c7d65e5114715db6db9253cb7d6 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:52:52 +0200 Subject: Make self.increment_lock private. --- bot/utils/redis_cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index b77ec47a2..89af225e2 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -81,7 +81,7 @@ class RedisCache: """Initialize the RedisCache.""" self._namespace = None self.bot = None - self.increment_lock = asyncio.Lock() + self._increment_lock = asyncio.Lock() def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" @@ -348,7 +348,7 @@ class RedisCache: log.trace(f"Attempting to increment/decrement the value with the key {key} by {amount}.") # Since this has several API calls, we need a lock to prevent race conditions - async with self.increment_lock: + async with self._increment_lock: value = await self.get(key) # Can't increment a non-existing value -- cgit v1.2.3 From 46a377deef15545d1b860e283d8d0f8291298cee Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:55:12 +0200 Subject: Improve some docstrings for RedisCache. Thanks @MarkKoz! --- bot/utils/redis_cache.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 89af225e2..a1196fcb5 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -134,8 +134,7 @@ class RedisCache: "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " "have a Bot instance. Please make sure that you're instantiating " - "the RedisCache inside a class that has a Bot instance " - "class attribute." + "the RedisCache inside a class that has a Bot instance attribute." ) log.error(error_message) raise RuntimeError(error_message) @@ -143,7 +142,7 @@ class RedisCache: if self._namespace is None: error_message = ( "Critical error: RedisCache has no namespace. " - "Did you initialize this object as a class attribute?" + "This object must be initialized as a class attribute." ) log.error(error_message) raise RuntimeError(error_message) @@ -201,8 +200,7 @@ class RedisCache: "Critical error: RedisCache has no `Bot` instance. " "This happens when the class RedisCache was created in doesn't " "have a Bot instance. Please make sure that you're instantiating " - "the RedisCache inside a class that has a Bot instance " - "class attribute." + "the RedisCache inside a class that has a Bot instance attribute." ) log.error(error_message) raise RuntimeError(error_message) -- cgit v1.2.3 From ec8205cfd7adb5e40aabd52e497e4e387b932211 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 18:56:30 +0200 Subject: Swap the order for the validate_cache checks. --- bot/utils/redis_cache.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index a1196fcb5..895a12da4 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -129,20 +129,20 @@ class RedisCache: async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" - if self.bot is None: + if self._namespace is None: error_message = ( - "Critical error: RedisCache has no `Bot` instance. " - "This happens when the class RedisCache was created in doesn't " - "have a Bot instance. Please make sure that you're instantiating " - "the RedisCache inside a class that has a Bot instance attribute." + "Critical error: RedisCache has no namespace. " + "This object must be initialized as a class attribute." ) log.error(error_message) raise RuntimeError(error_message) - if self._namespace is None: + if self.bot is None: error_message = ( - "Critical error: RedisCache has no namespace. " - "This object must be initialized as a class attribute." + "Critical error: RedisCache has no `Bot` instance. " + "This happens when the class RedisCache was created in doesn't " + "have a Bot instance. Please make sure that you're instantiating " + "the RedisCache inside a class that has a Bot instance attribute." ) log.error(error_message) raise RuntimeError(error_message) -- cgit v1.2.3 From 1ab34dd48fce2de70db1fb2dd6da06f752460829 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Tue, 26 May 2020 19:06:57 +0200 Subject: Add a test for RuntimeErrors. This just tests that the various RuntimeErrors are reachable - that includes the error about not having a bot instance, the one about not being a class attribute, and the one about not having instantiated the class. This test addresses a concern raised by @MarkKoz in a review. I've decided not to test that actual contents of these RuntimeErrors, because I believe that sort of testing is a bit too brittle. It shouldn't break a test just to change the content of an error string. --- tests/bot/utils/test_redis_cache.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 7405487ed..1b05ae350 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -215,3 +215,25 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): await self.redis.set("stringthing", "stringthing") with self.assertRaises(TypeError): await self.redis.increment("stringthing") + + async def test_exceptions_raised(self): + """Testing that the various RuntimeErrors are reachable.""" + class MyCog: + cache = RedisCache() + + def __init__(self): + self.other_cache = RedisCache() + + cog = MyCog() + + # Raises "No Bot instance" + with self.assertRaises(RuntimeError): + await cog.cache.get("john") + + # Raises "RedisCache has no namespace" + with self.assertRaises(RuntimeError): + await cog.other_cache.get("was") + + # Raises "You must access the RedisCache instance through the cog instance" + with self.assertRaises(RuntimeError): + await MyCog.cache.get("afraid") -- cgit v1.2.3 From 63a922e629af76692c4af3902a33942b13f784b6 Mon Sep 17 00:00:00 2001 From: Dennis Pham Date: Tue, 26 May 2020 17:11:49 -0400 Subject: Add /r/FlutterDev to the guild invite whitelist --- config-default.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config-default.yml b/config-default.yml index c7d25894c..7edfb131f 100644 --- a/config-default.yml +++ b/config-default.yml @@ -290,6 +290,7 @@ filter: - 81384788765712384 # Discord API - 613425648685547541 # Discord Developers - 185590609631903755 # Blender Hub + - 420324994703163402 # /r/FlutterDev domain_blacklist: - pornhub.com -- cgit v1.2.3 From d9190d997538f49c0a1b53d63a15bada3c89297f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 07:32:16 +0200 Subject: Refactor the in_whitelist deco to a check. We're moving the actual predicate into the `utils.checks` folder, just like we're doing with most of the other decorators. This is to allow us the flexibility to use it as a pure check, not only as a decorator. This commit doesn't actually change any functionality, just moves it around. --- bot/decorators.py | 54 +++-------------------------- bot/utils/checks.py | 81 ++++++++++++++++++++++++++++++++++++++++++-- tests/bot/test_decorators.py | 4 +-- 3 files changed, 86 insertions(+), 53 deletions(-) diff --git a/bot/decorators.py b/bot/decorators.py index 306f0830c..1e77afe60 100644 --- a/bot/decorators.py +++ b/bot/decorators.py @@ -9,37 +9,20 @@ from weakref import WeakValueDictionary from discord import Colour, Embed, Member from discord.errors import NotFound from discord.ext import commands -from discord.ext.commands import CheckFailure, Cog, Context +from discord.ext.commands import Cog, Context from bot.constants import Channels, ERROR_REPLIES, RedirectOutput -from bot.utils.checks import with_role_check, without_role_check +from bot.utils.checks import in_whitelist_check, with_role_check, without_role_check log = logging.getLogger(__name__) -class InWhitelistCheckFailure(CheckFailure): - """Raised when the `in_whitelist` check fails.""" - - def __init__(self, redirect_channel: Optional[int]) -> None: - self.redirect_channel = redirect_channel - - if redirect_channel: - redirect_message = f" here. Please use the <#{redirect_channel}> channel instead" - else: - redirect_message = "" - - error_message = f"You are not allowed to use that command{redirect_message}." - - super().__init__(error_message) - - def in_whitelist( *, channels: Container[int] = (), categories: Container[int] = (), roles: Container[int] = (), redirect: Optional[int] = Channels.bot_commands, - ) -> Callable: """ Check if a command was issued in a whitelisted context. @@ -54,36 +37,9 @@ def in_whitelist( redirected to the `redirect` channel that was passed (default: #bot-commands) or simply told that they're not allowed to use this particular command (if `None` was passed). """ - if redirect and redirect not in channels: - # It does not make sense for the channel whitelist to not contain the redirection - # channel (if applicable). That's why we add the redirection channel to the `channels` - # container if it's not already in it. As we allow any container type to be passed, - # we first create a tuple in order to safely add the redirection channel. - # - # Note: It's possible for the redirect channel to be in a whitelisted category, but - # there's no easy way to check that and as a channel can easily be moved in and out of - # categories, it's probably not wise to rely on its category in any case. - channels = tuple(channels) + (redirect,) - def predicate(ctx: Context) -> bool: - """Check if a command was issued in a whitelisted context.""" - if channels and ctx.channel.id in channels: - log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted channel.") - return True - - # Only check the category id if we have a category whitelist and the channel has a `category_id` - if categories and hasattr(ctx.channel, "category_id") and ctx.channel.category_id in categories: - log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted category.") - return True - - # Only check the roles whitelist if we have one and ensure the author's roles attribute returns - # an iterable to prevent breakage in DM channels (for if we ever decide to enable commands there). - if roles and any(r.id in roles for r in getattr(ctx.author, "roles", ())): - log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they have a whitelisted role.") - return True - - log.trace(f"{ctx.author} may not use the `{ctx.command.name}` command within this context.") - raise InWhitelistCheckFailure(redirect) + """Check if command was issued in a whitelisted context.""" + return in_whitelist_check(ctx, channels, categories, roles, redirect) return commands.check(predicate) @@ -121,7 +77,7 @@ def locked() -> Callable: embed = Embed() embed.colour = Colour.red() - log.debug(f"User tried to invoke a locked command.") + log.debug("User tried to invoke a locked command.") embed.description = ( "You're already using this command. Please wait until it is done before you use it again." ) diff --git a/bot/utils/checks.py b/bot/utils/checks.py index db56c347c..63568b29e 100644 --- a/bot/utils/checks.py +++ b/bot/utils/checks.py @@ -1,12 +1,89 @@ import datetime import logging -from typing import Callable, Iterable +from typing import Callable, Container, Iterable, Optional -from discord.ext.commands import BucketType, Cog, Command, CommandOnCooldown, Context, Cooldown, CooldownMapping +from discord.ext.commands import ( + BucketType, + CheckFailure, + Cog, + Command, + CommandOnCooldown, + Context, + Cooldown, + CooldownMapping, +) + +from bot import constants log = logging.getLogger(__name__) +class InWhitelistCheckFailure(CheckFailure): + """Raised when the `in_whitelist` check fails.""" + + def __init__(self, redirect_channel: Optional[int]) -> None: + self.redirect_channel = redirect_channel + + if redirect_channel: + redirect_message = f" here. Please use the <#{redirect_channel}> channel instead" + else: + redirect_message = "" + + error_message = f"You are not allowed to use that command{redirect_message}." + + super().__init__(error_message) + + +def in_whitelist_check( + ctx: Context, + channels: Container[int] = (), + categories: Container[int] = (), + roles: Container[int] = (), + redirect: Optional[int] = constants.Channels.bot_commands, +) -> bool: + """ + Check if a command was issued in a whitelisted context. + + The whitelists that can be provided are: + + - `channels`: a container with channel ids for whitelisted channels + - `categories`: a container with category ids for whitelisted categories + - `roles`: a container with with role ids for whitelisted roles + + If the command was invoked in a context that was not whitelisted, the member is either + redirected to the `redirect` channel that was passed (default: #bot-commands) or simply + told that they're not allowed to use this particular command (if `None` was passed). + """ + if redirect and redirect not in channels: + # It does not make sense for the channel whitelist to not contain the redirection + # channel (if applicable). That's why we add the redirection channel to the `channels` + # container if it's not already in it. As we allow any container type to be passed, + # we first create a tuple in order to safely add the redirection channel. + # + # Note: It's possible for the redirect channel to be in a whitelisted category, but + # there's no easy way to check that and as a channel can easily be moved in and out of + # categories, it's probably not wise to rely on its category in any case. + channels = tuple(channels) + (redirect,) + + if channels and ctx.channel.id in channels: + log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted channel.") + return True + + # Only check the category id if we have a category whitelist and the channel has a `category_id` + if categories and hasattr(ctx.channel, "category_id") and ctx.channel.category_id in categories: + log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted category.") + return True + + # Only check the roles whitelist if we have one and ensure the author's roles attribute returns + # an iterable to prevent breakage in DM channels (for if we ever decide to enable commands there). + if roles and any(r.id in roles for r in getattr(ctx.author, "roles", ())): + log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they have a whitelisted role.") + return True + + log.trace(f"{ctx.author} may not use the `{ctx.command.name}` command within this context.") + raise InWhitelistCheckFailure(redirect) + + def with_role_check(ctx: Context, *role_ids: int) -> bool: """Returns True if the user has any one of the roles in role_ids.""" if not ctx.guild: # Return False in a DM diff --git a/tests/bot/test_decorators.py b/tests/bot/test_decorators.py index a17dd3e16..3d450caa0 100644 --- a/tests/bot/test_decorators.py +++ b/tests/bot/test_decorators.py @@ -3,10 +3,10 @@ import unittest import unittest.mock from bot import constants -from bot.decorators import InWhitelistCheckFailure, in_whitelist +from bot.decorators import in_whitelist +from bot.utils.checks import InWhitelistCheckFailure from tests import helpers - InWhitelistTestCase = collections.namedtuple("WhitelistedContextTestCase", ("kwargs", "ctx", "description")) -- cgit v1.2.3 From b51c206e51d0906f326da1e504162920cd2d443d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 07:35:56 +0200 Subject: Allow infraction management in modmail category --- bot/cogs/moderation/management.py | 20 ++++++++++++-------- bot/constants.py | 5 +---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index edfdfd9e2..56f7c390c 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -12,7 +12,7 @@ from bot.bot import Bot from bot.converters import Expiry, InfractionSearchQuery, allowed_strings, proxy_user from bot.pagination import LinePaginator from bot.utils import time -from bot.utils.checks import in_channel_check, with_role_check +from bot.utils.checks import in_whitelist_check, with_role_check from . import utils from .infractions import Infractions from .modlog import ModLog @@ -49,8 +49,8 @@ class ModManagement(commands.Cog): async def infraction_edit( self, ctx: Context, - infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], - duration: t.Union[Expiry, allowed_strings("p", "permanent"), None], + infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], # noqa: F821 + duration: t.Union[Expiry, allowed_strings("p", "permanent"), None], # noqa: F821 *, reason: str = None ) -> None: @@ -83,14 +83,14 @@ class ModManagement(commands.Cog): "actor__id": ctx.author.id, "ordering": "-inserted_at" } - infractions = await self.bot.api_client.get(f"bot/infractions", params=params) + infractions = await self.bot.api_client.get("bot/infractions", params=params) if infractions: old_infraction = infractions[0] infraction_id = old_infraction["id"] else: await ctx.send( - f":x: Couldn't find most recent infraction; you have never given an infraction." + ":x: Couldn't find most recent infraction; you have never given an infraction." ) return else: @@ -224,7 +224,7 @@ class ModManagement(commands.Cog): ) -> None: """Send a paginated embed of infractions for the specified user.""" if not infractions: - await ctx.send(f":warning: No infractions could be found for that query.") + await ctx.send(":warning: No infractions could be found for that query.") return lines = tuple( @@ -283,10 +283,14 @@ class ModManagement(commands.Cog): # This cannot be static (must have a __func__ attribute). def cog_check(self, ctx: Context) -> bool: - """Only allow moderators from moderator channels to invoke the commands in this cog.""" + """Only allow moderators inside moderator channels to invoke the commands in this cog.""" checks = [ with_role_check(ctx, *constants.MODERATION_ROLES), - in_channel_check(ctx, *constants.MODERATION_CHANNELS) + in_whitelist_check( + ctx, + channels=constants.MODERATION_CHANNELS, + categories=[constants.Categories.modmail], + ) ] return all(checks) diff --git a/bot/constants.py b/bot/constants.py index 39de2ee41..2ce5355be 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -612,13 +612,10 @@ PROJECT_ROOT = os.path.abspath(os.path.join(BOT_DIR, os.pardir)) MODERATION_ROLES = Guild.moderation_roles STAFF_ROLES = Guild.staff_roles -# Roles combinations +# Channel combinations STAFF_CHANNELS = Guild.staff_channels - -# Default Channel combinations MODERATION_CHANNELS = Guild.moderation_channels - # Bot replies NEGATIVE_REPLIES = [ "Noooooo!!", -- cgit v1.2.3 From d310f42080278b35914bf5785fa322b97627c45f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 07:42:08 +0200 Subject: Find + change all InWhitelistCheckFailure imports --- bot/cogs/error_handler.py | 6 +++--- bot/cogs/information.py | 4 ++-- bot/cogs/verification.py | 4 ++-- tests/bot/cogs/test_information.py | 3 +-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index 23d1eed82..5de961116 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -9,7 +9,7 @@ from bot.api import ResponseCodeError from bot.bot import Bot from bot.constants import Channels from bot.converters import TagNameConverter -from bot.decorators import InWhitelistCheckFailure +from bot.utils.checks import InWhitelistCheckFailure log = logging.getLogger(__name__) @@ -166,7 +166,7 @@ class ErrorHandler(Cog): await prepared_help_command self.bot.stats.incr("errors.missing_required_argument") elif isinstance(e, errors.TooManyArguments): - await ctx.send(f"Too many arguments provided.") + await ctx.send("Too many arguments provided.") await prepared_help_command self.bot.stats.incr("errors.too_many_arguments") elif isinstance(e, errors.BadArgument): @@ -206,7 +206,7 @@ class ErrorHandler(Cog): if isinstance(e, bot_missing_errors): ctx.bot.stats.incr("errors.bot_permission_error") await ctx.send( - f"Sorry, it looks like I don't have the permissions or roles I need to do that." + "Sorry, it looks like I don't have the permissions or roles I need to do that." ) elif isinstance(e, (InWhitelistCheckFailure, errors.NoPrivateMessage)): ctx.bot.stats.incr("errors.wrong_channel_or_dm_error") diff --git a/bot/cogs/information.py b/bot/cogs/information.py index ef2f308ca..f0eb3a1ea 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -12,9 +12,9 @@ from discord.utils import escape_markdown from bot import constants from bot.bot import Bot -from bot.decorators import InWhitelistCheckFailure, in_whitelist, with_role +from bot.decorators import in_whitelist, with_role from bot.pagination import LinePaginator -from bot.utils.checks import cooldown_with_role_bypass, with_role_check +from bot.utils.checks import InWhitelistCheckFailure, cooldown_with_role_bypass, with_role_check from bot.utils.time import time_since log = logging.getLogger(__name__) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 77e8b5706..99be3cdaa 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -9,8 +9,8 @@ from discord.ext.commands import Cog, Context, command from bot import constants from bot.bot import Bot from bot.cogs.moderation import ModLog -from bot.decorators import InWhitelistCheckFailure, in_whitelist, without_role -from bot.utils.checks import without_role_check +from bot.decorators import in_whitelist, without_role +from bot.utils.checks import InWhitelistCheckFailure, without_role_check log = logging.getLogger(__name__) diff --git a/tests/bot/cogs/test_information.py b/tests/bot/cogs/test_information.py index b5f928dd6..aca6b594f 100644 --- a/tests/bot/cogs/test_information.py +++ b/tests/bot/cogs/test_information.py @@ -7,10 +7,9 @@ import discord from bot import constants from bot.cogs import information -from bot.decorators import InWhitelistCheckFailure +from bot.utils.checks import InWhitelistCheckFailure from tests import helpers - COG_PATH = "bot.cogs.information.Information" -- cgit v1.2.3 From 75622622696beee8299c24e9ddbc36f5eb4f104f Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 07:46:04 +0200 Subject: No redirect for mod management. --- bot/cogs/moderation/management.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index 56f7c390c..c7c19e89d 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -290,6 +290,7 @@ class ModManagement(commands.Cog): ctx, channels=constants.MODERATION_CHANNELS, categories=[constants.Categories.modmail], + redirect=None, ) ] return all(checks) -- cgit v1.2.3 From c3cbc842dce1c26f09d774b7ca85eff613765480 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 07:51:20 +0200 Subject: Allow some commands to fail checks silently. For example, we don't want the mod commands to produce any kind of error message when run by ordinary users in regular channels - these should have the perception of being invisible and unavailable. --- bot/cogs/moderation/management.py | 1 + bot/decorators.py | 3 ++- bot/utils/checks.py | 7 ++++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index c7c19e89d..c39c7f3bc 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -291,6 +291,7 @@ class ModManagement(commands.Cog): channels=constants.MODERATION_CHANNELS, categories=[constants.Categories.modmail], redirect=None, + fail_silently=True, ) ] return all(checks) diff --git a/bot/decorators.py b/bot/decorators.py index 1e77afe60..500197c89 100644 --- a/bot/decorators.py +++ b/bot/decorators.py @@ -23,6 +23,7 @@ def in_whitelist( categories: Container[int] = (), roles: Container[int] = (), redirect: Optional[int] = Channels.bot_commands, + fail_silently: bool = False, ) -> Callable: """ Check if a command was issued in a whitelisted context. @@ -39,7 +40,7 @@ def in_whitelist( """ def predicate(ctx: Context) -> bool: """Check if command was issued in a whitelisted context.""" - return in_whitelist_check(ctx, channels, categories, roles, redirect) + return in_whitelist_check(ctx, channels, categories, roles, redirect, fail_silently) return commands.check(predicate) diff --git a/bot/utils/checks.py b/bot/utils/checks.py index 63568b29e..d5ebe4ec9 100644 --- a/bot/utils/checks.py +++ b/bot/utils/checks.py @@ -40,6 +40,7 @@ def in_whitelist_check( categories: Container[int] = (), roles: Container[int] = (), redirect: Optional[int] = constants.Channels.bot_commands, + fail_silently: bool = False, ) -> bool: """ Check if a command was issued in a whitelisted context. @@ -81,7 +82,11 @@ def in_whitelist_check( return True log.trace(f"{ctx.author} may not use the `{ctx.command.name}` command within this context.") - raise InWhitelistCheckFailure(redirect) + + # Some commands are secret, and should produce no feedback at all. + if not fail_silently: + raise InWhitelistCheckFailure(redirect) + return False def with_role_check(ctx: Context, *role_ids: int) -> bool: -- cgit v1.2.3 From 35a1de37307b1745c061e490be4e96c8467de212 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 12:21:58 +0200 Subject: Clear cache in asyncSetUp instead of tests. --- tests/bot/utils/test_redis_cache.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 1b05ae350..900a6d035 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -15,6 +15,7 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): """Sets up the objects that only have to be initialized once.""" self.bot = helpers.MockBot() self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() + await self.redis.clear() def test_class_attribute_namespace(self): """Test that RedisDict creates a namespace automatically for class attributes.""" @@ -76,8 +77,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_items(self): """Test that the RedisDict can be iterated.""" - await self.redis.clear() - # Set up our test cases in the Redis cache test_cases = [ ('favorite_turtle', 'Donatello'), @@ -101,7 +100,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_length(self): """Test that we can get the correct .length from the RedisDict.""" - await self.redis.clear() await self.redis.set('one', 1) await self.redis.set('two', 2) await self.redis.set('three', 3) @@ -119,7 +117,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_clear(self): """Test that the .clear method removes the entire hash.""" - await self.redis.clear() await self.redis.set('teddy', 'with me') await self.redis.set('in my dreams', 'you have a weird hat') self.assertEqual(await self.redis.length(), 2) @@ -129,7 +126,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_pop(self): """Test that we can .pop an item from the RedisDict.""" - await self.redis.clear() await self.redis.set('john', 'was afraid') self.assertEqual(await self.redis.pop('john'), 'was afraid') @@ -138,7 +134,6 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_update(self): """Test that we can .update the RedisDict with multiple items.""" - await self.redis.clear() await self.redis.set("reckfried", "lona") await self.redis.set("bel air", "prince") await self.redis.update({ -- cgit v1.2.3 From b18930735e05e09ba615cb54fe1dbdfd41bb0f81 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 12:51:40 +0200 Subject: Refactor .increment and add lock test. The way we were doing the asyncio.Lock() stuff for increment was slightly problematic. @aeros has adviced us that it's better to just initialize the lock as None in __init__, and then initialize it inside the first coroutine that uses it instead. This ensures that the correct loop gets attached to the lock, so we don't end up getting errors like this one: RuntimeError: got Future attached to a different loop This happens because the lock and the actual calling coroutines aren't on the same loop. When creating a new test, test_increment_lock, we discovered that we needed a small refactor here and also in the test class to make this new test pass. So, now we're creating a DummyCog for every test method, and this will ensure the loop streams never cross. Cause we all know we must never cross the streams. --- bot/utils/redis_cache.py | 11 ++- tests/bot/utils/test_redis_cache.py | 163 ++++++++++++++++++++++-------------- 2 files changed, 109 insertions(+), 65 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 895a12da4..33e5d5852 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -81,7 +81,7 @@ class RedisCache: """Initialize the RedisCache.""" self._namespace = None self.bot = None - self._increment_lock = asyncio.Lock() + self._increment_lock = None def _set_namespace(self, namespace: str) -> None: """Try to set the namespace, but do not permit collisions.""" @@ -345,6 +345,15 @@ class RedisCache: """ log.trace(f"Attempting to increment/decrement the value with the key {key} by {amount}.") + # We initialize the lock here, because we need to ensure we get it + # running on the same loop as the calling coroutine. + # + # If we initialized the lock in the __init__, the loop that the coroutine this method + # would be called from might not exist yet, and so the lock would be on a different + # loop, which would raise RuntimeErrors. + if self._increment_lock is None: + self._increment_lock = asyncio.Lock() + # Since this has several API calls, we need a lock to prevent race conditions async with self._increment_lock: value = await self.get(key) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 900a6d035..efd168dac 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -1,3 +1,4 @@ +import asyncio import unittest import fakeredis.aioredis @@ -9,17 +10,30 @@ from tests import helpers class RedisCacheTests(unittest.IsolatedAsyncioTestCase): """Tests the RedisCache class from utils.redis_dict.py.""" - redis = RedisCache() - async def asyncSetUp(self): # noqa: N802 """Sets up the objects that only have to be initialized once.""" self.bot = helpers.MockBot() self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() - await self.redis.clear() + + # Okay, so this is necessary so that we can create a clean new + # class for every test method, and we want that because it will + # ensure we get a fresh loop, which is necessary for test_increment_lock + # to be able to pass. + class DummyCog: + """A dummy cog, for dummies.""" + + redis = RedisCache() + + def __init__(self, bot: helpers.MockBot): + self.bot = bot + + self.cog = DummyCog(self.bot) + + await self.cog.redis.clear() def test_class_attribute_namespace(self): """Test that RedisDict creates a namespace automatically for class attributes.""" - self.assertEqual(self.redis._namespace, "RedisCacheTests.redis") + self.assertEqual(self.cog.redis._namespace, "DummyCog.redis") async def test_class_attribute_required(self): """Test that errors are raised when not assigned as a class attribute.""" @@ -31,9 +45,13 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): def test_namespace_collision(self): """Test that we prevent colliding namespaces.""" - bad_cache = RedisCache() - bad_cache._set_namespace("RedisCacheTests.redis") - self.assertEqual(bad_cache._namespace, "RedisCacheTests.redis_") + bob_cache_1 = RedisCache() + bob_cache_1._set_namespace("BobRoss") + self.assertEqual(bob_cache_1._namespace, "BobRoss") + + bob_cache_2 = RedisCache() + bob_cache_2._set_namespace("BobRoss") + self.assertEqual(bob_cache_2._namespace, "BobRoss_") async def test_set_get_item(self): """Test that users can set and get items from the RedisDict.""" @@ -45,35 +63,35 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): # Test that we can get and set different types. for test in test_cases: - await self.redis.set(*test) - self.assertEqual(await self.redis.get(test[0]), test[1]) + await self.cog.redis.set(*test) + self.assertEqual(await self.cog.redis.get(test[0]), test[1]) # Test that .get allows a default value - self.assertEqual(await self.redis.get('favorite_nothing', "bearclaw"), "bearclaw") + self.assertEqual(await self.cog.redis.get('favorite_nothing', "bearclaw"), "bearclaw") async def test_set_item_type(self): """Test that .set rejects keys and values that are not strings, ints or floats.""" fruits = ["lemon", "melon", "apple"] with self.assertRaises(TypeError): - await self.redis.set(fruits, "nice") + await self.cog.redis.set(fruits, "nice") async def test_delete_item(self): """Test that .delete allows us to delete stuff from the RedisCache.""" # Add an item and verify that it gets added - await self.redis.set("internet", "firetruck") - self.assertEqual(await self.redis.get("internet"), "firetruck") + await self.cog.redis.set("internet", "firetruck") + self.assertEqual(await self.cog.redis.get("internet"), "firetruck") # Delete that item and verify that it gets deleted - await self.redis.delete("internet") - self.assertIs(await self.redis.get("internet"), None) + await self.cog.redis.delete("internet") + self.assertIs(await self.cog.redis.get("internet"), None) async def test_contains(self): """Test that we can check membership with .contains.""" - await self.redis.set('favorite_country', "Burkina Faso") + await self.cog.redis.set('favorite_country', "Burkina Faso") - self.assertIs(await self.redis.contains('favorite_country'), True) - self.assertIs(await self.redis.contains('favorite_dentist'), False) + self.assertIs(await self.cog.redis.contains('favorite_country'), True) + self.assertIs(await self.cog.redis.contains('favorite_dentist'), False) async def test_items(self): """Test that the RedisDict can be iterated.""" @@ -84,10 +102,10 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): ('third_favorite_turtle', 'Raphael'), ] for key, value in test_cases: - await self.redis.set(key, value) + await self.cog.redis.set(key, value) # Consume the AsyncIterator into a regular list, easier to compare that way. - redis_items = [item for item in await self.redis.items()] + redis_items = [item for item in await self.cog.redis.items()] # These sequences are probably in the same order now, but probably # isn't good enough for tests. Let's not rely on .hgetall always @@ -100,43 +118,43 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): async def test_length(self): """Test that we can get the correct .length from the RedisDict.""" - await self.redis.set('one', 1) - await self.redis.set('two', 2) - await self.redis.set('three', 3) - self.assertEqual(await self.redis.length(), 3) + await self.cog.redis.set('one', 1) + await self.cog.redis.set('two', 2) + await self.cog.redis.set('three', 3) + self.assertEqual(await self.cog.redis.length(), 3) - await self.redis.set('four', 4) - self.assertEqual(await self.redis.length(), 4) + await self.cog.redis.set('four', 4) + self.assertEqual(await self.cog.redis.length(), 4) async def test_to_dict(self): """Test that the .to_dict method returns a workable dictionary copy.""" - copy = await self.redis.to_dict() - local_copy = {key: value for key, value in await self.redis.items()} + copy = await self.cog.redis.to_dict() + local_copy = {key: value for key, value in await self.cog.redis.items()} self.assertIs(type(copy), dict) self.assertDictEqual(copy, local_copy) async def test_clear(self): """Test that the .clear method removes the entire hash.""" - await self.redis.set('teddy', 'with me') - await self.redis.set('in my dreams', 'you have a weird hat') - self.assertEqual(await self.redis.length(), 2) + await self.cog.redis.set('teddy', 'with me') + await self.cog.redis.set('in my dreams', 'you have a weird hat') + self.assertEqual(await self.cog.redis.length(), 2) - await self.redis.clear() - self.assertEqual(await self.redis.length(), 0) + await self.cog.redis.clear() + self.assertEqual(await self.cog.redis.length(), 0) async def test_pop(self): """Test that we can .pop an item from the RedisDict.""" - await self.redis.set('john', 'was afraid') + await self.cog.redis.set('john', 'was afraid') - self.assertEqual(await self.redis.pop('john'), 'was afraid') - self.assertEqual(await self.redis.pop('pete', 'breakneck'), 'breakneck') - self.assertEqual(await self.redis.length(), 0) + self.assertEqual(await self.cog.redis.pop('john'), 'was afraid') + self.assertEqual(await self.cog.redis.pop('pete', 'breakneck'), 'breakneck') + self.assertEqual(await self.cog.redis.length(), 0) async def test_update(self): """Test that we can .update the RedisDict with multiple items.""" - await self.redis.set("reckfried", "lona") - await self.redis.set("bel air", "prince") - await self.redis.update({ + await self.cog.redis.set("reckfried", "lona") + await self.cog.redis.set("bel air", "prince") + await self.cog.redis.update({ "reckfried": "jona", "mega": "hungry, though", }) @@ -146,7 +164,7 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): "bel air": "prince", "mega": "hungry, though", } - self.assertDictEqual(await self.redis.to_dict(), result) + self.assertDictEqual(await self.cog.redis.to_dict(), result) def test_typestring_conversion(self): """Test the typestring-related helper functions.""" @@ -158,58 +176,75 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): # Test conversion to typestring for _input, expected in conversion_tests: - self.assertEqual(self.redis._to_typestring(_input), expected) + self.assertEqual(self.cog.redis._to_typestring(_input), expected) # Test conversion from typestrings for _input, expected in conversion_tests: - self.assertEqual(self.redis._from_typestring(expected), _input) + self.assertEqual(self.cog.redis._from_typestring(expected), _input) # Test that exceptions are raised on invalid input with self.assertRaises(TypeError): - self.redis._to_typestring(["internet"]) - self.redis._from_typestring("o|firedog") + self.cog.redis._to_typestring(["internet"]) + self.cog.redis._from_typestring("o|firedog") async def test_increment_decrement(self): """Test .increment and .decrement methods.""" - await self.redis.set("entropic", 5) - await self.redis.set("disentropic", 12.5) + await self.cog.redis.set("entropic", 5) + await self.cog.redis.set("disentropic", 12.5) # Test default increment - await self.redis.increment("entropic") - self.assertEqual(await self.redis.get("entropic"), 6) + await self.cog.redis.increment("entropic") + self.assertEqual(await self.cog.redis.get("entropic"), 6) # Test default decrement - await self.redis.decrement("entropic") - self.assertEqual(await self.redis.get("entropic"), 5) + await self.cog.redis.decrement("entropic") + self.assertEqual(await self.cog.redis.get("entropic"), 5) # Test float increment with float - await self.redis.increment("disentropic", 2.0) - self.assertEqual(await self.redis.get("disentropic"), 14.5) + await self.cog.redis.increment("disentropic", 2.0) + self.assertEqual(await self.cog.redis.get("disentropic"), 14.5) # Test float increment with int - await self.redis.increment("disentropic", 2) - self.assertEqual(await self.redis.get("disentropic"), 16.5) + await self.cog.redis.increment("disentropic", 2) + self.assertEqual(await self.cog.redis.get("disentropic"), 16.5) # Test negative increments, because why not. - await self.redis.increment("entropic", -5) - self.assertEqual(await self.redis.get("entropic"), 0) + await self.cog.redis.increment("entropic", -5) + self.assertEqual(await self.cog.redis.get("entropic"), 0) # Negative decrements? Sure. - await self.redis.decrement("entropic", -5) - self.assertEqual(await self.redis.get("entropic"), 5) + await self.cog.redis.decrement("entropic", -5) + self.assertEqual(await self.cog.redis.get("entropic"), 5) # What about if we use a negative float to decrement an int? # This should convert the type into a float. - await self.redis.decrement("entropic", -2.5) - self.assertEqual(await self.redis.get("entropic"), 7.5) + await self.cog.redis.decrement("entropic", -2.5) + self.assertEqual(await self.cog.redis.get("entropic"), 7.5) # Let's test that they raise the right errors with self.assertRaises(KeyError): - await self.redis.increment("doesn't_exist!") + await self.cog.redis.increment("doesn't_exist!") - await self.redis.set("stringthing", "stringthing") + await self.cog.redis.set("stringthing", "stringthing") with self.assertRaises(TypeError): - await self.redis.increment("stringthing") + await self.cog.redis.increment("stringthing") + + async def test_increment_lock(self): + """Test that we can't produce a race condition in .increment.""" + await self.cog.redis.set("test_key", 0) + tasks = [] + + # Increment this a lot in different tasks + for _ in range(100): + task = asyncio.create_task( + self.cog.redis.increment("test_key", 1) + ) + tasks.append(task) + await asyncio.gather(*tasks) + + # Confirm that the value has been incremented the exact right number of times. + value = await self.cog.redis.get("test_key") + self.assertEqual(value, 100) async def test_exceptions_raised(self): """Testing that the various RuntimeErrors are reachable.""" -- cgit v1.2.3 From db0a384e91a463ff9668ab4f9ea5268aa332ab2d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 13:27:34 +0200 Subject: Remove the now deprecated in_channel_check. This check was no longer being used anywhere, having been replaced by in_whitelist_check. --- bot/utils/checks.py | 8 -------- tests/bot/utils/test_checks.py | 8 -------- 2 files changed, 16 deletions(-) diff --git a/bot/utils/checks.py b/bot/utils/checks.py index d5ebe4ec9..f0ef36302 100644 --- a/bot/utils/checks.py +++ b/bot/utils/checks.py @@ -120,14 +120,6 @@ def without_role_check(ctx: Context, *role_ids: int) -> bool: return check -def in_channel_check(ctx: Context, *channel_ids: int) -> bool: - """Checks if the command was executed inside the list of specified channels.""" - check = ctx.channel.id in channel_ids - log.trace(f"{ctx.author} tried to call the '{ctx.command.name}' command. " - f"The result of the in_channel check was {check}.") - return check - - def cooldown_with_role_bypass(rate: int, per: float, type: BucketType = BucketType.default, *, bypass_roles: Iterable[int]) -> Callable: """ diff --git a/tests/bot/utils/test_checks.py b/tests/bot/utils/test_checks.py index 9610771e5..d572b6299 100644 --- a/tests/bot/utils/test_checks.py +++ b/tests/bot/utils/test_checks.py @@ -41,11 +41,3 @@ class ChecksTests(unittest.TestCase): role_id = 42 self.ctx.author.roles.append(MockRole(id=role_id)) self.assertTrue(checks.without_role_check(self.ctx, role_id + 10)) - - def test_in_channel_check_for_correct_channel(self): - self.ctx.channel.id = 42 - self.assertTrue(checks.in_channel_check(self.ctx, *[42])) - - def test_in_channel_check_for_incorrect_channel(self): - self.ctx.channel.id = 42 + 10 - self.assertFalse(checks.in_channel_check(self.ctx, *[42])) -- cgit v1.2.3 From 876fae1856f1ad876d74036899739115fd8b86c3 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 13:39:32 +0200 Subject: Add some tests for `in_whitelist_check`. --- tests/bot/utils/test_checks.py | 48 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/bot/utils/test_checks.py b/tests/bot/utils/test_checks.py index d572b6299..de72e5748 100644 --- a/tests/bot/utils/test_checks.py +++ b/tests/bot/utils/test_checks.py @@ -1,6 +1,8 @@ import unittest +from unittest.mock import MagicMock from bot.utils import checks +from bot.utils.checks import InWhitelistCheckFailure from tests.helpers import MockContext, MockRole @@ -41,3 +43,49 @@ class ChecksTests(unittest.TestCase): role_id = 42 self.ctx.author.roles.append(MockRole(id=role_id)) self.assertTrue(checks.without_role_check(self.ctx, role_id + 10)) + + def test_in_whitelist_check_correct_channel(self): + """`in_whitelist_check` returns `True` if `Context.channel.id` is in the channel list.""" + channel_id = 3 + self.ctx.channel.id = channel_id + self.assertTrue(checks.in_whitelist_check(self.ctx, [channel_id])) + + def test_in_whitelist_check_incorrect_channel(self): + """`in_whitelist_check` raises InWhitelistCheckFailure if there's no channel match.""" + self.ctx.channel.id = 3 + with self.assertRaises(InWhitelistCheckFailure): + checks.in_whitelist_check(self.ctx, [4]) + + def test_in_whitelist_check_correct_category(self): + """`in_whitelist_check` returns `True` if `Context.channel.category_id` is in the category list.""" + category_id = 3 + self.ctx.channel.category_id = category_id + self.assertTrue(checks.in_whitelist_check(self.ctx, categories=[category_id])) + + def test_in_whitelist_check_incorrect_category(self): + """`in_whitelist_check` raises InWhitelistCheckFailure if there's no category match.""" + self.ctx.channel.category_id = 3 + with self.assertRaises(InWhitelistCheckFailure): + checks.in_whitelist_check(self.ctx, categories=[4]) + + def test_in_whitelist_check_correct_role(self): + """`in_whitelist_check` returns `True` if any of the `Context.author.roles` are in the roles list.""" + self.ctx.author.roles = (MagicMock(id=1), MagicMock(id=2)) + self.assertTrue(checks.in_whitelist_check(self.ctx, roles=[2, 6])) + + def test_in_whitelist_check_incorrect_role(self): + """`in_whitelist_check` raises InWhitelistCheckFailure if there's no role match.""" + self.ctx.author.roles = (MagicMock(id=1), MagicMock(id=2)) + with self.assertRaises(InWhitelistCheckFailure): + checks.in_whitelist_check(self.ctx, roles=[4]) + + def test_in_whitelist_check_fail_silently(self): + """`in_whitelist_check` test no exception raised if `fail_silently` is `True`""" + self.assertFalse(checks.in_whitelist_check(self.ctx, roles=[2, 6], fail_silently=True)) + + def test_in_whitelist_check_complex(self): + """`in_whitelist_check` test with multiple parameters""" + self.ctx.author.roles = (MagicMock(id=1), MagicMock(id=2)) + self.ctx.channel.category_id = 3 + self.ctx.channel.id = 5 + self.assertTrue(checks.in_whitelist_check(self.ctx, channels=[1], categories=[8], roles=[2])) -- cgit v1.2.3 From 4db313e9a7899666f1597094b0d88447c7b64311 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 20:15:19 +0200 Subject: Floats are no longer permitted as RedisCache keys. Also added a test for this. This is the DRYest approach I could find. It's a little ugly, but I think it's probably good enough. --- bot/utils/redis_cache.py | 116 ++++++++++++++++++++++++------------ tests/bot/utils/test_redis_cache.py | 13 ++-- 2 files changed, 86 insertions(+), 43 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 33e5d5852..afd37f8f8 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -2,26 +2,42 @@ from __future__ import annotations import asyncio import logging -from typing import Any, Dict, ItemsView, Optional, Union +import typing +from typing import Any, Dict, ItemsView, Optional, Tuple, Union from bot.bot import Bot log = logging.getLogger(__name__) -RedisType = Union[str, int, float] -TYPESTRING_PREFIXES = ( +# Type aliases +RedisKeyType = Union[str, int] +RedisValueType = Union[str, int, float] + +# Prefix tuples +PrefixTuple = Tuple[Tuple[str, Any]] +TYPESTRING_VALUE_PREFIXES = ( ("f|", float), ("i|", int), ("s|", str), ) +TYPESTRING_KEY_PREFIXES = ( + ("i|", int), + ("s|", str), +) # Makes a nice list like "float, int, and str" -NICE_TYPE_LIST = ", ".join(str(_type.__name__) for _, _type in TYPESTRING_PREFIXES) -NICE_TYPE_LIST = ", and ".join(NICE_TYPE_LIST.rsplit(", ", 1)) +NICE_VALUE_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisValueType)) +NICE_VALUE_TYPE_LIST = ", and ".join(NICE_VALUE_TYPE_LIST.rsplit(", ", 1)) + +NICE_KEY_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisKeyType)) +NICE_KEY_TYPE_LIST = ", and ".join(NICE_KEY_TYPE_LIST.rsplit(", ", 1)) # Makes a list like "'f|', 'i|', and 's|'" -NICE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_PREFIXES]) -NICE_PREFIX_LIST = ", and ".join(NICE_PREFIX_LIST.rsplit(", ", 1)) +NICE_VALUE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_VALUE_PREFIXES]) +NICE_VALUE_PREFIX_LIST = ", and ".join(NICE_VALUE_PREFIX_LIST.rsplit(", ", 1)) + +NICE_KEY_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_KEY_PREFIXES]) +NICE_KEY_PREFIX_LIST = ", and ".join(NICE_KEY_PREFIX_LIST.rsplit(", ", 1)) class RedisCache: @@ -99,33 +115,57 @@ class RedisCache: self._namespace = namespace @staticmethod - def _to_typestring(value: RedisType) -> str: + def _to_typestring( + key_or_value: Union[RedisKeyType, RedisValueType], + prefixes: PrefixTuple, + nice_type_list: str + ) -> str: """Turn a valid Redis type into a typestring.""" - for prefix, _type in TYPESTRING_PREFIXES: - if isinstance(value, _type): - return f"{prefix}{value}" - raise TypeError(f"RedisCache._from_typestring only supports the types {NICE_TYPE_LIST}.") + for prefix, _type in prefixes: + if isinstance(key_or_value, _type): + return f"{prefix}{key_or_value}" + raise TypeError(f"RedisCache._from_typestring only supports the types {nice_type_list}.") @staticmethod - def _from_typestring(value: Union[bytes, str]) -> RedisType: - """Turn a typestring into a valid Redis type.""" + def _from_typestring( + key_or_value: Union[bytes, str], + prefixes: PrefixTuple, + nice_prefix_list: str, + ) -> Union[RedisKeyType, RedisValueType]: + """Deserialize a typestring into a valid Redis type.""" # Stuff that comes out of Redis will be bytestrings, so let's decode those. - if isinstance(value, bytes): - value = value.decode('utf-8') + if isinstance(key_or_value, bytes): + key_or_value = key_or_value.decode('utf-8') # Now we convert our unicode string back into the type it originally was. - for prefix, _type in TYPESTRING_PREFIXES: - if value.startswith(prefix): - return _type(value[len(prefix):]) - raise TypeError(f"RedisCache._to_typestring only supports the prefixes {NICE_PREFIX_LIST}.") + for prefix, _type in prefixes: + if key_or_value.startswith(prefix): + return _type(key_or_value[len(prefix):]) + raise TypeError(f"RedisCache._to_typestring only supports the prefixes {nice_prefix_list}.") + + def _key_to_typestring(self, key: RedisKeyType) -> str: + """Serialize a RedisKeyType object into a typestring.""" + return self._to_typestring(key, TYPESTRING_KEY_PREFIXES, NICE_KEY_TYPE_LIST) + + def _value_to_typestring(self, value: RedisValueType) -> str: + """Serialize a RedisValueType object into a typestring.""" + return self._to_typestring(value, TYPESTRING_VALUE_PREFIXES, NICE_VALUE_TYPE_LIST) + + def _key_from_typestring(self, key: Union[bytes, str]) -> RedisKeyType: + """Deserialize a RedisKeyType object from a typestring.""" + return self._from_typestring(key, TYPESTRING_KEY_PREFIXES, NICE_KEY_PREFIX_LIST) + + def _value_from_typestring(self, value: Union[bytes, str]) -> RedisValueType: + """Deserialize a RedisValueType object from a typestring.""" + return self._from_typestring(value, TYPESTRING_VALUE_PREFIXES, NICE_VALUE_PREFIX_LIST) def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" - return {self._from_typestring(key): self._from_typestring(value) for key, value in dictionary.items()} + return {self._key_from_typestring(key): self._value_from_typestring(value) for key, value in dictionary.items()} def _dict_to_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into typestrings.""" - return {self._to_typestring(key): self._to_typestring(value) for key, value in dictionary.items()} + return {self._key_to_typestring(key): self._value_to_typestring(value) for key, value in dictionary.items()} async def _validate_cache(self) -> None: """Validate that the RedisCache is ready to be used.""" @@ -209,21 +249,21 @@ class RedisCache: """Return a beautiful representation of this object instance.""" return f"RedisCache(namespace={self._namespace!r})" - async def set(self, key: RedisType, value: RedisType) -> None: + async def set(self, key: RedisKeyType, value: RedisValueType) -> None: """Store an item in the Redis cache.""" await self._validate_cache() # Convert to a typestring and then set it - key = self._to_typestring(key) - value = self._to_typestring(value) + key = self._key_to_typestring(key) + value = self._value_to_typestring(value) log.trace(f"Setting {key} to {value}.") await self._redis.hset(self._namespace, key, value) - async def get(self, key: RedisType, default: Optional[RedisType] = None) -> Optional[RedisType]: + async def get(self, key: RedisKeyType, default: Optional[RedisValueType] = None) -> Optional[RedisValueType]: """Get an item from the Redis cache.""" await self._validate_cache() - key = self._to_typestring(key) + key = self._key_to_typestring(key) log.trace(f"Attempting to retrieve {key}.") value = await self._redis.hget(self._namespace, key) @@ -232,11 +272,11 @@ class RedisCache: log.trace(f"Value not found, returning default value {default}") return default else: - value = self._from_typestring(value) + value = self._value_from_typestring(value) log.trace(f"Value found, returning value {value}") return value - async def delete(self, key: RedisType) -> None: + async def delete(self, key: RedisKeyType) -> None: """ Delete an item from the Redis cache. @@ -245,19 +285,19 @@ class RedisCache: See https://redis.io/commands/hdel for more info on how this works. """ await self._validate_cache() - key = self._to_typestring(key) + key = self._key_to_typestring(key) log.trace(f"Attempting to delete {key}.") return await self._redis.hdel(self._namespace, key) - async def contains(self, key: RedisType) -> bool: + async def contains(self, key: RedisKeyType) -> bool: """ Check if a key exists in the Redis cache. Return True if the key exists, otherwise False. """ await self._validate_cache() - key = self._to_typestring(key) + key = self._key_to_typestring(key) exists = await self._redis.hexists(self._namespace, key) log.trace(f"Testing if {key} exists in the RedisCache - Result is {exists}") @@ -304,7 +344,7 @@ class RedisCache: log.trace("Clearing the cache of all key/value pairs.") await self._redis.delete(self._namespace) - async def pop(self, key: RedisType, default: Optional[RedisType] = None) -> RedisType: + async def pop(self, key: RedisKeyType, default: Optional[RedisValueType] = None) -> RedisValueType: """Get the item, remove it from the cache, and provide a default if not found.""" log.trace(f"Attempting to pop {key}.") value = await self.get(key, default) @@ -317,7 +357,7 @@ class RedisCache: return value - async def update(self, items: Dict[RedisType, RedisType]) -> None: + async def update(self, items: Dict[RedisKeyType, RedisValueType]) -> None: """ Update the Redis cache with multiple values. @@ -326,14 +366,14 @@ class RedisCache: do not exist in the RedisCache, they are created. If they do exist, the values are updated with the new ones from `items`. - Please note that both the keys and the values in the `items` dictionary - must consist of valid RedisTypes - ints, floats, or strings. + Please note that keys and the values in the `items` dictionary + must consist of valid RedisKeyTypes and RedisValueTypes. """ await self._validate_cache() log.trace(f"Updating the cache with the following items:\n{items}") await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) - async def increment(self, key: RedisType, amount: Optional[int, float] = 1) -> None: + async def increment(self, key: RedisKeyType, amount: Optional[int, float] = 1) -> None: """ Increment the value by `amount`. @@ -373,7 +413,7 @@ class RedisCache: log.error(error_message) raise TypeError(error_message) - async def decrement(self, key: RedisType, amount: Optional[int, float] = 1) -> None: + async def decrement(self, key: RedisKeyType, amount: Optional[int, float] = 1) -> None: """ Decrement the value by `amount`. diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index efd168dac..4f95dff03 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -70,12 +70,15 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(await self.cog.redis.get('favorite_nothing', "bearclaw"), "bearclaw") async def test_set_item_type(self): - """Test that .set rejects keys and values that are not strings, ints or floats.""" + """Test that .set rejects keys and values that are not permitted.""" fruits = ["lemon", "melon", "apple"] with self.assertRaises(TypeError): await self.cog.redis.set(fruits, "nice") + with self.assertRaises(TypeError): + await self.cog.redis.set(4.23, "nice") + async def test_delete_item(self): """Test that .delete allows us to delete stuff from the RedisCache.""" # Add an item and verify that it gets added @@ -176,16 +179,16 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): # Test conversion to typestring for _input, expected in conversion_tests: - self.assertEqual(self.cog.redis._to_typestring(_input), expected) + self.assertEqual(self.cog.redis._value_to_typestring(_input), expected) # Test conversion from typestrings for _input, expected in conversion_tests: - self.assertEqual(self.cog.redis._from_typestring(expected), _input) + self.assertEqual(self.cog.redis._value_from_typestring(expected), _input) # Test that exceptions are raised on invalid input with self.assertRaises(TypeError): - self.cog.redis._to_typestring(["internet"]) - self.cog.redis._from_typestring("o|firedog") + self.cog.redis._value_to_typestring(["internet"]) + self.cog.redis._value_from_typestring("o|firedog") async def test_increment_decrement(self): """Test .increment and .decrement methods.""" -- cgit v1.2.3 From b6093bf7df00be1ed04a51119a65dbdd74ae0e58 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 21:01:24 +0200 Subject: Refactor typestring converters to partialmethods. We're using functools.partialmethod to make the code a little cleaner and more readable here. Read more about them here: https://docs.python.org/3/library/functools.html#functools.partial https://docs.python.org/3/library/functools.html#functools.partialmethod --- bot/utils/redis_cache.py | 54 +++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index afd37f8f8..dd24b83e8 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import logging import typing +from functools import partialmethod from typing import Any, Dict, ItemsView, Optional, Tuple, Union from bot.bot import Bot @@ -15,29 +16,29 @@ RedisValueType = Union[str, int, float] # Prefix tuples PrefixTuple = Tuple[Tuple[str, Any]] -TYPESTRING_VALUE_PREFIXES = ( +VALUE_PREFIXES = ( ("f|", float), ("i|", int), ("s|", str), ) -TYPESTRING_KEY_PREFIXES = ( +KEY_PREFIXES = ( ("i|", int), ("s|", str), ) # Makes a nice list like "float, int, and str" -NICE_VALUE_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisValueType)) -NICE_VALUE_TYPE_LIST = ", and ".join(NICE_VALUE_TYPE_LIST.rsplit(", ", 1)) +VALUE_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisValueType)) +VALUE_TYPE_LIST = ", and ".join(VALUE_TYPE_LIST.rsplit(", ", 1)) -NICE_KEY_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisKeyType)) -NICE_KEY_TYPE_LIST = ", and ".join(NICE_KEY_TYPE_LIST.rsplit(", ", 1)) +KEY_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisKeyType)) +KEY_TYPE_LIST = ", and ".join(KEY_TYPE_LIST.rsplit(", ", 1)) # Makes a list like "'f|', 'i|', and 's|'" -NICE_VALUE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_VALUE_PREFIXES]) -NICE_VALUE_PREFIX_LIST = ", and ".join(NICE_VALUE_PREFIX_LIST.rsplit(", ", 1)) +VALUE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in VALUE_PREFIXES]) +VALUE_PREFIX_LIST = ", and ".join(VALUE_PREFIX_LIST.rsplit(", ", 1)) -NICE_KEY_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in TYPESTRING_KEY_PREFIXES]) -NICE_KEY_PREFIX_LIST = ", and ".join(NICE_KEY_PREFIX_LIST.rsplit(", ", 1)) +KEY_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in KEY_PREFIXES]) +KEY_PREFIX_LIST = ", and ".join(KEY_PREFIX_LIST.rsplit(", ", 1)) class RedisCache: @@ -118,19 +119,19 @@ class RedisCache: def _to_typestring( key_or_value: Union[RedisKeyType, RedisValueType], prefixes: PrefixTuple, - nice_type_list: str + types_string: str ) -> str: """Turn a valid Redis type into a typestring.""" for prefix, _type in prefixes: if isinstance(key_or_value, _type): return f"{prefix}{key_or_value}" - raise TypeError(f"RedisCache._from_typestring only supports the types {nice_type_list}.") + raise TypeError(f"RedisCache._from_typestring only supports the types {types_string}.") @staticmethod def _from_typestring( key_or_value: Union[bytes, str], prefixes: PrefixTuple, - nice_prefix_list: str, + prefixes_string: str, ) -> Union[RedisKeyType, RedisValueType]: """Deserialize a typestring into a valid Redis type.""" # Stuff that comes out of Redis will be bytestrings, so let's decode those. @@ -141,23 +142,16 @@ class RedisCache: for prefix, _type in prefixes: if key_or_value.startswith(prefix): return _type(key_or_value[len(prefix):]) - raise TypeError(f"RedisCache._to_typestring only supports the prefixes {nice_prefix_list}.") - - def _key_to_typestring(self, key: RedisKeyType) -> str: - """Serialize a RedisKeyType object into a typestring.""" - return self._to_typestring(key, TYPESTRING_KEY_PREFIXES, NICE_KEY_TYPE_LIST) - - def _value_to_typestring(self, value: RedisValueType) -> str: - """Serialize a RedisValueType object into a typestring.""" - return self._to_typestring(value, TYPESTRING_VALUE_PREFIXES, NICE_VALUE_TYPE_LIST) - - def _key_from_typestring(self, key: Union[bytes, str]) -> RedisKeyType: - """Deserialize a RedisKeyType object from a typestring.""" - return self._from_typestring(key, TYPESTRING_KEY_PREFIXES, NICE_KEY_PREFIX_LIST) - - def _value_from_typestring(self, value: Union[bytes, str]) -> RedisValueType: - """Deserialize a RedisValueType object from a typestring.""" - return self._from_typestring(value, TYPESTRING_VALUE_PREFIXES, NICE_VALUE_PREFIX_LIST) + raise TypeError(f"RedisCache._to_typestring only supports the prefixes {prefixes_string}.") + + # Add some nice partials to call our generic typestring converters. + # These are basically functions that will fill in some of the parameters for you, so that + # any call to _key_to_typestring will be like calling _to_typestring with those two parameters + # pre-filled. + _key_to_typestring = partialmethod(_to_typestring, prefixes=KEY_PREFIXES, types_string=KEY_TYPE_LIST) + _value_to_typestring = partialmethod(_to_typestring, prefixes=VALUE_PREFIXES, types_string=VALUE_TYPE_LIST) + _key_from_typestring = partialmethod(_from_typestring, prefixes=KEY_PREFIXES, prefixes_string=KEY_PREFIX_LIST) + _value_from_typestring = partialmethod(_from_typestring, prefixes=VALUE_PREFIXES, prefixes_string=VALUE_PREFIX_LIST) def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" -- cgit v1.2.3 From 63b81b04da3cbc4d1824e65c977ec61532dbe605 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Wed, 27 May 2020 21:04:08 +0200 Subject: Fix ATROCIOUS comment. I should be shot. --- bot/utils/redis_cache.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index dd24b83e8..a71ad2191 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -145,9 +145,11 @@ class RedisCache: raise TypeError(f"RedisCache._to_typestring only supports the prefixes {prefixes_string}.") # Add some nice partials to call our generic typestring converters. - # These are basically functions that will fill in some of the parameters for you, so that - # any call to _key_to_typestring will be like calling _to_typestring with those two parameters - # pre-filled. + # These are basically methods that will fill in some of the parameters for you, so that + # any call to _key_to_typestring will be like calling _to_typestring with the two parameters + # at `prefixes` and `types_string` pre-filled. + # + # See https://docs.python.org/3/library/functools.html#functools.partialmethod _key_to_typestring = partialmethod(_to_typestring, prefixes=KEY_PREFIXES, types_string=KEY_TYPE_LIST) _value_to_typestring = partialmethod(_to_typestring, prefixes=VALUE_PREFIXES, types_string=VALUE_TYPE_LIST) _key_from_typestring = partialmethod(_from_typestring, prefixes=KEY_PREFIXES, prefixes_string=KEY_PREFIX_LIST) -- cgit v1.2.3 From bdb7bbc5e98bd840785def6ac08c9f5a313847cb Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 28 May 2020 00:43:58 +0200 Subject: Reduce complexity on some of the typestring stuff. - Refactor error messages in _to_typestring and _from_typestring to just print the prefix tuples instead of that custom error string. - Create a RedisKeyOrValue type to simplify some annotations. - Simplify partialmethod calls. - Make the signatures for _to_typestring and _from_typestring one-liners - Fix a typo in the errors. --- bot/utils/redis_cache.py | 40 +++++++++------------------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index a71ad2191..0b682d378 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -2,7 +2,6 @@ from __future__ import annotations import asyncio import logging -import typing from functools import partialmethod from typing import Any, Dict, ItemsView, Optional, Tuple, Union @@ -13,6 +12,7 @@ log = logging.getLogger(__name__) # Type aliases RedisKeyType = Union[str, int] RedisValueType = Union[str, int, float] +RedisKeyOrValue = Union[RedisKeyType, RedisValueType] # Prefix tuples PrefixTuple = Tuple[Tuple[str, Any]] @@ -26,20 +26,6 @@ KEY_PREFIXES = ( ("s|", str), ) -# Makes a nice list like "float, int, and str" -VALUE_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisValueType)) -VALUE_TYPE_LIST = ", and ".join(VALUE_TYPE_LIST.rsplit(", ", 1)) - -KEY_TYPE_LIST = ", ".join(str(_type.__name__) for _type in typing.get_args(RedisKeyType)) -KEY_TYPE_LIST = ", and ".join(KEY_TYPE_LIST.rsplit(", ", 1)) - -# Makes a list like "'f|', 'i|', and 's|'" -VALUE_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in VALUE_PREFIXES]) -VALUE_PREFIX_LIST = ", and ".join(VALUE_PREFIX_LIST.rsplit(", ", 1)) - -KEY_PREFIX_LIST = ", ".join([f"'{prefix}'" for prefix, _ in KEY_PREFIXES]) -KEY_PREFIX_LIST = ", and ".join(KEY_PREFIX_LIST.rsplit(", ", 1)) - class RedisCache: """ @@ -116,23 +102,15 @@ class RedisCache: self._namespace = namespace @staticmethod - def _to_typestring( - key_or_value: Union[RedisKeyType, RedisValueType], - prefixes: PrefixTuple, - types_string: str - ) -> str: + def _to_typestring(key_or_value: RedisKeyOrValue, prefixes: PrefixTuple) -> str: """Turn a valid Redis type into a typestring.""" for prefix, _type in prefixes: if isinstance(key_or_value, _type): return f"{prefix}{key_or_value}" - raise TypeError(f"RedisCache._from_typestring only supports the types {types_string}.") + raise TypeError(f"RedisCache._to_typestring only supports the following: {prefixes}.") @staticmethod - def _from_typestring( - key_or_value: Union[bytes, str], - prefixes: PrefixTuple, - prefixes_string: str, - ) -> Union[RedisKeyType, RedisValueType]: + def _from_typestring(key_or_value: Union[bytes, str], prefixes: PrefixTuple) -> RedisKeyOrValue: """Deserialize a typestring into a valid Redis type.""" # Stuff that comes out of Redis will be bytestrings, so let's decode those. if isinstance(key_or_value, bytes): @@ -142,7 +120,7 @@ class RedisCache: for prefix, _type in prefixes: if key_or_value.startswith(prefix): return _type(key_or_value[len(prefix):]) - raise TypeError(f"RedisCache._to_typestring only supports the prefixes {prefixes_string}.") + raise TypeError(f"RedisCache._from_typestring only supports the following: {prefixes}.") # Add some nice partials to call our generic typestring converters. # These are basically methods that will fill in some of the parameters for you, so that @@ -150,10 +128,10 @@ class RedisCache: # at `prefixes` and `types_string` pre-filled. # # See https://docs.python.org/3/library/functools.html#functools.partialmethod - _key_to_typestring = partialmethod(_to_typestring, prefixes=KEY_PREFIXES, types_string=KEY_TYPE_LIST) - _value_to_typestring = partialmethod(_to_typestring, prefixes=VALUE_PREFIXES, types_string=VALUE_TYPE_LIST) - _key_from_typestring = partialmethod(_from_typestring, prefixes=KEY_PREFIXES, prefixes_string=KEY_PREFIX_LIST) - _value_from_typestring = partialmethod(_from_typestring, prefixes=VALUE_PREFIXES, prefixes_string=VALUE_PREFIX_LIST) + _key_to_typestring = partialmethod(_to_typestring, prefixes=KEY_PREFIXES) + _value_to_typestring = partialmethod(_to_typestring, prefixes=VALUE_PREFIXES) + _key_from_typestring = partialmethod(_from_typestring, prefixes=KEY_PREFIXES) + _value_from_typestring = partialmethod(_from_typestring, prefixes=VALUE_PREFIXES) def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" -- cgit v1.2.3 From 11542fbcc7c32fb9a18577c45ae3c331eaa12db8 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 28 May 2020 00:56:10 +0200 Subject: Make prefix consts private and more precise. --- bot/utils/redis_cache.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 0b682d378..979ea5d47 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -15,13 +15,13 @@ RedisValueType = Union[str, int, float] RedisKeyOrValue = Union[RedisKeyType, RedisValueType] # Prefix tuples -PrefixTuple = Tuple[Tuple[str, Any]] -VALUE_PREFIXES = ( +_PrefixTuple = Tuple[Tuple[str, Any], ...] +_VALUE_PREFIXES = ( ("f|", float), ("i|", int), ("s|", str), ) -KEY_PREFIXES = ( +_KEY_PREFIXES = ( ("i|", int), ("s|", str), ) @@ -102,7 +102,7 @@ class RedisCache: self._namespace = namespace @staticmethod - def _to_typestring(key_or_value: RedisKeyOrValue, prefixes: PrefixTuple) -> str: + def _to_typestring(key_or_value: RedisKeyOrValue, prefixes: _PrefixTuple) -> str: """Turn a valid Redis type into a typestring.""" for prefix, _type in prefixes: if isinstance(key_or_value, _type): @@ -110,7 +110,7 @@ class RedisCache: raise TypeError(f"RedisCache._to_typestring only supports the following: {prefixes}.") @staticmethod - def _from_typestring(key_or_value: Union[bytes, str], prefixes: PrefixTuple) -> RedisKeyOrValue: + def _from_typestring(key_or_value: Union[bytes, str], prefixes: _PrefixTuple) -> RedisKeyOrValue: """Deserialize a typestring into a valid Redis type.""" # Stuff that comes out of Redis will be bytestrings, so let's decode those. if isinstance(key_or_value, bytes): @@ -128,10 +128,10 @@ class RedisCache: # at `prefixes` and `types_string` pre-filled. # # See https://docs.python.org/3/library/functools.html#functools.partialmethod - _key_to_typestring = partialmethod(_to_typestring, prefixes=KEY_PREFIXES) - _value_to_typestring = partialmethod(_to_typestring, prefixes=VALUE_PREFIXES) - _key_from_typestring = partialmethod(_from_typestring, prefixes=KEY_PREFIXES) - _value_from_typestring = partialmethod(_from_typestring, prefixes=VALUE_PREFIXES) + _key_to_typestring = partialmethod(_to_typestring, prefixes=_KEY_PREFIXES) + _value_to_typestring = partialmethod(_to_typestring, prefixes=_VALUE_PREFIXES) + _key_from_typestring = partialmethod(_from_typestring, prefixes=_KEY_PREFIXES) + _value_from_typestring = partialmethod(_from_typestring, prefixes=_VALUE_PREFIXES) def _dict_from_typestring(self, dictionary: Dict) -> Dict: """Turns all contents of a dict into valid Redis types.""" -- cgit v1.2.3 From f66a63501fe1ef8fb5390dfbe42ae9f95ea2bc28 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 28 May 2020 01:29:34 +0200 Subject: Add custom exceptions for each error state. The bot can get into trouble in three distinct ways: - It has no Bot instance - It has no namespace - It has no parent instance. These happen only if you're using it wrong. To make the test more precise, and to add a little bit more readability (RuntimeError could be anything!), we'll introduce some custom exceptions for these three states. This addresses a review comment by @aeros. --- bot/utils/redis_cache.py | 22 +++++++++++++++++----- tests/bot/utils/test_redis_cache.py | 7 ++++--- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 979ea5d47..6b3c68979 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -27,6 +27,18 @@ _KEY_PREFIXES = ( ) +class NoBotInstanceError(RuntimeError): + """Raised when RedisCache is created without an available bot instance on the owner class.""" + + +class NoNamespaceError(RuntimeError): + """Raised when RedisCache has no namespace, for example if it is not assigned to a class attribute.""" + + +class NoParentInstanceError(RuntimeError): + """Raised when the parent instance is available, for example if called by accessing the parent class directly.""" + + class RedisCache: """ A simplified interface for a Redis connection. @@ -149,7 +161,7 @@ class RedisCache: "This object must be initialized as a class attribute." ) log.error(error_message) - raise RuntimeError(error_message) + raise NoNamespaceError(error_message) if self.bot is None: error_message = ( @@ -159,7 +171,7 @@ class RedisCache: "the RedisCache inside a class that has a Bot instance attribute." ) log.error(error_message) - raise RuntimeError(error_message) + raise NoBotInstanceError(error_message) await self.bot.redis_ready.wait() @@ -194,7 +206,7 @@ class RedisCache: if self._namespace is None: error_message = "RedisCache must be a class attribute." log.error(error_message) - raise RuntimeError(error_message) + raise NoNamespaceError(error_message) if instance is None: error_message = ( @@ -202,7 +214,7 @@ class RedisCache: "before accessing it using the cog's class object." ) log.error(error_message) - raise RuntimeError(error_message) + raise NoParentInstanceError(error_message) for attribute in vars(instance).values(): if isinstance(attribute, Bot): @@ -217,7 +229,7 @@ class RedisCache: "the RedisCache inside a class that has a Bot instance attribute." ) log.error(error_message) - raise RuntimeError(error_message) + raise NoBotInstanceError(error_message) def __repr__(self) -> str: """Return a beautiful representation of this object instance.""" diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py index 4f95dff03..8c1a40640 100644 --- a/tests/bot/utils/test_redis_cache.py +++ b/tests/bot/utils/test_redis_cache.py @@ -4,6 +4,7 @@ import unittest import fakeredis.aioredis from bot.utils import RedisCache +from bot.utils.redis_cache import NoBotInstanceError, NoNamespaceError, NoParentInstanceError from tests import helpers @@ -260,13 +261,13 @@ class RedisCacheTests(unittest.IsolatedAsyncioTestCase): cog = MyCog() # Raises "No Bot instance" - with self.assertRaises(RuntimeError): + with self.assertRaises(NoBotInstanceError): await cog.cache.get("john") # Raises "RedisCache has no namespace" - with self.assertRaises(RuntimeError): + with self.assertRaises(NoNamespaceError): await cog.other_cache.get("was") # Raises "You must access the RedisCache instance through the cog instance" - with self.assertRaises(RuntimeError): + with self.assertRaises(NoParentInstanceError): await MyCog.cache.get("afraid") -- cgit v1.2.3 From b7c30d41e263605a680cfe0f623b8e7ed5936b7d Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 28 May 2020 10:35:17 +0200 Subject: Prevent a state where a coro could wait forever. This addresses a review comment by @aeros. --- bot/bot.py | 5 ++++- bot/utils/redis_cache.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bot/bot.py b/bot/bot.py index f1365d532..ba09ce207 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -32,6 +32,7 @@ class Bot(commands.Bot): self.http_session: Optional[aiohttp.ClientSession] = None self.redis_session: Optional[aioredis.Redis] = None self.redis_ready = asyncio.Event() + self.redis_closed = False self.api_client = api.APIClient(loop=self.loop) self._connector = None @@ -106,8 +107,9 @@ class Bot(commands.Bot): self.stats._transport.close() if self.redis_session: - self.redis_ready.clear() + self.redis_closed = True self.redis_session.close() + self.redis_ready.clear() await self.redis_session.wait_closed() async def login(self, *args, **kwargs) -> None: @@ -135,6 +137,7 @@ class Bot(commands.Bot): # Create the redis session self.loop.create_task(self._create_redis_session()) + self.redis_closed = False # Use AF_INET as its socket family to prevent HTTPS related problems both locally # and in production. diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 6b3c68979..de80cee84 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -173,7 +173,8 @@ class RedisCache: log.error(error_message) raise NoBotInstanceError(error_message) - await self.bot.redis_ready.wait() + if not self.bot.redis_closed: + await self.bot.redis_ready.wait() def __set_name__(self, owner: Any, attribute_name: str) -> None: """ -- cgit v1.2.3 From cc45960406f64a791a15cf9de76614103fda384b Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Thu, 28 May 2020 12:45:26 +0200 Subject: Move the `self.redis_closed` into session create. --- bot/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/bot.py b/bot/bot.py index ba09ce207..313652d11 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -70,6 +70,7 @@ class Bot(commands.Bot): password=constants.Redis.password, ) + self.redis_closed = False self.redis_ready.set() def add_cog(self, cog: commands.Cog) -> None: @@ -137,7 +138,6 @@ class Bot(commands.Bot): # Create the redis session self.loop.create_task(self._create_redis_session()) - self.redis_closed = False # Use AF_INET as its socket family to prevent HTTPS related problems both locally # and in production. -- cgit v1.2.3 From d12e84fe6834a0bc574e365a3283bc358c2ae4d9 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Fri, 29 May 2020 17:57:09 +0200 Subject: Ignore response when posting python news Sometimes a mailing list user doesn't press respond correctly to the email, and so a response is sent as a separate thread. To keep only new threads in the channel, we need to ignore those. --- bot/cogs/python_news.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/cogs/python_news.py b/bot/cogs/python_news.py index d28af4a0b..d15d0371e 100644 --- a/bot/cogs/python_news.py +++ b/bot/cogs/python_news.py @@ -153,6 +153,7 @@ class PythonNews(Cog): if ( thread_information["thread_id"] in existing_news["data"][maillist] + or 'Re: ' in thread_information["subject"] or new_date.date() < date.today() ): continue -- cgit v1.2.3 From 63f5028641e9b78c61d3bcfe3bbaa6f80c8a288a Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff Date: Fri, 29 May 2020 19:56:40 +0200 Subject: Fix `check_for_answer` breaking on missing cache The `check_for_answer` method of the HelpChannels cog relies on the channel->claimant cache being available. However, as this cache is (currently) lost during bot restarts, this method may fail with a KeyError exception. I've used `dict.get` with an `if not claimant: return` to circumvent this issue. --- bot/cogs/help_channels.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index d2a55fba6..2221132d4 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -660,10 +660,13 @@ class HelpChannels(Scheduler, commands.Cog): # Check if there is an entry in unanswered (does not persist across restarts) if channel.id in self.unanswered: - claimant_id = self.help_channel_claimants[channel].id + claimant = self.help_channel_claimants.get(channel) + if not claimant: + # The mapping for this channel was lost, we can't do anything. + return # Check the message did not come from the claimant - if claimant_id != message.author.id: + if claimant.id != message.author.id: # Mark the channel as answered self.unanswered[channel.id] = False -- cgit v1.2.3 From 28f20b969556e0ce1363ac44a5b9ff2bff2a6575 Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff Date: Fri, 29 May 2020 19:49:00 +0200 Subject: Reduce the number of help channel name changes Discord has introduced a new, strict rate limit for individual channel edits that reduces the number of allow channel name/channel topic changes to 2 per 10 minutes per channel. Unfortunately, our help channel system frequently goes over that rate limit as it edits the name and topic of a channel on all three "move" actions we have: to available, to occupied, and to dormant. In addition, our "unanswered" feature adds another channel name change on top of the move-related edits. That's why I've removed the topic/emoji changing features from the help channel system. This means we now have a generic topic that fits all three categories and no status emojis in the channel names. --- bot/cogs/help_channels.py | 32 +++----------------------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 2221132d4..70cef339a 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -24,18 +24,8 @@ ASKING_GUIDE_URL = "https://pythondiscord.com/pages/asking-good-questions/" MAX_CHANNELS_PER_CATEGORY = 50 EXCLUDED_CHANNELS = (constants.Channels.how_to_get_help,) -AVAILABLE_TOPIC = """ -This channel is available. Feel free to ask a question in order to claim this channel! -""" - -IN_USE_TOPIC = """ -This channel is currently in use. If you'd like to discuss a different problem, please claim a new \ -channel from the Help: Available category. -""" - -DORMANT_TOPIC = """ -This channel is temporarily archived. If you'd like to ask a question, please use one of the \ -channels in the Help: Available category. +HELP_CHANNEL_TOPIC = """ +This is a Python help channel. You can claim your own help channel in the Python Help: Available category. """ AVAILABLE_MSG = f""" @@ -64,11 +54,6 @@ question to maximize your chance of getting a good answer. If you're not sure ho through our guide for [asking a good question]({ASKING_GUIDE_URL}). """ -AVAILABLE_EMOJI = "✅" -IN_USE_ANSWERED_EMOJI = "⌛" -IN_USE_UNANSWERED_EMOJI = "⏳" -NAME_SEPARATOR = "｜" - CoroutineFunc = t.Callable[..., t.Coroutine] @@ -196,7 +181,7 @@ class HelpChannels(Scheduler, commands.Cog): return None log.debug(f"Creating a new dormant channel named {name}.") - return await self.dormant_category.create_text_channel(name) + return await self.dormant_category.create_text_channel(name, topic=HELP_CHANNEL_TOPIC) def create_name_queue(self) -> deque: """Return a queue of element names to use for creating new channels.""" @@ -542,8 +527,6 @@ class HelpChannels(Scheduler, commands.Cog): await self.move_to_bottom_position( channel=channel, category_id=constants.Categories.help_available, - name=f"{AVAILABLE_EMOJI}{NAME_SEPARATOR}{self.get_clean_channel_name(channel)}", - topic=AVAILABLE_TOPIC, ) self.report_stats() @@ -559,8 +542,6 @@ class HelpChannels(Scheduler, commands.Cog): await self.move_to_bottom_position( channel=channel, category_id=constants.Categories.help_dormant, - name=self.get_clean_channel_name(channel), - topic=DORMANT_TOPIC, ) self.bot.stats.incr(f"help.dormant_calls.{caller}") @@ -593,8 +574,6 @@ class HelpChannels(Scheduler, commands.Cog): await self.move_to_bottom_position( channel=channel, category_id=constants.Categories.help_in_use, - name=f"{IN_USE_UNANSWERED_EMOJI}{NAME_SEPARATOR}{self.get_clean_channel_name(channel)}", - topic=IN_USE_TOPIC, ) timeout = constants.HelpChannels.idle_minutes * 60 @@ -670,11 +649,6 @@ class HelpChannels(Scheduler, commands.Cog): # Mark the channel as answered self.unanswered[channel.id] = False - # Change the emoji in the channel name to signify activity - log.trace(f"#{channel} ({channel.id}) has been answered; changing its emoji") - name = self.get_clean_channel_name(channel) - await channel.edit(name=f"{IN_USE_ANSWERED_EMOJI}{NAME_SEPARATOR}{name}") - @commands.Cog.listener() async def on_message(self, message: discord.Message) -> None: """Move an available channel to the In Use category and replace it with a dormant one.""" -- cgit v1.2.3 From b258f77d1c9a1b62fef26e7fecdb89e57719dfac Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 00:12:44 +0200 Subject: Removing the periodic ping from verification. It's no longer needed, and causes problems with anti-raid and anti-spam. --- bot/cogs/verification.py | 44 +------------------------------------------- 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 99be3cdaa..0a087cee9 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -1,9 +1,7 @@ import logging from contextlib import suppress -from datetime import datetime from discord import Colour, Forbidden, Message, NotFound, Object -from discord.ext import tasks from discord.ext.commands import Cog, Context, command from bot import constants @@ -34,14 +32,6 @@ If you'd like to unsubscribe from the announcement notifications, simply send `! <#{constants.Channels.bot_commands}>. """ -if constants.DEBUG_MODE: - PERIODIC_PING = "Periodic checkpoint message successfully sent." -else: - PERIODIC_PING = ( - f"@everyone To verify that you have read our rules, please type `{constants.Bot.prefix}accept`." - " If you encounter any problems during the verification process, " - f"send a direct message to a staff member." - ) BOT_MESSAGE_DELETE_DELAY = 10 @@ -50,7 +40,6 @@ class Verification(Cog): def __init__(self, bot: Bot): self.bot = bot - self.periodic_ping.start() @property def mod_log(self) -> ModLog: @@ -65,10 +54,7 @@ class Verification(Cog): if message.author.bot: # They're a bot, delete their message after the delay. - # But not the periodic ping; we like that one. - if message.content != PERIODIC_PING: - await message.delete(delay=BOT_MESSAGE_DELETE_DELAY) - return + await message.delete(delay=BOT_MESSAGE_DELETE_DELAY) # if a user mentions a role or guild member # alert the mods in mod-alerts channel @@ -198,34 +184,6 @@ class Verification(Cog): else: return True - @tasks.loop(hours=12) - async def periodic_ping(self) -> None: - """Every week, mention @everyone to remind them to verify.""" - messages = self.bot.get_channel(constants.Channels.verification).history(limit=10) - need_to_post = True # True if a new message needs to be sent. - - async for message in messages: - if message.author == self.bot.user and message.content == PERIODIC_PING: - delta = datetime.utcnow() - message.created_at # Time since last message. - if delta.days >= 7: # Message is older than a week. - await message.delete() - else: - need_to_post = False - - break - - if need_to_post: - await self.bot.get_channel(constants.Channels.verification).send(PERIODIC_PING) - - @periodic_ping.before_loop - async def before_ping(self) -> None: - """Only start the loop when the bot is ready.""" - await self.bot.wait_until_guild_available() - - def cog_unload(self) -> None: - """Cancel the periodic ping task when the cog is unloaded.""" - self.periodic_ping.cancel() - def setup(bot: Bot) -> None: """Load the Verification cog.""" -- cgit v1.2.3 From f7fe7df271b0e0930fa8b997c087bb3be141d016 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 11 Apr 2020 07:43:56 +0200 Subject: Tags: explicitly use UTF-8 to read files Not all operating systems use UTF-8 as the default encoding. For systems that don't, reading tag files with Unicode would cause an unhandled exception. (cherry picked from commit adc75ff9bbcf8b905bd78c78f253522ae5e42fc3) --- bot/cogs/tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index bc7f53f68..6f03a3475 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -44,7 +44,7 @@ class Tags(Cog): tag = { "title": tag_title, "embed": { - "description": file.read_text(), + "description": file.read_text(encoding="utf8"), }, "restricted_to": "developers", } -- cgit v1.2.3 From 96b026198a4ca2074f4fd7ea68e8a09acd5b38e4 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:34:39 +0300 Subject: Simplify infraction reason truncation tests --- tests/bot/cogs/moderation/test_infractions.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index 5548d9f68..ad3c95958 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -27,15 +27,14 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): self.cog.apply_infraction = AsyncMock() self.bot.get_cog.return_value = AsyncMock() self.cog.mod_log.ignore = Mock() + self.ctx.guild.ban = Mock() await self.cog.apply_ban(self.ctx, self.target, "foo bar" * 3000) - ban = self.cog.apply_infraction.call_args[0][3] - self.assertEqual( - ban.cr_frame.f_locals["kwargs"]["reason"], - textwrap.shorten("foo bar" * 3000, 512, placeholder="...") + self.ctx.guild.ban.assert_called_once_with( + self.target, + reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="..."), + delete_message_days=0 ) - # Await ban to avoid not awaited coroutine warning - await ban @patch("bot.cogs.moderation.utils.post_infraction") async def test_apply_kick_reason_truncation(self, post_infraction_mock): @@ -44,12 +43,7 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): self.cog.apply_infraction = AsyncMock() self.cog.mod_log.ignore = Mock() + self.target.kick = Mock() await self.cog.apply_kick(self.ctx, self.target, "foo bar" * 3000) - kick = self.cog.apply_infraction.call_args[0][3] - self.assertEqual( - kick.cr_frame.f_locals["kwargs"]["reason"], - textwrap.shorten("foo bar" * 3000, 512, placeholder="...") - ) - # Await kick to avoid not awaited coroutine warning - await kick + self.target.kick.assert_called_once_with(reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="...")) -- cgit v1.2.3 From 7f827abfa1922a4ec81d2f49fa2811471588269d Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:44:01 +0300 Subject: Scheduler: Move inline f-string if-else statement to normal if statement --- bot/cogs/moderation/scheduler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index b65048f4c..80a58484c 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -174,12 +174,15 @@ class InfractionScheduler(Scheduler): dm_result = f"{constants.Emojis.failmail} " log.trace(f"Deleted infraction {infraction['id']} from database because applying infraction failed.") await self.bot.api_client.delete(f"bot/infractions/{infraction['id']}") + infr_message = "" + else: + infr_message = f"**{infr_type}** to {user.mention}{expiry_msg}{end_msg}" # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} confirmation message.") await ctx.send( f"{dm_result}{confirm_msg} " - f"{f'**{infr_type}** to {user.mention}{expiry_msg}{end_msg}' if not failed else ''}." + f"{infr_message}." ) # Send a log message to the mod log. -- cgit v1.2.3 From 136ef112999b9387f04c3e0b800a3008ac07934f Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:44:36 +0300 Subject: Scheduler: Remove invalid comment --- bot/cogs/moderation/scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 80a58484c..7e8455740 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -84,7 +84,6 @@ class InfractionScheduler(Scheduler): """Apply an infraction to the user, log the infraction, and optionally notify the user.""" infr_type = infraction["type"] icon = utils.INFRACTION_ICONS[infr_type][0] - # Truncate reason when it's too long to avoid raising error on sending ModLog entry reason = infraction["reason"] expiry = time.format_infraction_with_duration(infraction["expires_at"]) id_ = infraction['id'] -- cgit v1.2.3 From f42ddf64abfd487fd69eec275d1011112eb76166 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:53:38 +0300 Subject: Scheduler: Add try-except to infraction deletion --- bot/cogs/moderation/scheduler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 7e8455740..dcc0001f8 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -172,7 +172,12 @@ class InfractionScheduler(Scheduler): dm_log_text = "\nDM: **Canceled**" dm_result = f"{constants.Emojis.failmail} " log.trace(f"Deleted infraction {infraction['id']} from database because applying infraction failed.") - await self.bot.api_client.delete(f"bot/infractions/{infraction['id']}") + try: + await self.bot.api_client.delete(f"bot/infractions/{id_}") + except ResponseCodeError as e: + confirm_msg += f" and failed to delete" + log_title += " and failed to delete" + log.error(f"Deletion of {infr_type} infraction #{id_} failed with error code {e.status}.") infr_message = "" else: infr_message = f"**{infr_type}** to {user.mention}{expiry_msg}{end_msg}" -- cgit v1.2.3 From e71beb79f6f78b348ff17974844806c981da3c2d Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:56:37 +0300 Subject: Scheduler: Remove unnecessary `f` before string --- bot/cogs/moderation/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index dcc0001f8..0d4f0ffba 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -175,7 +175,7 @@ class InfractionScheduler(Scheduler): try: await self.bot.api_client.delete(f"bot/infractions/{id_}") except ResponseCodeError as e: - confirm_msg += f" and failed to delete" + confirm_msg += " and failed to delete" log_title += " and failed to delete" log.error(f"Deletion of {infr_type} infraction #{id_} failed with error code {e.status}.") infr_message = "" -- cgit v1.2.3 From 854b27593e13f1e35810c37f4be91e5c8c4516b2 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 09:58:14 +0300 Subject: Scheduler: Fix spaces for modlog text Co-authored-by: Mark --- bot/cogs/moderation/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 0d4f0ffba..8b28afa69 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -198,7 +198,7 @@ class InfractionScheduler(Scheduler): thumbnail=user.avatar_url_as(static_format="png"), text=textwrap.dedent(f""" Member: {user.mention} (`{user.id}`) - Actor: {ctx.message.author}{dm_log_text} {expiry_log_text} + Actor: {ctx.message.author}{dm_log_text}{expiry_log_text} Reason: {reason} """), content=log_content, -- cgit v1.2.3 From 323317496310ef474a39d468e273703106e44768 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 10:07:21 +0300 Subject: Infr. Tests: Add `apply_infraction` awaiting assertion with args --- tests/bot/cogs/moderation/test_infractions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py index ad3c95958..da4e92ccc 100644 --- a/tests/bot/cogs/moderation/test_infractions.py +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -35,6 +35,9 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="..."), delete_message_days=0 ) + self.cog.apply_infraction.assert_awaited_once_with( + self.ctx, {"foo": "bar"}, self.target, self.ctx.guild.ban.return_value + ) @patch("bot.cogs.moderation.utils.post_infraction") async def test_apply_kick_reason_truncation(self, post_infraction_mock): @@ -47,3 +50,6 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): await self.cog.apply_kick(self.ctx, self.target, "foo bar" * 3000) self.target.kick.assert_called_once_with(reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="...")) + self.cog.apply_infraction.assert_awaited_once_with( + self.ctx, {"foo": "bar"}, self.target, self.target.kick.return_value + ) -- cgit v1.2.3 From e236113612c560326176da91f5a743c514ac988b Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 10:10:34 +0300 Subject: Scheduler: Remove line splitting from `ctx.send` after 7f827ab --- bot/cogs/moderation/scheduler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 8b28afa69..3679561a3 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -184,10 +184,7 @@ class InfractionScheduler(Scheduler): # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} confirmation message.") - await ctx.send( - f"{dm_result}{confirm_msg} " - f"{infr_message}." - ) + await ctx.send(f"{dm_result}{confirm_msg} {infr_message}.") # Send a log message to the mod log. log.trace(f"Sending apply mod log for infraction #{id_}.") -- cgit v1.2.3 From f1f2c488dc29d731b3343c949fe49cc3eaced842 Mon Sep 17 00:00:00 2001 From: ks129 <45097959+ks129@users.noreply.github.com> Date: Sat, 30 May 2020 10:17:05 +0300 Subject: Scheduler: Move space from f-string of `ctx.send` to `infr_message` --- bot/cogs/moderation/scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 3679561a3..1c7786df4 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -180,11 +180,11 @@ class InfractionScheduler(Scheduler): log.error(f"Deletion of {infr_type} infraction #{id_} failed with error code {e.status}.") infr_message = "" else: - infr_message = f"**{infr_type}** to {user.mention}{expiry_msg}{end_msg}" + infr_message = f" **{infr_type}** to {user.mention}{expiry_msg}{end_msg}" # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} confirmation message.") - await ctx.send(f"{dm_result}{confirm_msg} {infr_message}.") + await ctx.send(f"{dm_result}{confirm_msg}{infr_message}.") # Send a log message to the mod log. log.trace(f"Sending apply mod log for infraction #{id_}.") -- cgit v1.2.3 From 4bbb5b127315727b1534c5a0e77bfdd48173847b Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sat, 30 May 2020 21:05:13 +0200 Subject: Free tag: link #how-to-get-help This creates a clickable link in the response embed. Referencing the category is no longer necessary. --- bot/resources/tags/free.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/resources/tags/free.md b/bot/resources/tags/free.md index 582cca9da..1493076c7 100644 --- a/bot/resources/tags/free.md +++ b/bot/resources/tags/free.md @@ -1,5 +1,5 @@ **We have a new help channel system!** -We recently moved to a new help channel system. You can now use any channel in the **<#691405807388196926>** category to ask your question. +Please see <#704250143020417084> for further information. -For more information, check out [our website](https://pythondiscord.com/pages/resources/guides/help-channels/). +A more detailed guide can be found on [our website](https://pythondiscord.com/pages/resources/guides/help-channels/). -- cgit v1.2.3 From 1cc1b3851871dfff5690432960ade09fe8ab5794 Mon Sep 17 00:00:00 2001 From: Leon Sandøy Date: Sat, 30 May 2020 23:14:21 +0200 Subject: Oops, add the return back. We do not wanna process bot messages. --- bot/cogs/verification.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 0a087cee9..ae156cf70 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -55,6 +55,7 @@ class Verification(Cog): if message.author.bot: # They're a bot, delete their message after the delay. await message.delete(delay=BOT_MESSAGE_DELETE_DELAY) + return # if a user mentions a role or guild member # alert the mods in mod-alerts channel -- cgit v1.2.3