diff options
author | 2022-02-22 18:17:32 -0800 | |
---|---|---|
committer | 2022-02-22 18:17:32 -0800 | |
commit | 64ca00f0d306ab9d8bfd78b9af9e99d33283cb03 (patch) | |
tree | af7231269aca52153f7f19df8a2d9b96bb1fb6b5 | |
parent | Clarify that a resent infraction DM is not a new infraction. (diff) | |
parent | Merge pull request #2078 from python-discord/chris/fix/help-channel-errors (diff) |
Merge main into feat/mod/1664/resend-infraction
169 files changed, 7418 insertions, 3699 deletions
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6dfe7e859..ea69f7677 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,10 +4,10 @@ **/bot/exts/moderation/*silence.py @MarkKoz bot/exts/info/codeblock/** @MarkKoz bot/exts/utils/extensions.py @MarkKoz -bot/exts/utils/snekbox.py @MarkKoz @Akarys42 @jb3 -bot/exts/help_channels/** @MarkKoz @Akarys42 -bot/exts/moderation/** @Akarys42 @mbaruh @Den4200 @ks129 @jb3 -bot/exts/info/** @Akarys42 @Den4200 @jb3 +bot/exts/utils/snekbox.py @MarkKoz @jb3 +bot/exts/help_channels/** @MarkKoz +bot/exts/moderation/** @mbaruh @Den4200 @ks129 @jb3 +bot/exts/info/** @Den4200 @jb3 bot/exts/info/information.py @mbaruh @jb3 bot/exts/filters/** @mbaruh @jb3 bot/exts/fun/** @ks129 @@ -21,22 +21,16 @@ bot/rules/** @mbaruh bot/utils/extensions.py @MarkKoz bot/utils/function.py @MarkKoz bot/utils/lock.py @MarkKoz -bot/utils/regex.py @Akarys42 bot/utils/scheduling.py @MarkKoz # Tests tests/_autospec.py @MarkKoz tests/bot/exts/test_cogs.py @MarkKoz -tests/** @Akarys42 # CI & Docker -.github/workflows/** @MarkKoz @Akarys42 @SebastiaanZ @Den4200 @jb3 -Dockerfile @MarkKoz @Akarys42 @Den4200 @jb3 -docker-compose.yml @MarkKoz @Akarys42 @Den4200 @jb3 - -# Tools -poetry.lock @Akarys42 -pyproject.toml @Akarys42 +.github/workflows/** @MarkKoz @SebastiaanZ @Den4200 @jb3 +Dockerfile @MarkKoz @Den4200 @jb3 +docker-compose.yml @MarkKoz @Den4200 @jb3 # Statistics bot/async_stats.py @jb3 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 84a671917..f8f2c8888 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,10 @@ on: types: - completed +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8b809b777..88abe6fb6 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -8,6 +8,10 @@ on: types: - completed +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: environment: production @@ -38,6 +42,6 @@ jobs: uses: Azure/k8s-deploy@v1 with: manifests: | - bot/deployment.yaml + namespaces/default/bot/deployment.yaml images: 'ghcr.io/python-discord/bot:${{ steps.sha_tag.outputs.tag }}' kubectl-version: 'latest' diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index e99e6d181..57cc544d9 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -6,11 +6,25 @@ on: - main pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: lint-test: runs-on: ubuntu-latest env: + # List of licenses that are compatible with the MIT License and + # can be used in our project + ALLOWED_LICENSE: Apache Software License; + BSD License; + GNU Library or Lesser General Public License (LGPL); + ISC License (ISCL); + MIT License; + Mozilla Public License 2.0 (MPL 2.0); + Public Domain; + Python Software Foundation License + # Dummy values for required bot environment variables BOT_API_KEY: foo BOT_SENTRY_DSN: blah @@ -32,6 +46,10 @@ jobs: PYTHONUSERBASE: ${{ github.workspace }}/.cache/py-user-base PRE_COMMIT_HOME: ${{ github.workspace }}/.cache/pre-commit-cache + # See https://github.com/pre-commit/pre-commit/issues/2178#issuecomment-1002163763 + # for why we set this. + SETUPTOOLS_USE_DISTUTILS: stdlib + steps: - name: Add custom PYTHONUSERBASE to PATH run: echo '${{ env.PYTHONUSERBASE }}/bin/' >> $GITHUB_PATH @@ -67,6 +85,15 @@ jobs: pip install poetry poetry install + # Check all of our non-dev dependencies are compatible with the MIT license. + # If you added a new dependencies that is being rejected, + # please make sure it is compatible with the license for this project, + # and add it to the ALLOWED_LICENSE variable + - name: Check Dependencies License + run: | + pip-licenses --allow-only="$ALLOWED_LICENSE" \ + --package $(poetry export -f requirements.txt --without-hashes | sed "s/==.*//g" | tr "\n" " ") + # This step caches our pre-commit environment. To make sure we # do create a new environment when our pre-commit setup changes, # we create a cache key based on relevant factors. @@ -100,13 +127,6 @@ jobs: - name: Run tests and generate coverage report run: pytest -n auto --cov --disable-warnings -q - # This step will publish the coverage reports coveralls.io and - # print a "job" link in the output of the GitHub Action - - name: Publish coverage report to coveralls.io - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: coveralls - # Prepare the Pull Request Payload artifact. If this fails, we # we fail silently using the `continue-on-error` option. It's # nice if this succeeds, but if it fails for any reason, it diff --git a/.github/workflows/sentry_release.yml b/.github/workflows/sentry_release.yml index f6a1e1f0e..48f5e50f4 100644 --- a/.github/workflows/sentry_release.yml +++ b/.github/workflows/sentry_release.yml @@ -5,6 +5,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: create_sentry_release: runs-on: ubuntu-latest diff --git a/.github/workflows/status_embed.yaml b/.github/workflows/status_embed.yaml index b6a71b887..4178c366d 100644 --- a/.github/workflows/status_embed.yaml +++ b/.github/workflows/status_embed.yaml @@ -9,6 +9,10 @@ on: types: - completed +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: status_embed: # We need to send a status embed whenever the workflow diff --git a/.gitignore b/.gitignore index f74a142f3..177345908 100644 --- a/.gitignore +++ b/.gitignore @@ -116,6 +116,7 @@ log.* # Custom user configuration config.yml docker-compose.override.yml +metricity-config.toml # xmlrunner unittest XML reports TEST-**.xml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a9412f07d..d8a90ac00 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,6 +13,11 @@ repos: rev: v1.5.1 hooks: - id: python-check-blanket-noqa + - repo: https://github.com/pycqa/isort + rev: 5.8.0 + hooks: + - id: isort + name: isort (python) - repo: local hooks: - id: flake8 diff --git a/Dockerfile b/Dockerfile index c285898dc..30bf8a361 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.5-slim +FROM --platform=linux/amd64 python:3.9-slim # Set pip to have no saved cache ENV PIP_NO_CACHE_DIR=false \ @@ -4,7 +4,6 @@ [![Lint & Test][1]][2] [![Build][3]][4] [![Deploy][5]][6] -[](https://coveralls.io/github/python-discord/bot) [](LICENSE) This project is a Discord bot specifically for use with the Python Discord server. It provides numerous utilities diff --git a/bot/__init__.py b/bot/__init__.py index 8f880b8e6..17d99105a 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -5,8 +5,7 @@ from typing import TYPE_CHECKING from discord.ext import commands -from bot import log -from bot.command import Command +from bot import log, monkey_patches if TYPE_CHECKING: from bot.bot import Bot @@ -17,9 +16,16 @@ log.setup() if os.name == "nt": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +monkey_patches.patch_typing() + +# This patches any convertors that use PartialMessage, but not the PartialMessageConverter itself +# as library objects are made by this mapping. +# https://github.com/Rapptz/discord.py/blob/1a4e73d59932cdbe7bf2c281f25e32529fc7ae1f/discord/ext/commands/converter.py#L984-L1004 +commands.converter.PartialMessageConverter = monkey_patches.FixedPartialMessageConverter + # Monkey-patch discord.py decorators to use the Command subclass which supports root aliases. # Must be patched before any cogs are added. -commands.command = partial(commands.command, cls=Command) -commands.GroupMixin.command = partialmethod(commands.GroupMixin.command, cls=Command) +commands.command = partial(commands.command, cls=monkey_patches.Command) +commands.GroupMixin.command = partialmethod(commands.GroupMixin.command, cls=monkey_patches.Command) instance: "Bot" = None # Global Bot instance. diff --git a/bot/__main__.py b/bot/__main__.py index 9317563c8..0d3fce180 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -1,11 +1,9 @@ -import logging - import aiohttp import bot from bot import constants from bot.bot import Bot, StartupError -from bot.log import setup_sentry +from bot.log import get_logger, setup_sentry setup_sentry() @@ -21,7 +19,7 @@ except StartupError as e: message = "Could not connect to Redis. Is it running?" # The exception is logged with an empty message so the actual message is visible at the bottom - log = logging.getLogger("bot") + log = get_logger("bot") log.fatal("", exc_info=e.exception) log.fatal(message) diff --git a/bot/api.py b/bot/api.py index 6ce9481f4..856f7c865 100644 --- a/bot/api.py +++ b/bot/api.py @@ -1,13 +1,14 @@ import asyncio -import logging from typing import Optional from urllib.parse import quote as quote_url import aiohttp +from bot.log import get_logger + from .constants import Keys, URLs -log = logging.getLogger(__name__) +log = get_logger(__name__) class ResponseCodeError(ValueError): diff --git a/bot/async_stats.py b/bot/async_stats.py index 58a80f528..2af832e5b 100644 --- a/bot/async_stats.py +++ b/bot/async_stats.py @@ -3,6 +3,8 @@ import socket from statsd.client.base import StatsClientBase +from bot.utils import scheduling + class AsyncStatsClient(StatsClientBase): """An async transport method for statsd communication.""" @@ -32,7 +34,7 @@ class AsyncStatsClient(StatsClientBase): def _send(self, data: str) -> None: """Start an async task to send data to statsd.""" - self._loop.create_task(self._async_send(data)) + scheduling.create_task(self._async_send(data), event_loop=self._loop) async def _async_send(self, data: str) -> None: """Send data to the statsd server using the async transport.""" diff --git a/bot/bot.py b/bot/bot.py index 914da9c98..94783a466 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -1,5 +1,4 @@ import asyncio -import logging import socket import warnings from collections import defaultdict @@ -14,8 +13,9 @@ from sentry_sdk import push_scope from bot import api, constants from bot.async_stats import AsyncStatsClient +from bot.log import get_logger -log = logging.getLogger('bot') +log = get_logger('bot') LOCALHOST = "127.0.0.1" @@ -109,7 +109,7 @@ class Bot(commands.Bot): def create(cls) -> "Bot": """Create and return an instance of a Bot.""" loop = asyncio.get_event_loop() - allowed_roles = [discord.Object(id_) for id_ in constants.MODERATION_ROLES] + allowed_roles = list({discord.Object(id_) for id_ in constants.MODERATION_ROLES}) intents = discord.Intents.all() intents.presences = False diff --git a/bot/command.py b/bot/command.py deleted file mode 100644 index 0fb900f7b..000000000 --- a/bot/command.py +++ /dev/null @@ -1,18 +0,0 @@ -from discord.ext import commands - - -class Command(commands.Command): - """ - A `discord.ext.commands.Command` subclass which supports root aliases. - - A `root_aliases` keyword argument is added, which is a sequence of alias names that will act as - top-level commands rather than being aliases of the command's group. It's stored as an attribute - also named `root_aliases`. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.root_aliases = kwargs.get("root_aliases", []) - - if not isinstance(self.root_aliases, (list, tuple)): - raise TypeError("Root aliases of a command must be a list or a tuple of strings.") diff --git a/bot/constants.py b/bot/constants.py index 6ff0ceebe..f31f74136 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -9,8 +9,6 @@ the custom configuration. Any settings left out in the custom user configuration will stay their default values from `config-default.yml`. """ - -import logging import os from collections.abc import Mapping from enum import Enum @@ -25,8 +23,6 @@ try: except ModuleNotFoundError: pass -log = logging.getLogger(__name__) - def _env_var_constructor(loader, node): """ @@ -104,7 +100,7 @@ def _recursive_update(original, new): if Path("config.yml").exists(): - log.info("Found `config.yml` file, loading constants from it.") + print("Found `config.yml` file, loading constants from it.") with open("config.yml", encoding="UTF-8") as f: user_config = yaml.safe_load(f) _recursive_update(_CONFIG_YAML, user_config) @@ -123,11 +119,10 @@ def check_required_keys(keys): if lookup is None: raise KeyError(key) except KeyError: - log.critical( + raise KeyError( f"A configuration for `{key_path}` is required, but was not found. " "Please set it in `config.yml` or setup an environment variable and try again." ) - raise try: @@ -186,8 +181,7 @@ class YAMLGetter(type): (cls.section, cls.subsection, name) if cls.subsection is not None else (cls.section, name) ) - # Only an INFO log since this can be caught through `hasattr` or `getattr`. - log.info(f"Tried accessing configuration variable at `{dotted_path}`, but it could not be found.") + print(f"Tried accessing configuration variable at `{dotted_path}`, but it could not be found.") raise AttributeError(repr(name)) from e def __getitem__(cls, name): @@ -436,7 +430,7 @@ class Channels(metaclass=YAMLGetter): off_topic_2: int bot_commands: int - discord_py: int + discord_bots: int esoteric: int voice_gate: int code_jam_planning: int @@ -448,6 +442,8 @@ class Channels(metaclass=YAMLGetter): incidents: int incidents_archive: int mod_alerts: int + mod_meta: int + mods: int nominations: int nomination_voting: int organisation: int @@ -470,7 +466,6 @@ class Channels(metaclass=YAMLGetter): voice_chat_1: int big_brother_logs: int - talent_pool: int class Webhooks(metaclass=YAMLGetter): @@ -480,15 +475,20 @@ class Webhooks(metaclass=YAMLGetter): big_brother: int dev_log: int duck_pond: int + incidents: int incidents_archive: int - talent_pool: int class Roles(metaclass=YAMLGetter): section = "guild" subsection = "roles" + # Self-assignable roles, see the Subscribe cog + advent_of_code: int announcements: int + lovefest: int + pyweek_announcements: int + contributors: int help_cooldown: int muted: int @@ -566,13 +566,16 @@ class Metabase(metaclass=YAMLGetter): username: Optional[str] password: Optional[str] - url: str + base_url: str + public_url: str max_session_age: int class AntiSpam(metaclass=YAMLGetter): section = 'anti_spam' + cache_size: int + clean_offending: bool ping_everyone: bool @@ -684,8 +687,16 @@ class VideoPermission(metaclass=YAMLGetter): default_permission_duration: int +class ThreadArchiveTimes(Enum): + HOUR = 60 + DAY = 1440 + THREE_DAY = 4320 + WEEK = 10080 + + # Debug mode -DEBUG_MODE = 'local' in os.environ.get("SITE_URL", "local") +DEBUG_MODE: bool = _CONFIG_YAML["debug"] == "true" +FILE_LOGS: bool = _CONFIG_YAML["file_logs"].lower() == "true" # Paths BOT_DIR = os.path.dirname(__file__) @@ -694,6 +705,7 @@ PROJECT_ROOT = os.path.abspath(os.path.join(BOT_DIR, os.pardir)) # Default role combinations MODERATION_ROLES = Guild.moderation_roles STAFF_ROLES = Guild.staff_roles +STAFF_PARTNERS_COMMUNITY_ROLES = STAFF_ROLES + [Roles.partners, Roles.python_community] # Channel combinations MODERATION_CHANNELS = Guild.moderation_channels diff --git a/bot/converters.py b/bot/converters.py index 595809517..3522a32aa 100644 --- a/bot/converters.py +++ b/bot/converters.py @@ -1,27 +1,34 @@ -import logging +from __future__ import annotations + import re import typing as t -from datetime import datetime -from functools import partial +from datetime import datetime, timezone from ssl import CertificateError import dateutil.parser -import dateutil.tz import discord from aiohttp import ClientConnectorError +from botcore.regex import DISCORD_INVITE from dateutil.relativedelta import relativedelta -from discord.ext.commands import BadArgument, Bot, Context, Converter, IDConverter, UserConverter -from discord.utils import DISCORD_EPOCH, snowflake_time +from discord.ext.commands import BadArgument, Bot, Context, Converter, IDConverter, MemberConverter, UserConverter +from discord.utils import escape_markdown, snowflake_time +from bot import exts from bot.api import ResponseCodeError from bot.constants import URLs +from bot.errors import InvalidInfraction from bot.exts.info.doc import _inventory_parser -from bot.utils.regex import INVITE_RE -from bot.utils.time import parse_duration_string +from bot.exts.info.tags import TagIdentifier +from bot.log import get_logger +from bot.utils import time +from bot.utils.extensions import EXTENSIONS, unqualify + +if t.TYPE_CHECKING: + from bot.exts.info.source import SourceType -log = logging.getLogger(__name__) +log = get_logger(__name__) -DISCORD_EPOCH_DT = datetime.utcfromtimestamp(DISCORD_EPOCH / 1000) +DISCORD_EPOCH_DT = snowflake_time(0) RE_USER_MENTION = re.compile(r"<@!?([0-9]+)>$") @@ -64,10 +71,10 @@ class ValidDiscordServerInvite(Converter): async def convert(self, ctx: Context, server_invite: str) -> dict: """Check whether the string is a valid Discord server invite.""" - invite_code = INVITE_RE.search(server_invite) + invite_code = DISCORD_INVITE.match(server_invite) if invite_code: response = await ctx.bot.http_session.get( - f"{URLs.discord_invite_api}/{invite_code[1]}" + f"{URLs.discord_invite_api}/{invite_code.group('invite')}" ) if response.status != 404: invite_data = await response.json() @@ -128,6 +135,44 @@ class ValidFilterListType(Converter): return list_type +class Extension(Converter): + """ + Fully qualify the name of an extension and ensure it exists. + + The * and ** values bypass this when used with the reload command. + """ + + async def convert(self, ctx: Context, argument: str) -> str: + """Fully qualify the name of an extension and ensure it exists.""" + # Special values to reload all extensions + if argument == "*" or argument == "**": + return argument + + argument = argument.lower() + + if argument in EXTENSIONS: + return argument + elif (qualified_arg := f"{exts.__name__}.{argument}") in EXTENSIONS: + return qualified_arg + + matches = [] + for ext in EXTENSIONS: + if argument == unqualify(ext): + matches.append(ext) + + if len(matches) > 1: + matches.sort() + names = "\n".join(matches) + raise BadArgument( + f":x: `{argument}` is an ambiguous extension name. " + f"Please use one of the following fully-qualified names.```\n{names}```" + ) + elif matches: + return matches[0] + else: + raise BadArgument(f":x: Could not find the extension `{argument}`.") + + class PackageName(Converter): """ A converter that checks whether the given string is a valid package name. @@ -191,11 +236,16 @@ class Inventory(Converter): async def convert(ctx: Context, url: str) -> t.Tuple[str, _inventory_parser.InventoryDict]: """Convert url to Intersphinx inventory URL.""" await ctx.trigger_typing() - if (inventory := await _inventory_parser.fetch_inventory(url)) is None: - raise BadArgument( - f"Failed to fetch inventory file after {_inventory_parser.FAILED_REQUEST_ATTEMPTS} attempts." - ) - return url, inventory + try: + inventory = await _inventory_parser.fetch_inventory(url) + except _inventory_parser.InvalidHeaderError: + raise BadArgument("Unable to parse inventory because of invalid header, check if URL is correct.") + else: + if inventory is None: + raise BadArgument( + f"Failed to fetch inventory file after {_inventory_parser.FAILED_REQUEST_ATTEMPTS} attempts." + ) + return url, inventory class Snowflake(IDConverter): @@ -230,64 +280,43 @@ class Snowflake(IDConverter): if time < DISCORD_EPOCH_DT: raise BadArgument(f"{error}: timestamp is before the Discord epoch.") - elif (datetime.utcnow() - time).days < -1: + elif (datetime.now(timezone.utc) - time).days < -1: raise BadArgument(f"{error}: timestamp is too far into the future.") return snowflake -class TagNameConverter(Converter): - """ - Ensure that a proposed tag name is valid. - - Valid tag names meet the following conditions: - * All ASCII characters - * Has at least one non-whitespace character - * Not solely numeric - * Shorter than 127 characters - """ +class SourceConverter(Converter): + """Convert an argument into a help command, tag, command, or cog.""" @staticmethod - async def convert(ctx: Context, tag_name: str) -> str: - """Lowercase & strip whitespace from proposed tag_name & ensure it's valid.""" - tag_name = tag_name.lower().strip() - - # The tag name has at least one invalid character. - if ascii(tag_name)[1:-1] != tag_name: - raise BadArgument("Don't be ridiculous, you can't use that character!") - - # The tag name is either empty, or consists of nothing but whitespace. - elif not tag_name: - raise BadArgument("Tag names should not be empty, or filled with whitespace.") - - # The tag name is longer than 127 characters. - elif len(tag_name) > 127: - raise BadArgument("Are you insane? That's way too long!") - - # The tag name is ascii but does not contain any letters. - elif not any(character.isalpha() for character in tag_name): - raise BadArgument("Tag names must contain at least one letter.") + async def convert(ctx: Context, argument: str) -> SourceType: + """Convert argument into source object.""" + if argument.lower() == "help": + return ctx.bot.help_command - return tag_name + cog = ctx.bot.get_cog(argument) + if cog: + return cog + cmd = ctx.bot.get_command(argument) + if cmd: + return cmd -class TagContentConverter(Converter): - """Ensure proposed tag content is not empty and contains at least one non-whitespace character.""" + tags_cog = ctx.bot.get_cog("Tags") + show_tag = True - @staticmethod - async def convert(ctx: Context, tag_content: str) -> str: - """ - Ensure tag_content is non-empty and contains at least one non-whitespace character. - - If tag_content is valid, return the stripped version. - """ - tag_content = tag_content.strip() - - # The tag contents should not be empty, or filled with whitespace. - if not tag_content: - raise BadArgument("Tag contents should not be empty, or filled with whitespace.") + if not tags_cog: + show_tag = False + else: + identifier = TagIdentifier.from_string(argument.lower()) + if identifier in tags_cog.tags: + return identifier + escaped_arg = escape_markdown(argument) - return tag_content + raise BadArgument( + f"Unable to convert '{escaped_arg}' to valid command{', tag,' if show_tag else ''} or Cog." + ) class DurationDelta(Converter): @@ -308,7 +337,7 @@ class DurationDelta(Converter): The units need to be provided in descending order of magnitude. """ - if not (delta := parse_duration_string(duration)): + if not (delta := time.parse_duration_string(duration)): raise BadArgument(f"`{duration}` is not a valid duration string.") return delta @@ -324,7 +353,7 @@ class Duration(DurationDelta): The converter supports the same symbols for each unit of time as its parent class. """ delta = await super().convert(ctx, duration) - now = datetime.utcnow() + now = datetime.now(timezone.utc) try: return now + delta @@ -332,10 +361,29 @@ class Duration(DurationDelta): raise BadArgument(f"`{duration}` results in a datetime outside the supported range.") +class Age(DurationDelta): + """Convert duration strings into UTC datetime.datetime objects.""" + + async def convert(self, ctx: Context, duration: str) -> datetime: + """ + Converts a `duration` string to a datetime object that's `duration` in the past. + + The converter supports the same symbols for each unit of time as its parent class. + """ + delta = await super().convert(ctx, duration) + now = datetime.now(timezone.utc) + + try: + return now - delta + except (ValueError, OverflowError): + raise BadArgument(f"`{duration}` results in a datetime outside the supported range.") + + class OffTopicName(Converter): """A converter that ensures an added off-topic name is valid.""" - ALLOWED_CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ!?'`-" + ALLOWED_CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ!?'`-<>" + TRANSLATED_CHARACTERS = "𝖠𝖡𝖢𝖣𝖤𝖥𝖦𝖧𝖨𝖩𝖪𝖫𝖬𝖭𝖮𝖯𝖰𝖱𝖲𝖳𝖴𝖵𝖶𝖷𝖸𝖹ǃ?’’-<>" @classmethod def translate_name(cls, name: str, *, from_unicode: bool = True) -> str: @@ -345,9 +393,9 @@ class OffTopicName(Converter): If `from_unicode` is True, the name is translated from a discord-safe format, back to normalized text. """ if from_unicode: - table = str.maketrans(cls.ALLOWED_CHARACTERS, '𝖠𝖡𝖢𝖣𝖤𝖥𝖦𝖧𝖨𝖩𝖪𝖫𝖬𝖭𝖮𝖯𝖰𝖱𝖲𝖳𝖴𝖵𝖶𝖷𝖸𝖹ǃ?’’-') + table = str.maketrans(cls.ALLOWED_CHARACTERS, cls.TRANSLATED_CHARACTERS) else: - table = str.maketrans('𝖠𝖡𝖢𝖣𝖤𝖥𝖦𝖧𝖨𝖩𝖪𝖫𝖬𝖭𝖮𝖯𝖰𝖱𝖲𝖳𝖴𝖵𝖶𝖷𝖸𝖹ǃ?’’-', cls.ALLOWED_CHARACTERS) + table = str.maketrans(cls.TRANSLATED_CHARACTERS, cls.ALLOWED_CHARACTERS) return name.translate(table) @@ -379,8 +427,8 @@ class ISODateTime(Converter): The converter is flexible in the formats it accepts, as it uses the `isoparse` method of `dateutil.parser`. In general, it accepts datetime strings that start with a date, optionally followed by a time. Specifying a timezone offset in the datetime string is - supported, but the `datetime` object will be converted to UTC and will be returned without - `tzinfo` as a timezone-unaware `datetime` object. + supported, but the `datetime` object will be converted to UTC. If no timezone is specified, the datetime will + be assumed to be in UTC already. In all cases, the returned object will have the UTC timezone. See: https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.isoparse @@ -405,8 +453,9 @@ class ISODateTime(Converter): raise BadArgument(f"`{datetime_string}` is not a valid ISO-8601 datetime string") if dt.tzinfo: - dt = dt.astimezone(dateutil.tz.UTC) - dt = dt.replace(tzinfo=None) + dt = dt.astimezone(timezone.utc) + else: # Without a timezone, assume it represents UTC. + dt = dt.replace(tzinfo=timezone.utc) return dt @@ -439,103 +488,51 @@ class HushDurationConverter(Converter): return duration -def proxy_user(user_id: str) -> discord.Object: - """ - Create a proxy user object from the given id. +def _is_an_unambiguous_user_argument(argument: str) -> bool: + """Check if the provided argument is a user mention, user id, or username (name#discrim).""" + has_id_or_mention = bool(IDConverter()._get_id_match(argument) or RE_USER_MENTION.match(argument)) - Used when a Member or User object cannot be resolved. - """ - log.trace(f"Attempting to create a proxy user for the user id {user_id}.") + # Check to see if the author passed a username (a discriminator exists) + argument = argument.removeprefix('@') + has_username = len(argument) > 5 and argument[-5] == '#' - try: - user_id = int(user_id) - except ValueError: - log.debug(f"Failed to create proxy user {user_id}: could not convert to int.") - raise BadArgument(f"User ID `{user_id}` is invalid - could not convert to an integer.") + return has_id_or_mention or has_username - user = discord.Object(user_id) - user.mention = user.id - user.display_name = f"<@{user.id}>" - user.avatar_url_as = lambda static_format: None - user.bot = False - return user +AMBIGUOUS_ARGUMENT_MSG = ("`{argument}` is not a User mention, a User ID or a Username in the format" + " `name#discriminator`.") -class UserMentionOrID(UserConverter): +class UnambiguousUser(UserConverter): """ - Converts to a `discord.User`, but only if a mention or userID is provided. + Converts to a `discord.User`, but only if a mention, userID or a username (name#discrim) is provided. - Unlike the default `UserConverter`, it doesn't allow conversion from a name or name#descrim. - This is useful in cases where that lookup strategy would lead to ambiguity. + Unlike the default `UserConverter`, it doesn't allow conversion from a name. + This is useful in cases where that lookup strategy would lead to too much ambiguity. """ async def convert(self, ctx: Context, argument: str) -> discord.User: - """Convert the `arg` to a `discord.User`.""" - match = self._get_id_match(argument) or RE_USER_MENTION.match(argument) - - if match is not None: + """Convert the `argument` to a `discord.User`.""" + if _is_an_unambiguous_user_argument(argument): return await super().convert(ctx, argument) else: - raise BadArgument(f"`{argument}` is not a User mention or a User ID.") + raise BadArgument(AMBIGUOUS_ARGUMENT_MSG.format(argument=argument)) -class FetchedUser(UserConverter): +class UnambiguousMember(MemberConverter): """ - Converts to a `discord.User` or, if it fails, a `discord.Object`. - - Unlike the default `UserConverter`, which only does lookups via the global user cache, this - converter attempts to fetch the user via an API call to Discord when the using the cache is - unsuccessful. - - If the fetch also fails and the error doesn't imply the user doesn't exist, then a - `discord.Object` is returned via the `user_proxy` converter. + Converts to a `discord.Member`, but only if a mention, userID or a username (name#discrim) is provided. - The lookup strategy is as follows (in order): - - 1. Lookup by ID. - 2. Lookup by mention. - 3. Lookup by name#discrim - 4. Lookup by name - 5. Lookup via API - 6. Create a proxy user with discord.Object - """ - - async def convert(self, ctx: Context, arg: str) -> t.Union[discord.User, discord.Object]: - """Convert the `arg` to a `discord.User` or `discord.Object`.""" - try: - return await super().convert(ctx, arg) - except BadArgument: - pass - - try: - user_id = int(arg) - log.trace(f"Fetching user {user_id}...") - return await ctx.bot.fetch_user(user_id) - except ValueError: - log.debug(f"Failed to fetch user {arg}: could not convert to int.") - raise BadArgument(f"The provided argument can't be turned into integer: `{arg}`") - except discord.HTTPException as e: - # If the Discord error isn't `Unknown user`, return a proxy instead - if e.code != 10013: - log.info(f"Failed to fetch user, returning a proxy instead: status {e.status}") - return proxy_user(arg) - - log.debug(f"Failed to fetch user {arg}: user does not exist.") - raise BadArgument(f"User `{arg}` does not exist") - - -def _snowflake_from_regex(pattern: t.Pattern, arg: str) -> int: - """ - Extract the snowflake from `arg` using a regex `pattern` and return it as an int. - - The snowflake is expected to be within the first capture group in `pattern`. + Unlike the default `MemberConverter`, it doesn't allow conversion from a name or nickname. + This is useful in cases where that lookup strategy would lead to too much ambiguity. """ - match = pattern.match(arg) - if not match: - raise BadArgument(f"Mention {str!r} is invalid.") - return int(match.group(1)) + async def convert(self, ctx: Context, argument: str) -> discord.Member: + """Convert the `argument` to a `discord.Member`.""" + if _is_an_unambiguous_user_argument(argument): + return await super().convert(ctx, argument) + else: + raise BadArgument(AMBIGUOUS_ARGUMENT_MSG.format(argument=argument)) class Infraction(Converter): @@ -554,7 +551,7 @@ class Infraction(Converter): "ordering": "-inserted_at" } - infractions = await ctx.bot.api_client.get("bot/infractions", params=params) + infractions = await ctx.bot.api_client.get("bot/infractions/expanded", params=params) if not infractions: raise BadArgument( @@ -564,9 +561,37 @@ class Infraction(Converter): return infractions[0] else: - return await ctx.bot.api_client.get(f"bot/infractions/{arg}") - + try: + return await ctx.bot.api_client.get(f"bot/infractions/{arg}/expanded") + except ResponseCodeError as e: + if e.status == 404: + raise InvalidInfraction( + converter=Infraction, + original=e, + infraction_arg=arg + ) + raise e + + +if t.TYPE_CHECKING: + ValidDiscordServerInvite = dict # noqa: F811 + ValidFilterListType = str # noqa: F811 + Extension = str # noqa: F811 + PackageName = str # noqa: F811 + ValidURL = str # noqa: F811 + Inventory = t.Tuple[str, _inventory_parser.InventoryDict] # noqa: F811 + Snowflake = int # noqa: F811 + SourceConverter = SourceType # noqa: F811 + DurationDelta = relativedelta # noqa: F811 + Duration = datetime # noqa: F811 + Age = datetime # noqa: F811 + OffTopicName = str # noqa: F811 + ISODateTime = datetime # noqa: F811 + HushDurationConverter = int # noqa: F811 + UnambiguousUser = discord.User # noqa: F811 + UnambiguousMember = discord.Member # noqa: F811 + Infraction = t.Optional[dict] # noqa: F811 Expiry = t.Union[Duration, ISODateTime] -FetchedMember = t.Union[discord.Member, FetchedUser] -UserMention = partial(_snowflake_from_regex, RE_USER_MENTION) +MemberOrUser = t.Union[discord.Member, discord.User] +UnambiguousMemberOrUser = t.Union[UnambiguousMember, UnambiguousUser] diff --git a/bot/decorators.py b/bot/decorators.py index f65ec4103..f4331264f 100644 --- a/bot/decorators.py +++ b/bot/decorators.py @@ -1,6 +1,5 @@ import asyncio import functools -import logging import types import typing as t from contextlib import suppress @@ -10,11 +9,12 @@ from discord.ext import commands from discord.ext.commands import Cog, Context from bot.constants import Channels, DEBUG_MODE, RedirectOutput -from bot.utils import function +from bot.log import get_logger +from bot.utils import function, scheduling from bot.utils.checks import ContextCheckFailure, in_whitelist_check from bot.utils.function import command_wraps -log = logging.getLogger(__name__) +log = get_logger(__name__) def in_whitelist( @@ -154,7 +154,7 @@ def redirect_output( if ping_user: await ctx.send(f"Here's the output of your command, {ctx.author.mention}") - asyncio.create_task(func(self, ctx, *args, **kwargs)) + scheduling.create_task(func(self, ctx, *args, **kwargs)) message = await old_channel.send( f"Hey, {ctx.author.mention}, you can find the output of your command here: " @@ -188,7 +188,7 @@ def respect_role_hierarchy(member_arg: function.Argument) -> t.Callable: """ def decorator(func: types.FunctionType) -> types.FunctionType: @command_wraps(func) - async def wrapper(*args, **kwargs) -> None: + async def wrapper(*args, **kwargs) -> t.Any: log.trace(f"{func.__name__}: respect role hierarchy decorator called") bound_args = function.get_bound_args(func, args, kwargs) @@ -196,8 +196,7 @@ def respect_role_hierarchy(member_arg: function.Argument) -> t.Callable: if not isinstance(target, Member): log.trace("The target is not a discord.Member; skipping role hierarchy check.") - await func(*args, **kwargs) - return + return await func(*args, **kwargs) ctx = function.get_arg_value(1, bound_args) cmd = ctx.command.name @@ -214,7 +213,7 @@ def respect_role_hierarchy(member_arg: function.Argument) -> t.Callable: ) else: log.trace(f"{func.__name__}: {target.top_role=} < {actor.top_role=}; calling func") - await func(*args, **kwargs) + return await func(*args, **kwargs) return wrapper return decorator diff --git a/bot/errors.py b/bot/errors.py index 46efb6d4f..078b645f1 100644 --- a/bot/errors.py +++ b/bot/errors.py @@ -1,6 +1,11 @@ -from typing import Hashable, Union +from __future__ import annotations -from discord import Member, User +from typing import Hashable, TYPE_CHECKING, Union + +from discord.ext.commands import ConversionError, Converter + +if TYPE_CHECKING: + from bot.converters import MemberOrUser class LockedResourceError(RuntimeError): @@ -30,14 +35,43 @@ class InvalidInfractedUserError(Exception): `user` -- User or Member which is invalid """ - def __init__(self, user: Union[Member, User], reason: str = "User infracted is a bot."): + def __init__(self, user: MemberOrUser, reason: str = "User infracted is a bot."): + self.user = user self.reason = reason super().__init__(reason) +class InvalidInfraction(ConversionError): + """ + Raised by the Infraction converter when trying to fetch an invalid infraction id. + + Attributes: + `infraction_arg` -- the value that we attempted to convert into an Infraction + """ + + def __init__(self, converter: Converter, original: Exception, infraction_arg: Union[int, str]): + + self.infraction_arg = infraction_arg + super().__init__(converter, original) + + class BrandingMisconfiguration(RuntimeError): """Raised by the Branding cog when a misconfigured event is encountered.""" pass + + +class NonExistentRoleError(ValueError): + """ + Raised by the Information Cog when encountering a Role that does not exist. + + Attributes: + `role_id` -- the ID of the role that does not exist + """ + + def __init__(self, role_id: int): + super().__init__(f"Could not fetch data for role {role_id}") + + self.role_id = role_id diff --git a/bot/exts/backend/branding/_cog.py b/bot/exts/backend/branding/_cog.py index 47c379a34..0c5839a7a 100644 --- a/bot/exts/backend/branding/_cog.py +++ b/bot/exts/backend/branding/_cog.py @@ -1,6 +1,5 @@ import asyncio import contextlib -import logging import random import typing as t from datetime import timedelta @@ -17,8 +16,10 @@ from bot.bot import Bot from bot.constants import Branding as BrandingConfig, Channels, Colours, Guild, MODERATION_ROLES from bot.decorators import mock_in_debug from bot.exts.backend.branding._repository import BrandingRepository, Event, RemoteObject +from bot.log import get_logger +from bot.utils import scheduling -log = logging.getLogger(__name__) +log = get_logger(__name__) class AssetType(Enum): @@ -50,7 +51,7 @@ def make_embed(title: str, description: str, *, success: bool) -> discord.Embed: For both `title` and `description`, empty string are valid values ~ fields will be empty. """ colour = Colours.soft_green if success else Colours.soft_red - return discord.Embed(title=title[:256], description=description[:2048], colour=colour) + return discord.Embed(title=title[:256], description=description[:4096], colour=colour) def extract_event_duration(event: Event) -> str: @@ -126,7 +127,7 @@ class Branding(commands.Cog): self.bot = bot self.repository = BrandingRepository(bot) - self.bot.loop.create_task(self.maybe_start_daemon()) # Start depending on cache. + scheduling.create_task(self.maybe_start_daemon(), event_loop=self.bot.loop) # Start depending on cache. # region: Internal logic & state management @@ -293,8 +294,8 @@ class Branding(commands.Cog): else: content = "Python Discord is entering a new event!" if is_notification else None - embed = discord.Embed(description=description[:2048], colour=discord.Colour.blurple()) - embed.set_footer(text=duration[:2048]) + embed = discord.Embed(description=description[:4096], colour=discord.Colour.og_blurple()) + embed.set_footer(text=duration[:4096]) await channel.send(content=content, embed=embed) @@ -572,7 +573,7 @@ class Branding(commands.Cog): await ctx.send(embed=resp) return - embed = discord.Embed(title="Current event calendar", colour=discord.Colour.blurple()) + embed = discord.Embed(title="Current event calendar", colour=discord.Colour.og_blurple()) # Because Discord embeds can only contain up to 25 fields, we only show the first 25. first_25 = list(available_events.items())[:25] diff --git a/bot/exts/backend/branding/_repository.py b/bot/exts/backend/branding/_repository.py index 7b09d4641..d88ea67f3 100644 --- a/bot/exts/backend/branding/_repository.py +++ b/bot/exts/backend/branding/_repository.py @@ -1,4 +1,3 @@ -import logging import typing as t from datetime import date, datetime @@ -7,6 +6,7 @@ import frontmatter from bot.bot import Bot from bot.constants import Keys from bot.errors import BrandingMisconfiguration +from bot.log import get_logger # Base URL for requests into the branding repository. BRANDING_URL = "https://api.github.com/repos/python-discord/branding/contents" @@ -25,7 +25,7 @@ ARBITRARY_YEAR = 2020 # Format used to parse date strings after we inject `ARBITRARY_YEAR` at the end. DATE_FMT = "%B %d %Y" # Ex: July 10 2020 -log = logging.getLogger(__name__) +log = get_logger(__name__) class RemoteObject: diff --git a/bot/exts/backend/config_verifier.py b/bot/exts/backend/config_verifier.py index d72c6c22e..dc85a65a2 100644 --- a/bot/exts/backend/config_verifier.py +++ b/bot/exts/backend/config_verifier.py @@ -1,12 +1,11 @@ -import logging - from discord.ext.commands import Cog from bot import constants from bot.bot import Bot +from bot.log import get_logger +from bot.utils import scheduling - -log = logging.getLogger(__name__) +log = get_logger(__name__) class ConfigVerifier(Cog): @@ -14,7 +13,7 @@ class ConfigVerifier(Cog): def __init__(self, bot: Bot): self.bot = bot - self.channel_verify_task = self.bot.loop.create_task(self.verify_channels()) + self.channel_verify_task = scheduling.create_task(self.verify_channels(), event_loop=self.bot.loop) async def verify_channels(self) -> None: """ diff --git a/bot/exts/backend/error_handler.py b/bot/exts/backend/error_handler.py index 578c372c3..c79c7b2a7 100644 --- a/bot/exts/backend/error_handler.py +++ b/bot/exts/backend/error_handler.py @@ -1,6 +1,4 @@ import difflib -import logging -import typing as t from discord import Embed from discord.ext.commands import ChannelNotFound, Cog, Context, TextChannelConverter, VoiceChannelConverter, errors @@ -9,11 +7,11 @@ from sentry_sdk import push_scope from bot.api import ResponseCodeError from bot.bot import Bot from bot.constants import Colours, Icons, MODERATION_ROLES -from bot.converters import TagNameConverter from bot.errors import InvalidInfractedUserError, LockedResourceError +from bot.log import get_logger from bot.utils.checks import ContextCheckFailure -log = logging.getLogger(__name__) +log = get_logger(__name__) class ErrorHandler(Cog): @@ -59,17 +57,23 @@ class ErrorHandler(Cog): log.trace(f"Command {command} had its error already handled locally; ignoring.") return + debug_message = ( + f"Command {command} invoked by {ctx.message.author} with error " + f"{e.__class__.__name__}: {e}" + ) + if isinstance(e, errors.CommandNotFound) and not getattr(ctx, "invoked_from_error_handler", False): if await self.try_silence(ctx): return - # Try to look for a tag with the command's name - await self.try_get_tag(ctx) - return # Exit early to avoid logging. + await self.try_get_tag(ctx) # Try to look for a tag with the command's name elif isinstance(e, errors.UserInputError): + log.debug(debug_message) await self.handle_user_input_error(ctx, e) elif isinstance(e, errors.CheckFailure): + log.debug(debug_message) await self.handle_check_failure(ctx, e) elif isinstance(e, errors.CommandOnCooldown): + log.debug(debug_message) await ctx.send(e) elif isinstance(e, errors.CommandInvokeError): if isinstance(e.original, ResponseCodeError): @@ -80,30 +84,25 @@ class ErrorHandler(Cog): await ctx.send(f"Cannot infract that user. {e.original.reason}") else: await self.handle_unexpected_error(ctx, e.original) - return # Exit early to avoid logging. elif isinstance(e, errors.ConversionError): if isinstance(e.original, ResponseCodeError): await self.handle_api_error(ctx, e.original) else: await self.handle_unexpected_error(ctx, e.original) - return # Exit early to avoid logging. - elif not isinstance(e, errors.DisabledCommand): + elif isinstance(e, errors.DisabledCommand): + log.debug(debug_message) + else: # MaxConcurrencyReached, ExtensionError await self.handle_unexpected_error(ctx, e) - return # Exit early to avoid logging. - log.debug( - f"Command {command} invoked by {ctx.message.author} with error " - f"{e.__class__.__name__}: {e}" - ) - - @staticmethod - def get_help_command(ctx: Context) -> t.Coroutine: + async def send_command_help(self, ctx: Context) -> None: """Return a prepared `help` command invocation coroutine.""" if ctx.command: - return ctx.send_help(ctx.command) + self.bot.help_command.context = ctx + await ctx.send_help(ctx.command) + return - return ctx.send_help() + await ctx.send_help() async def try_silence(self, ctx: Context) -> bool: """ @@ -174,23 +173,12 @@ class ErrorHandler(Cog): await self.on_command_error(ctx, tag_error) return - try: - tag_name = await TagNameConverter.convert(ctx, ctx.invoked_with) - except errors.BadArgument: - log.debug( - f"{ctx.author} tried to use an invalid command " - f"and the fallback tag failed validation in TagNameConverter." - ) - else: - if await ctx.invoke(tags_get_command, tag_name=tag_name): - return + if await ctx.invoke(tags_get_command, argument_string=ctx.message.content): + return if not any(role.id in MODERATION_ROLES for role in ctx.author.roles): await self.send_command_suggestion(ctx, ctx.invoked_with) - # Return to not raise the exception - return - async def send_command_suggestion(self, ctx: Context, command_name: str) -> None: """Sends user similar commands if any can be found.""" # No similar tag found, or tag on cooldown - @@ -235,38 +223,31 @@ class ErrorHandler(Cog): """ if isinstance(e, errors.MissingRequiredArgument): embed = self._get_error_embed("Missing required argument", e.param.name) - await ctx.send(embed=embed) - await self.get_help_command(ctx) self.bot.stats.incr("errors.missing_required_argument") elif isinstance(e, errors.TooManyArguments): embed = self._get_error_embed("Too many arguments", str(e)) - await ctx.send(embed=embed) - await self.get_help_command(ctx) self.bot.stats.incr("errors.too_many_arguments") elif isinstance(e, errors.BadArgument): embed = self._get_error_embed("Bad argument", str(e)) - await ctx.send(embed=embed) - await self.get_help_command(ctx) self.bot.stats.incr("errors.bad_argument") elif isinstance(e, errors.BadUnionArgument): embed = self._get_error_embed("Bad argument", f"{e}\n{e.errors[-1]}") - await ctx.send(embed=embed) - await self.get_help_command(ctx) self.bot.stats.incr("errors.bad_union_argument") elif isinstance(e, errors.ArgumentParsingError): embed = self._get_error_embed("Argument parsing error", str(e)) await ctx.send(embed=embed) - self.get_help_command(ctx).close() self.bot.stats.incr("errors.argument_parsing_error") + return else: embed = self._get_error_embed( "Input error", "Something about your input seems off. Check the arguments and try again." ) - await ctx.send(embed=embed) - await self.get_help_command(ctx) self.bot.stats.incr("errors.other_user_input_error") + await ctx.send(embed=embed) + await self.send_command_help(ctx) + @staticmethod async def handle_check_failure(ctx: Context, e: errors.CheckFailure) -> None: """ @@ -299,8 +280,8 @@ class ErrorHandler(Cog): async def handle_api_error(ctx: Context, e: ResponseCodeError) -> None: """Send an error message in `ctx` for ResponseCodeError and log it.""" if e.status == 404: - await ctx.send("There does not seem to be anything matching your query.") log.debug(f"API responded with 404 for command {ctx.command}") + await ctx.send("There does not seem to be anything matching your query.") ctx.bot.stats.incr("errors.api_error_404") elif e.status == 400: content = await e.response.json() @@ -308,12 +289,12 @@ class ErrorHandler(Cog): await ctx.send("According to the API, your request is malformed.") ctx.bot.stats.incr("errors.api_error_400") elif 500 <= e.status < 600: - await ctx.send("Sorry, there seems to be an internal issue with the API.") log.warning(f"API responded with {e.status} for command {ctx.command}") + await ctx.send("Sorry, there seems to be an internal issue with the API.") ctx.bot.stats.incr("errors.api_internal_server_error") else: - await ctx.send(f"Got an unexpected status code from the API (`{e.status}`).") log.warning(f"Unexpected API response for command {ctx.command}: {e.status}") + await ctx.send(f"Got an unexpected status code from the API (`{e.status}`).") ctx.bot.stats.incr(f"errors.api_error_{e.status}") @staticmethod diff --git a/bot/exts/backend/logging.py b/bot/exts/backend/logging.py index 823f14ea4..2d03cd580 100644 --- a/bot/exts/backend/logging.py +++ b/bot/exts/backend/logging.py @@ -1,13 +1,12 @@ -import logging - from discord import Embed from discord.ext.commands import Cog from bot.bot import Bot from bot.constants import Channels, DEBUG_MODE +from bot.log import get_logger +from bot.utils import scheduling - -log = logging.getLogger(__name__) +log = get_logger(__name__) class Logging(Cog): @@ -16,7 +15,7 @@ class Logging(Cog): def __init__(self, bot: Bot): self.bot = bot - self.bot.loop.create_task(self.startup_greeting()) + scheduling.create_task(self.startup_greeting(), event_loop=self.bot.loop) async def startup_greeting(self) -> None: """Announce our presence to the configured devlog channel.""" diff --git a/bot/exts/backend/sync/_cog.py b/bot/exts/backend/sync/_cog.py index 48d2b6f02..80f5750bc 100644 --- a/bot/exts/backend/sync/_cog.py +++ b/bot/exts/backend/sync/_cog.py @@ -1,4 +1,3 @@ -import logging from typing import Any, Dict from discord import Member, Role, User @@ -9,8 +8,10 @@ from bot import constants from bot.api import ResponseCodeError from bot.bot import Bot from bot.exts.backend.sync import _syncers +from bot.log import get_logger +from bot.utils import scheduling -log = logging.getLogger(__name__) +log = get_logger(__name__) class Sync(Cog): @@ -18,7 +19,7 @@ class Sync(Cog): def __init__(self, bot: Bot) -> None: self.bot = bot - self.bot.loop.create_task(self.sync_guild()) + scheduling.create_task(self.sync_guild(), event_loop=self.bot.loop) async def sync_guild(self) -> None: """Syncs the roles/users of the guild with the database.""" diff --git a/bot/exts/backend/sync/_syncers.py b/bot/exts/backend/sync/_syncers.py index c9f2d2da8..45301b098 100644 --- a/bot/exts/backend/sync/_syncers.py +++ b/bot/exts/backend/sync/_syncers.py @@ -1,5 +1,4 @@ import abc -import logging import typing as t from collections import namedtuple @@ -9,8 +8,10 @@ from more_itertools import chunked import bot from bot.api import ResponseCodeError +from bot.log import get_logger +from bot.utils.members import get_or_fetch_member -log = logging.getLogger(__name__) +log = get_logger(__name__) CHUNK_SIZE = 1000 @@ -156,7 +157,7 @@ class UserSyncer(Syncer): if db_user[db_field] != guild_value: updated_fields[db_field] = guild_value - if guild_user := guild.get_member(db_user["id"]): + if guild_user := await get_or_fetch_member(guild, db_user["id"]): seen_guild_users.add(guild_user.id) maybe_update("name", guild_user.name) diff --git a/bot/exts/events/__init__.py b/bot/exts/events/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/bot/exts/events/__init__.py diff --git a/bot/exts/events/code_jams/__init__.py b/bot/exts/events/code_jams/__init__.py new file mode 100644 index 000000000..16e81e365 --- /dev/null +++ b/bot/exts/events/code_jams/__init__.py @@ -0,0 +1,8 @@ +from bot.bot import Bot + + +def setup(bot: Bot) -> None: + """Load the CodeJams cog.""" + from bot.exts.events.code_jams._cog import CodeJams + + bot.add_cog(CodeJams(bot)) diff --git a/bot/exts/events/code_jams/_channels.py b/bot/exts/events/code_jams/_channels.py new file mode 100644 index 000000000..e8cf5f7bf --- /dev/null +++ b/bot/exts/events/code_jams/_channels.py @@ -0,0 +1,113 @@ +import typing as t + +import discord + +from bot.constants import Categories, Channels, Roles +from bot.log import get_logger + +log = get_logger(__name__) + +MAX_CHANNELS = 50 +CATEGORY_NAME = "Code Jam" + + +async def _get_category(guild: discord.Guild) -> discord.CategoryChannel: + """ + Return a code jam category. + + If all categories are full or none exist, create a new category. + """ + for category in guild.categories: + if category.name == CATEGORY_NAME and len(category.channels) < MAX_CHANNELS: + return category + + return await _create_category(guild) + + +async def _create_category(guild: discord.Guild) -> discord.CategoryChannel: + """Create a new code jam category and return it.""" + log.info("Creating a new code jam category.") + + category_overwrites = { + guild.default_role: discord.PermissionOverwrite(read_messages=False), + guild.me: discord.PermissionOverwrite(read_messages=True) + } + + category = await guild.create_category_channel( + CATEGORY_NAME, + overwrites=category_overwrites, + reason="It's code jam time!" + ) + + await _send_status_update( + guild, f"Created a new category with the ID {category.id} for this Code Jam's team channels." + ) + + return category + + +def _get_overwrites( + members: list[tuple[discord.Member, bool]], + guild: discord.Guild, +) -> dict[t.Union[discord.Member, discord.Role], discord.PermissionOverwrite]: + """Get code jam team channels permission overwrites.""" + team_channel_overwrites = { + guild.default_role: discord.PermissionOverwrite(read_messages=False), + guild.get_role(Roles.code_jam_event_team): discord.PermissionOverwrite(read_messages=True) + } + + for member, _ in members: + team_channel_overwrites[member] = discord.PermissionOverwrite( + read_messages=True + ) + + return team_channel_overwrites + + +async def create_team_channel( + guild: discord.Guild, + team_name: str, + members: list[tuple[discord.Member, bool]], + team_leaders: discord.Role +) -> None: + """Create the team's text channel.""" + await _add_team_leader_roles(members, team_leaders) + + # Get permission overwrites and category + team_channel_overwrites = _get_overwrites(members, guild) + code_jam_category = await _get_category(guild) + + # Create a text channel for the team + await code_jam_category.create_text_channel( + team_name, + overwrites=team_channel_overwrites, + ) + + +async def create_team_leader_channel(guild: discord.Guild, team_leaders: discord.Role) -> None: + """Create the Team Leader Chat channel for the Code Jam team leaders.""" + category: discord.CategoryChannel = guild.get_channel(Categories.summer_code_jam) + + team_leaders_chat = await category.create_text_channel( + name="team-leaders-chat", + overwrites={ + guild.default_role: discord.PermissionOverwrite(read_messages=False), + team_leaders: discord.PermissionOverwrite(read_messages=True) + } + ) + + await _send_status_update(guild, f"Created {team_leaders_chat.mention} in the {category} category.") + + +async def _send_status_update(guild: discord.Guild, message: str) -> None: + """Inform the events lead with a status update when the command is ran.""" + channel: discord.TextChannel = guild.get_channel(Channels.code_jam_planning) + + await channel.send(f"<@&{Roles.events_lead}>\n\n{message}") + + +async def _add_team_leader_roles(members: list[tuple[discord.Member, bool]], team_leaders: discord.Role) -> None: + """Assign the team leader role to the team leaders.""" + for member, is_leader in members: + if is_leader: + await member.add_roles(team_leaders) diff --git a/bot/exts/events/code_jams/_cog.py b/bot/exts/events/code_jams/_cog.py new file mode 100644 index 000000000..452199f5f --- /dev/null +++ b/bot/exts/events/code_jams/_cog.py @@ -0,0 +1,236 @@ +import asyncio +import csv +import typing as t +from collections import defaultdict + +import discord +from discord import Colour, Embed, Guild, Member +from discord.ext import commands + +from bot.bot import Bot +from bot.constants import Emojis, Roles +from bot.exts.events.code_jams import _channels +from bot.log import get_logger +from bot.utils.members import get_or_fetch_member +from bot.utils.services import send_to_paste_service + +log = get_logger(__name__) + +TEAM_LEADERS_COLOUR = 0x11806a +DELETION_REACTION = "\U0001f4a5" + + +class CodeJams(commands.Cog): + """Manages the code-jam related parts of our server.""" + + def __init__(self, bot: Bot): + self.bot = bot + + @commands.group(aliases=("cj", "jam")) + @commands.has_any_role(Roles.admins) + async def codejam(self, ctx: commands.Context) -> None: + """A Group of commands for managing Code Jams.""" + if ctx.invoked_subcommand is None: + await ctx.send_help(ctx.command) + + @codejam.command() + async def create(self, ctx: commands.Context, csv_file: t.Optional[str] = None) -> None: + """ + Create code-jam teams from a CSV file or a link to one, specifying the team names, leaders and members. + + The CSV file must have 3 columns: 'Team Name', 'Team Member Discord ID', and 'Team Leader'. + + This will create the text channels for the teams, and give the team leaders their roles. + """ + async with ctx.typing(): + if csv_file: + async with self.bot.http_session.get(csv_file) as response: + if response.status != 200: + await ctx.send(f"Got a bad response from the URL: {response.status}") + return + + csv_file = await response.text() + + elif ctx.message.attachments: + csv_file = (await ctx.message.attachments[0].read()).decode("utf8") + else: + raise commands.BadArgument("You must include either a CSV file or a link to one.") + + teams = defaultdict(list) + reader = csv.DictReader(csv_file.splitlines()) + + for row in reader: + member = await get_or_fetch_member(ctx.guild, int(row["Team Member Discord ID"])) + + if member is None: + log.trace(f"Got an invalid member ID: {row['Team Member Discord ID']}") + continue + + teams[row["Team Name"]].append((member, row["Team Leader"].upper() == "Y")) + + team_leaders = await ctx.guild.create_role(name="Code Jam Team Leaders", colour=TEAM_LEADERS_COLOUR) + + for team_name, team_members in teams.items(): + await _channels.create_team_channel(ctx.guild, team_name, team_members, team_leaders) + + await _channels.create_team_leader_channel(ctx.guild, team_leaders) + await ctx.send(f"{Emojis.check_mark} Created Code Jam with {len(teams)} teams.") + + @codejam.command() + @commands.has_any_role(Roles.admins) + async def end(self, ctx: commands.Context) -> None: + """ + Delete all code jam channels. + + A confirmation message is displayed with the categories and channels to be deleted.. Pressing the added reaction + deletes those channels. + """ + def predicate_deletion_emoji_reaction(reaction: discord.Reaction, user: discord.User) -> bool: + """Return True if the reaction :boom: was added by the context message author on this message.""" + return ( + reaction.message.id == message.id + and user.id == ctx.author.id + and str(reaction) == DELETION_REACTION + ) + + # A copy of the list of channels is stored. This is to make sure that we delete precisely the channels displayed + # in the confirmation message. + categories = self.jam_categories(ctx.guild) + category_channels = {category: category.channels.copy() for category in categories} + + confirmation_message = await self._build_confirmation_message(category_channels) + message = await ctx.send(confirmation_message) + await message.add_reaction(DELETION_REACTION) + try: + await self.bot.wait_for( + 'reaction_add', + check=predicate_deletion_emoji_reaction, + timeout=10 + ) + + except asyncio.TimeoutError: + await message.clear_reaction(DELETION_REACTION) + await ctx.send("Command timed out.", reference=message) + return + + else: + await message.clear_reaction(DELETION_REACTION) + for category, channels in category_channels.items(): + for channel in channels: + await channel.delete(reason="Code jam ended.") + await category.delete(reason="Code jam ended.") + + await message.add_reaction(Emojis.check_mark) + + @staticmethod + async def _build_confirmation_message( + categories: dict[discord.CategoryChannel, list[discord.abc.GuildChannel]] + ) -> str: + """Sends details of the channels to be deleted to the pasting service, and formats the confirmation message.""" + def channel_repr(channel: discord.abc.GuildChannel) -> str: + """Formats the channel name and ID and a readable format.""" + return f"{channel.name} ({channel.id})" + + def format_category_info(category: discord.CategoryChannel, channels: list[discord.abc.GuildChannel]) -> str: + """Displays the category and the channels within it in a readable format.""" + return f"{channel_repr(category)}:\n" + "\n".join(" - " + channel_repr(channel) for channel in channels) + + deletion_details = "\n\n".join( + format_category_info(category, channels) for category, channels in categories.items() + ) + + url = await send_to_paste_service(deletion_details) + if url is None: + url = "**Unable to send deletion details to the pasting service.**" + + return f"Are you sure you want to delete all code jam channels?\n\nThe channels to be deleted: {url}" + + @codejam.command() + @commands.has_any_role(Roles.admins, Roles.code_jam_event_team) + async def info(self, ctx: commands.Context, member: Member) -> None: + """ + Send an info embed about the member with the team they're in. + + The team is found by searching the permissions of the team channels. + """ + channel = self.team_channel(ctx.guild, member) + if not channel: + await ctx.send(":x: I can't find the team channel for this member.") + return + + embed = Embed( + title=str(member), + colour=Colour.og_blurple() + ) + embed.add_field(name="Team", value=self.team_name(channel), inline=True) + + await ctx.send(embed=embed) + + @codejam.command() + @commands.has_any_role(Roles.admins) + async def move(self, ctx: commands.Context, member: Member, new_team_name: str) -> None: + """Move participant from one team to another by changing the user's permissions for the relevant channels.""" + old_team_channel = self.team_channel(ctx.guild, member) + if not old_team_channel: + await ctx.send(":x: I can't find the team channel for this member.") + return + + if old_team_channel.name == new_team_name or self.team_name(old_team_channel) == new_team_name: + await ctx.send(f"`{member}` is already in `{new_team_name}`.") + return + + new_team_channel = self.team_channel(ctx.guild, new_team_name) + if not new_team_channel: + await ctx.send(f":x: I can't find a team channel named `{new_team_name}`.") + return + + await old_team_channel.set_permissions(member, overwrite=None, reason=f"Participant moved to {new_team_name}") + await new_team_channel.set_permissions( + member, + overwrite=discord.PermissionOverwrite(read_messages=True), + reason=f"Participant moved from {old_team_channel.name}" + ) + + await ctx.send( + f"Participant moved from `{self.team_name(old_team_channel)}` to `{self.team_name(new_team_channel)}`." + ) + + @codejam.command() + @commands.has_any_role(Roles.admins) + async def remove(self, ctx: commands.Context, member: Member) -> None: + """Remove the participant from their team. Does not remove the participants or leader roles.""" + channel = self.team_channel(ctx.guild, member) + if not channel: + await ctx.send(":x: I can't find the team channel for this member.") + return + + await channel.set_permissions( + member, + overwrite=None, + reason=f"Participant removed from the team {self.team_name(channel)}." + ) + await ctx.send(f"Removed the participant from `{self.team_name(channel)}`.") + + @staticmethod + def jam_categories(guild: Guild) -> list[discord.CategoryChannel]: + """Get all the code jam team categories.""" + return [category for category in guild.categories if category.name == _channels.CATEGORY_NAME] + + @staticmethod + def team_channel(guild: Guild, criterion: t.Union[str, Member]) -> t.Optional[discord.TextChannel]: + """Get a team channel through either a participant or the team name.""" + for category in CodeJams.jam_categories(guild): + for channel in category.channels: + if isinstance(channel, discord.TextChannel): + if ( + # If it's a string. + criterion == channel.name or criterion == CodeJams.team_name(channel) + # If it's a member. + or criterion in channel.overwrites + ): + return channel + + @staticmethod + def team_name(channel: discord.TextChannel) -> str: + """Retrieves the team name from the given channel.""" + return channel.name.replace("-", " ").title() diff --git a/bot/exts/filters/antimalware.py b/bot/exts/filters/antimalware.py index 4c4836c88..6cccf3680 100644 --- a/bot/exts/filters/antimalware.py +++ b/bot/exts/filters/antimalware.py @@ -1,4 +1,3 @@ -import logging import typing as t from os.path import splitext @@ -7,9 +6,10 @@ from discord.ext.commands import Cog from bot.bot import Bot from bot.constants import Channels, Filter, URLs -from bot.exts.utils.jams import CATEGORY_NAME as JAM_CATEGORY_NAME +from bot.exts.events.code_jams._channels import CATEGORY_NAME as JAM_CATEGORY_NAME +from bot.log import get_logger -log = logging.getLogger(__name__) +log = get_logger(__name__) PY_EMBED_DESCRIPTION = ( "It looks like you tried to attach a Python file - " @@ -18,14 +18,8 @@ PY_EMBED_DESCRIPTION = ( TXT_LIKE_FILES = {".txt", ".csv", ".json"} TXT_EMBED_DESCRIPTION = ( - "**Uh-oh!** It looks like your message got zapped by our spam filter. " - "We currently don't allow `{blocked_extension}` attachments, " - "so here are some tips to help you travel safely: \n\n" - "• If you attempted to send a message longer than 2000 characters, try shortening your message " - "to fit within the character limit or use a pasting service (see below) \n\n" - "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " - "{cmd_channel_mention} for more information) or use a pasting service like: " - f"\n\n{URLs.site_schema}{URLs.site_paste}" + "You either uploaded a `{blocked_extension}` file or entered a message that was too long. " + f"Please use our [paste bin]({URLs.site_schema}{URLs.site_paste}) instead." ) DISALLOWED_EMBED_DESCRIPTION = ( @@ -63,7 +57,7 @@ class AntiMalware(Cog): return # Ignore code jam channels - if hasattr(message.channel, "category") and message.channel.category.name == JAM_CATEGORY_NAME: + if getattr(message.channel, "category", None) and message.channel.category.name == JAM_CATEGORY_NAME: return # Check if user is staff, if is, return diff --git a/bot/exts/filters/antispam.py b/bot/exts/filters/antispam.py index 48c3aa5a6..bcd845a43 100644 --- a/bot/exts/filters/antispam.py +++ b/bot/exts/filters/antispam.py @@ -1,29 +1,30 @@ import asyncio -import logging +from collections import defaultdict from collections.abc import Mapping from dataclasses import dataclass, field -from datetime import datetime, timedelta +from datetime import timedelta +from itertools import takewhile from operator import attrgetter, itemgetter from typing import Dict, Iterable, List, Set +import arrow from discord import Colour, Member, Message, NotFound, Object, TextChannel from discord.ext.commands import Cog from bot import rules from bot.bot import Bot from bot.constants import ( - AntiSpam as AntiSpamConfig, Channels, - Colours, DEBUG_MODE, Event, Filter, - Guild as GuildConfig, Icons, + AntiSpam as AntiSpamConfig, Channels, Colours, DEBUG_MODE, Event, Filter, Guild as GuildConfig, Icons ) from bot.converters import Duration +from bot.exts.events.code_jams._channels import CATEGORY_NAME as JAM_CATEGORY_NAME from bot.exts.moderation.modlog import ModLog -from bot.exts.utils.jams import CATEGORY_NAME as JAM_CATEGORY_NAME +from bot.log import get_logger from bot.utils import lock, scheduling +from bot.utils.message_cache import MessageCache from bot.utils.messages import format_user, send_attachments - -log = logging.getLogger(__name__) +log = get_logger(__name__) RULE_FUNCTION_MAPPING = { 'attachments': rules.apply_attachments, @@ -44,19 +45,18 @@ RULE_FUNCTION_MAPPING = { class DeletionContext: """Represents a Deletion Context for a single spam event.""" - channel: TextChannel - members: Dict[int, Member] = field(default_factory=dict) + members: frozenset[Member] + triggered_in: TextChannel + channels: set[TextChannel] = field(default_factory=set) rules: Set[str] = field(default_factory=set) messages: Dict[int, Message] = field(default_factory=dict) attachments: List[List[str]] = field(default_factory=list) - async def add(self, rule_name: str, members: Iterable[Member], messages: Iterable[Message]) -> None: + async def add(self, rule_name: str, channels: Iterable[TextChannel], messages: Iterable[Message]) -> None: """Adds new rule violation events to the deletion context.""" self.rules.add(rule_name) - for member in members: - if member.id not in self.members: - self.members[member.id] = member + self.channels.update(channels) for message in messages: if message.id not in self.messages: @@ -69,36 +69,46 @@ class DeletionContext: async def upload_messages(self, actor_id: int, modlog: ModLog) -> None: """Method that takes care of uploading the queue and posting modlog alert.""" - triggered_by_users = ", ".join(format_user(m) for m in self.members.values()) + triggered_by_users = ", ".join(format_user(m) for m in self.members) + triggered_in_channel = f"**Triggered in:** {self.triggered_in.mention}\n" if len(self.channels) > 1 else "" + channels_description = ", ".join(channel.mention for channel in self.channels) mod_alert_message = ( f"**Triggered by:** {triggered_by_users}\n" - f"**Channel:** {self.channel.mention}\n" + f"{triggered_in_channel}" + f"**Channels:** {channels_description}\n" f"**Rules:** {', '.join(rule for rule in self.rules)}\n" ) - # For multiple messages or those with excessive newlines, use the logs API - if len(self.messages) > 1 or 'newlines' in self.rules: + messages_as_list = list(self.messages.values()) + first_message = messages_as_list[0] + # For multiple messages and those with attachments or excessive newlines, use the logs API + if any(( + len(messages_as_list) > 1, + len(first_message.attachments) > 0, + first_message.content.count('\n') > 15 + )): url = await modlog.upload_log(self.messages.values(), actor_id, self.attachments) mod_alert_message += f"A complete log of the offending messages can be found [here]({url})" else: mod_alert_message += "Message:\n" - [message] = self.messages.values() - content = message.clean_content - remaining_chars = 2040 - len(mod_alert_message) + content = first_message.clean_content + remaining_chars = 4080 - len(mod_alert_message) if len(content) > remaining_chars: - content = content[:remaining_chars] + "..." + url = await modlog.upload_log([first_message], actor_id, self.attachments) + log_site_msg = f"The full message can be found [here]({url})" + content = content[:remaining_chars - (3 + len(log_site_msg))] + "..." - mod_alert_message += f"{content}" + mod_alert_message += content - *_, last_message = self.messages.values() await modlog.send_log_message( + content=", ".join(str(m.id) for m in self.members), # quality-of-life improvement for mobile moderators icon_url=Icons.filtering, colour=Colour(Colours.soft_red), title="Spam detected!", text=mod_alert_message, - thumbnail=last_message.author.avatar_url_as(static_format="png"), + thumbnail=first_message.author.display_avatar.url, channel_id=Channels.mod_alerts, ping_everyone=AntiSpamConfig.ping_everyone ) @@ -116,7 +126,19 @@ class AntiSpam(Cog): self.message_deletion_queue = dict() - self.bot.loop.create_task(self.alert_on_validation_error(), name="AntiSpam.alert_on_validation_error") + # Fetch the rule configuration with the highest rule interval. + max_interval_config = max( + AntiSpamConfig.rules.values(), + key=itemgetter('interval') + ) + self.max_interval = max_interval_config['interval'] + self.cache = MessageCache(AntiSpamConfig.cache_size, newest_first=True) + + scheduling.create_task( + self.alert_on_validation_error(), + name="AntiSpam.alert_on_validation_error", + event_loop=self.bot.loop, + ) @property def mod_log(self) -> ModLog: @@ -149,35 +171,27 @@ class AntiSpam(Cog): not message.guild or message.guild.id != GuildConfig.id or message.author.bot - or (hasattr(message.channel, "category") and message.channel.category.name == JAM_CATEGORY_NAME) + or (getattr(message.channel, "category", None) and message.channel.category.name == JAM_CATEGORY_NAME) or (message.channel.id in Filter.channel_whitelist and not DEBUG_MODE) or (any(role.id in Filter.role_whitelist for role in message.author.roles) and not DEBUG_MODE) ): return - # Fetch the rule configuration with the highest rule interval. - max_interval_config = max( - AntiSpamConfig.rules.values(), - key=itemgetter('interval') - ) - max_interval = max_interval_config['interval'] + self.cache.append(message) - # Store history messages since `interval` seconds ago in a list to prevent unnecessary API calls. - earliest_relevant_at = datetime.utcnow() - timedelta(seconds=max_interval) - relevant_messages = [ - msg async for msg in message.channel.history(after=earliest_relevant_at, oldest_first=False) - if not msg.author.bot - ] + earliest_relevant_at = arrow.utcnow() - timedelta(seconds=self.max_interval) + relevant_messages = list(takewhile(lambda msg: msg.created_at > earliest_relevant_at, self.cache)) for rule_name in AntiSpamConfig.rules: rule_config = AntiSpamConfig.rules[rule_name] rule_function = RULE_FUNCTION_MAPPING[rule_name] # Create a list of messages that were sent in the interval that the rule cares about. - latest_interesting_stamp = datetime.utcnow() - timedelta(seconds=rule_config['interval']) - messages_for_rule = [ - msg for msg in relevant_messages if msg.created_at > latest_interesting_stamp - ] + latest_interesting_stamp = arrow.utcnow() - timedelta(seconds=rule_config['interval']) + messages_for_rule = list( + takewhile(lambda msg: msg.created_at > latest_interesting_stamp, relevant_messages) + ) + result = await rule_function(message, messages_for_rule, rule_config) # If the rule returns `None`, that means the message didn't violate it. @@ -190,19 +204,19 @@ class AntiSpam(Cog): full_reason = f"`{rule_name}` rule: {reason}" # If there's no spam event going on for this channel, start a new Message Deletion Context - channel = message.channel - if channel.id not in self.message_deletion_queue: - log.trace(f"Creating queue for channel `{channel.id}`") - self.message_deletion_queue[message.channel.id] = DeletionContext(channel) + authors_set = frozenset(members) + if authors_set not in self.message_deletion_queue: + log.trace(f"Creating queue for members `{authors_set}`") + self.message_deletion_queue[authors_set] = DeletionContext(authors_set, message.channel) scheduling.create_task( - self._process_deletion_context(message.channel.id), - name=f"AntiSpam._process_deletion_context({message.channel.id})" + self._process_deletion_context(authors_set), + name=f"AntiSpam._process_deletion_context({authors_set})" ) # Add the relevant of this trigger to the Deletion Context - await self.message_deletion_queue[message.channel.id].add( + await self.message_deletion_queue[authors_set].add( rule_name=rule_name, - members=members, + channels=set(message.channel for message in relevant_messages), messages=relevant_messages ) @@ -212,7 +226,7 @@ class AntiSpam(Cog): name=f"AntiSpam.punish(message={message.id}, member={member.id}, rule={rule_name})" ) - await self.maybe_delete_messages(channel, relevant_messages) + await self.maybe_delete_messages(relevant_messages) break @lock.lock_arg("antispam.punish", "member", attrgetter("id")) @@ -234,14 +248,31 @@ class AntiSpam(Cog): reason=reason ) - async def maybe_delete_messages(self, channel: TextChannel, messages: List[Message]) -> None: + async def maybe_delete_messages(self, messages: List[Message]) -> None: """Cleans the messages if cleaning is configured.""" if AntiSpamConfig.clean_offending: # If we have more than one message, we can use bulk delete. if len(messages) > 1: message_ids = [message.id for message in messages] self.mod_log.ignore(Event.message_delete, *message_ids) - await channel.delete_messages(messages) + channel_messages = defaultdict(list) + for message in messages: + channel_messages[message.channel].append(message) + for channel, messages in channel_messages.items(): + try: + await channel.delete_messages(messages) + except NotFound: + # In the rare case where we found messages matching the + # spam filter across multiple channels, it is possible + # that a single channel will only contain a single message + # to delete. If that should be the case, discord.py will + # use the "delete single message" endpoint instead of the + # bulk delete endpoint, and the single message deletion + # endpoint will complain if you give it that does not exist. + # As this means that we have no other message to delete in + # this channel (and message deletes work per-channel), + # we can just log an exception and carry on with business. + log.info(f"Tried to delete message `{messages[0].id}`, but message could not be found.") # Otherwise, the bulk delete endpoint will throw up. # Delete the message directly instead. @@ -252,7 +283,7 @@ class AntiSpam(Cog): except NotFound: log.info(f"Tried to delete message `{messages[0].id}`, but message could not be found.") - async def _process_deletion_context(self, context_id: int) -> None: + async def _process_deletion_context(self, context_id: frozenset) -> None: """Processes the Deletion Context queue.""" log.trace("Sleeping before processing message deletion queue.") await asyncio.sleep(10) @@ -264,6 +295,11 @@ class AntiSpam(Cog): deletion_context = self.message_deletion_queue.pop(context_id) await deletion_context.upload_messages(self.bot.user.id, self.mod_log) + @Cog.listener() + async def on_message_edit(self, before: Message, after: Message) -> None: + """Updates the message in the cache, if it's cached.""" + self.cache.update(after) + def validate_config(rules_: Mapping = AntiSpamConfig.rules) -> Dict[str, str]: """Validates the antispam configs.""" diff --git a/bot/exts/filters/filter_lists.py b/bot/exts/filters/filter_lists.py index 232c1e48b..a883ddf54 100644 --- a/bot/exts/filters/filter_lists.py +++ b/bot/exts/filters/filter_lists.py @@ -1,4 +1,4 @@ -import logging +import re from typing import Optional from discord import Colour, Embed @@ -7,10 +7,13 @@ from discord.ext.commands import BadArgument, Cog, Context, IDConverter, group, from bot import constants from bot.api import ResponseCodeError from bot.bot import Bot +from bot.constants import Channels from bot.converters import ValidDiscordServerInvite, ValidFilterListType +from bot.log import get_logger from bot.pagination import LinePaginator +from bot.utils import scheduling -log = logging.getLogger(__name__) +log = get_logger(__name__) class FilterLists(Cog): @@ -27,7 +30,7 @@ class FilterLists(Cog): def __init__(self, bot: Bot) -> None: self.bot = bot - self.bot.loop.create_task(self._amend_docstrings()) + scheduling.create_task(self._amend_docstrings(), event_loop=self.bot.loop) async def _amend_docstrings(self) -> None: """Add the valid FilterList types to the docstrings, so they'll appear in !help invocations.""" @@ -70,6 +73,18 @@ class FilterLists(Cog): elif list_type == "FILE_FORMAT" and not content.startswith("."): content = f".{content}" + # If it's a filter token, validate the passed regex + elif list_type == "FILTER_TOKEN": + try: + re.compile(content) + except re.error as e: + await ctx.message.add_reaction("❌") + await ctx.send( + f"{ctx.author.mention} that's not a valid regex! " + f"Regex error message: {e.msg}." + ) + return + # Try to add the item to the database log.trace(f"Trying to add the {content} item to the {list_type} {allow_type}") payload = { @@ -99,6 +114,12 @@ class FilterLists(Cog): ) raise + # If it is an autoban trigger we send a warning in #mod-meta + if comment and "[autoban]" in comment: + await self.bot.get_channel(Channels.mod_meta).send( + f":warning: Heads-up! The new filter `{content}` (`{comment}`) will automatically ban users." + ) + # Insert the item into the cache self.bot.insert_item_into_filter_list_cache(item) await ctx.message.add_reaction("✅") diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index 16aaf11cf..f44b28125 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -1,13 +1,16 @@ import asyncio -import logging import re -from datetime import datetime, timedelta +import unicodedata +from datetime import timedelta from typing import Any, Dict, List, Mapping, NamedTuple, Optional, Tuple, Union -import dateutil +import arrow +import dateutil.parser import discord.errors import regex +import tldextract from async_rediscache import RedisCache +from botcore.regex import DISCORD_INVITE from dateutil.relativedelta import relativedelta from discord import Colour, HTTPException, Member, Message, NotFound, TextChannel from discord.ext.commands import Cog @@ -15,17 +18,14 @@ from discord.utils import escape_markdown from bot.api import ResponseCodeError from bot.bot import Bot -from bot.constants import ( - Channels, Colours, Filter, - Guild, Icons, URLs -) +from bot.constants import Channels, Colours, Filter, Guild, Icons, URLs +from bot.exts.events.code_jams._channels import CATEGORY_NAME as JAM_CATEGORY_NAME from bot.exts.moderation.modlog import ModLog -from bot.exts.utils.jams import CATEGORY_NAME as JAM_CATEGORY_NAME +from bot.log import get_logger +from bot.utils import scheduling from bot.utils.messages import format_user -from bot.utils.regex import INVITE_RE -from bot.utils.scheduling import Scheduler -log = logging.getLogger(__name__) +log = get_logger(__name__) # Regular expressions CODE_BLOCK_RE = re.compile( @@ -46,6 +46,23 @@ ZALGO_RE = regex.compile(rf"[\p{{NONSPACING MARK}}\p{{ENCLOSING MARK}}--[{VARIAT DAYS_BETWEEN_ALERTS = 3 OFFENSIVE_MSG_DELETE_TIME = timedelta(days=Filter.offensive_msg_delete_days) +# Autoban +LINK_PASSWORD = "https://support.discord.com/hc/en-us/articles/218410947-I-forgot-my-Password-Where-can-I-set-a-new-one" +LINK_2FA = "https://support.discord.com/hc/en-us/articles/219576828-Setting-up-Two-Factor-Authentication" +AUTO_BAN_REASON = ( + "Your account has been used to send links to a phishing website. You have been automatically banned. " + "If you are not aware of sending them, that means your account has been compromised.\n\n" + + f"Here is a guide from Discord on [how to change your password]({LINK_PASSWORD}).\n\n" + + f"We also highly recommend that you [enable 2 factor authentication on your account]({LINK_2FA}), " + "for heightened security.\n\n" + + "Once you have changed your password, feel free to follow the instructions at the bottom of " + "this message to appeal your ban." +) +AUTO_BAN_DURATION = timedelta(days=4) + FilterMatch = Union[re.Match, dict, bool, List[discord.Embed]] @@ -64,7 +81,7 @@ class Filtering(Cog): def __init__(self, bot: Bot): self.bot = bot - self.scheduler = Scheduler(self.__class__.__name__) + self.scheduler = scheduling.Scheduler(self.__class__.__name__) self.name_lock = asyncio.Lock() staff_mistake_str = "If you believe this was a mistake, please let staff know!" @@ -133,7 +150,7 @@ class Filtering(Cog): }, } - self.bot.loop.create_task(self.reschedule_offensive_msg_deletion()) + scheduling.create_task(self.reschedule_offensive_msg_deletion(), event_loop=self.bot.loop) def cog_unload(self) -> None: """Cancel scheduled tasks.""" @@ -174,29 +191,43 @@ class Filtering(Cog): """ Invoke message filter for message edits. - If there have been multiple edits, calculate the time delta from the previous edit. + Also calculates the time delta from the previous edit or when message was sent if there's no prior edits. """ + # We only care about changes to the message contents/attachments and embed additions, not pin status etc. + if all(( + before.content == after.content, # content hasn't changed + before.attachments == after.attachments, # attachments haven't changed + len(before.embeds) >= len(after.embeds) # embeds haven't been added + )): + return + if not before.edited_at: delta = relativedelta(after.edited_at, before.created_at).microseconds else: delta = relativedelta(after.edited_at, before.edited_at).microseconds await self._filter_message(after, delta) - def get_name_matches(self, name: str) -> List[re.Match]: - """Check bad words from passed string (name). Return list of matches.""" - name = self.clean_input(name) - matches = [] + def get_name_match(self, name: str) -> Optional[re.Match]: + """Check bad words from passed string (name). Return the first match found.""" + normalised_name = unicodedata.normalize("NFKC", name) + cleaned_normalised_name = "".join([c for c in normalised_name if not unicodedata.combining(c)]) + + # Run filters against normalised, cleaned normalised and the original name, + # in case we have filters for one but not the other. + names_to_check = (name, normalised_name, cleaned_normalised_name) + watchlist_patterns = self._get_filterlist_items('filter_token', allowed=False) for pattern in watchlist_patterns: - if match := re.search(pattern, name, flags=re.IGNORECASE): - matches.append(match) - return matches + for name in names_to_check: + if match := re.search(pattern, name, flags=re.IGNORECASE): + return match + return None async def check_send_alert(self, member: Member) -> bool: """When there is less than 3 days after last alert, return `False`, otherwise `True`.""" if last_alert := await self.name_alerts.get(member.id): - last_alert = datetime.utcfromtimestamp(last_alert) - if datetime.utcnow() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert: + last_alert = arrow.get(last_alert) + if arrow.utcnow() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert: log.trace(f"Last alert was too recent for {member}'s nickname.") return False @@ -206,10 +237,14 @@ class Filtering(Cog): """Send a mod alert every 3 days if a username still matches a watchlist pattern.""" # Use lock to avoid race conditions async with self.name_lock: - # Check whether the users display name contains any words in our blacklist - matches = self.get_name_matches(member.display_name) + # Check if we recently alerted about this user first, + # to avoid running all the filter tokens against their name again. + if not await self.check_send_alert(member): + return - if not matches or not await self.check_send_alert(member): + # Check whether the users display name contains any words in our blacklist + match = self.get_name_match(member.display_name) + if not match: return log.info(f"Sending bad nickname alert for '{member.display_name}' ({member.id}).") @@ -217,20 +252,21 @@ class Filtering(Cog): log_string = ( f"**User:** {format_user(member)}\n" f"**Display Name:** {escape_markdown(member.display_name)}\n" - f"**Bad Matches:** {', '.join(match.group() for match in matches)}" + f"**Bad Match:** {match.group()}" ) await self.mod_log.send_log_message( + content=str(member.id), # quality-of-life improvement for mobile moderators icon_url=Icons.token_removed, colour=Colours.soft_red, title="Username filtering alert", text=log_string, channel_id=Channels.mod_alerts, - thumbnail=member.avatar_url + thumbnail=member.display_avatar.url ) # Update time when alert sent - await self.name_alerts.set(member.id, datetime.utcnow().timestamp()) + await self.name_alerts.set(member.id, arrow.utcnow().timestamp()) async def filter_eval(self, result: str, msg: Message) -> bool: """ @@ -326,7 +362,7 @@ class Filtering(Cog): await self.notify_member(msg.author, _filter["notification_msg"], msg.channel) # If the message is classed as offensive, we store it in the site db and - # it will be deleted it after one week. + # it will be deleted after one week. if _filter["schedule_deletion"] and not is_private: delete_date = (msg.created_at + OFFENSIVE_MSG_DELETE_TIME).isoformat() data = { @@ -349,6 +385,24 @@ class Filtering(Cog): stats = self._add_stats(filter_name, match, msg.content) await self._send_log(filter_name, _filter, msg, stats, reason) + # If the filter reason contains `[autoban]`, we want to auto-ban the user + if reason and "[autoban]" in reason.lower(): + # Create a new context, with the author as is the bot, and the channel as #mod-alerts. + # This sends the ban confirmation directly under watchlist trigger embed, to inform + # mods that the user was auto-banned for the message. + context = await self.bot.get_context(msg) + context.guild = self.bot.get_guild(Guild.id) + context.author = context.guild.get_member(self.bot.user.id) + context.channel = self.bot.get_channel(Channels.mod_alerts) + context.command = self.bot.get_command("tempban") + + await context.invoke( + context.command, + msg.author, + arrow.utcnow() + AUTO_BAN_DURATION, + reason=AUTO_BAN_REASON + ) + break # We don't want multiple filters to trigger async def _send_log( @@ -370,6 +424,13 @@ class Filtering(Cog): # Allow specific filters to override ping_everyone ping_everyone = Filter.ping_everyone and _filter.get("ping_everyone", True) + content = str(msg.author.id) # quality-of-life improvement for mobile moderators + + # If we are going to autoban, we don't want to ping and don't need the user ID + if reason and "[autoban]" in reason: + ping_everyone = False + content = None + eval_msg = "using !eval " if is_eval else "" footer = f"Reason: {reason}" if reason else None message = ( @@ -382,11 +443,12 @@ class Filtering(Cog): # Send pretty mod log embed to mod-alerts await self.mod_log.send_log_message( + content=content, icon_url=Icons.filtering, colour=Colour(Colours.soft_red), title=f"{_filter['type'].title()} triggered!", text=message, - thumbnail=msg.author.avatar_url_as(static_format="png"), + thumbnail=msg.author.display_avatar.url, channel_id=Channels.mod_alerts, ping_everyone=ping_everyone, additional_embeds=stats.additional_embeds, @@ -459,10 +521,6 @@ class Filtering(Cog): text = self.clean_input(text) - # Make sure it's not a URL - if URL_RE.search(text): - return False, None - watchlist_patterns = self._get_filterlist_items('filter_token', allowed=False) for pattern in watchlist_patterns: match = re.search(pattern, text, flags=re.IGNORECASE) @@ -478,16 +536,15 @@ class Filtering(Cog): Second return value is a reason of URL blacklisting (can be None). """ text = self.clean_input(text) - if not URL_RE.search(text): - return False, None - text = text.lower() domain_blacklist = self._get_filterlist_items("domain_name", allowed=False) - - for url in domain_blacklist: - if url.lower() in text: - return True, self._get_filterlist_value("domain_name", url, allowed=False)["comment"] - + for match in URL_RE.finditer(text): + for url in domain_blacklist: + if url.lower() in match.group(1).lower(): + blacklisted_parsed = tldextract.extract(url.lower()) + url_parsed = tldextract.extract(match.group(1).lower()) + if blacklisted_parsed.registered_domain == url_parsed.registered_domain: + return True, self._get_filterlist_value("domain_name", url, allowed=False)["comment"] return False, None @staticmethod @@ -514,7 +571,7 @@ class Filtering(Cog): # discord\.gg/gdudes-pony-farm text = text.replace("\\", "") - invites = INVITE_RE.findall(text) + invites = [m.group("invite") for m in DISCORD_INVITE.finditer(text)] invite_data = dict() for invite in invites: if invite in invite_data: @@ -610,7 +667,7 @@ class Filtering(Cog): def schedule_msg_delete(self, msg: dict) -> None: """Delete an offensive message once its deletion date is reached.""" - delete_at = dateutil.parser.isoparse(msg['delete_date']).replace(tzinfo=None) + delete_at = dateutil.parser.isoparse(msg['delete_date']) self.scheduler.schedule_at(delete_at, msg['id'], self.delete_offensive_msg(msg)) async def reschedule_offensive_msg_deletion(self) -> None: @@ -618,17 +675,17 @@ class Filtering(Cog): await self.bot.wait_until_ready() response = await self.bot.api_client.get('bot/offensive-messages',) - now = datetime.utcnow() + now = arrow.utcnow() for msg in response: - delete_at = dateutil.parser.isoparse(msg['delete_date']).replace(tzinfo=None) + delete_at = dateutil.parser.isoparse(msg['delete_date']) if delete_at < now: await self.delete_offensive_msg(msg) else: self.schedule_msg_delete(msg) - async def delete_offensive_msg(self, msg: Mapping[str, str]) -> None: + async def delete_offensive_msg(self, msg: Mapping[str, int]) -> None: """Delete an offensive message, and then delete it from the db.""" try: channel = self.bot.get_channel(msg['channel_id']) diff --git a/bot/exts/filters/security.py b/bot/exts/filters/security.py index c680c5e27..fe3918423 100644 --- a/bot/exts/filters/security.py +++ b/bot/exts/filters/security.py @@ -1,10 +1,9 @@ -import logging - from discord.ext.commands import Cog, Context, NoPrivateMessage from bot.bot import Bot +from bot.log import get_logger -log = logging.getLogger(__name__) +log = get_logger(__name__) class Security(Cog): diff --git a/bot/exts/filters/token_remover.py b/bot/exts/filters/token_remover.py index 93f1f3c33..520283ba3 100644 --- a/bot/exts/filters/token_remover.py +++ b/bot/exts/filters/token_remover.py @@ -1,6 +1,5 @@ import base64 import binascii -import logging import re import typing as t @@ -11,9 +10,11 @@ from bot import utils from bot.bot import Bot from bot.constants import Channels, Colours, Event, Icons from bot.exts.moderation.modlog import ModLog +from bot.log import get_logger +from bot.utils.members import get_or_fetch_member from bot.utils.messages import format_user -log = logging.getLogger(__name__) +log = get_logger(__name__) LOG_MESSAGE = ( "Censored a seemingly valid token sent by {author} in {channel}, " @@ -99,7 +100,7 @@ class TokenRemover(Cog): await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention)) log_message = self.format_log_message(msg, found_token) - userid_message, mention_everyone = self.format_userid_log_message(msg, found_token) + userid_message, mention_everyone = await self.format_userid_log_message(msg, found_token) log.debug(log_message) # Send pretty mod log embed to mod-alerts @@ -108,7 +109,7 @@ class TokenRemover(Cog): colour=Colour(Colours.soft_red), title="Token removed!", text=log_message + "\n" + userid_message, - thumbnail=msg.author.avatar_url_as(static_format="png"), + thumbnail=msg.author.display_avatar.url, channel_id=Channels.mod_alerts, ping_everyone=mention_everyone, ) @@ -116,7 +117,7 @@ class TokenRemover(Cog): self.bot.stats.incr("tokens.removed_tokens") @classmethod - def format_userid_log_message(cls, msg: Message, token: Token) -> t.Tuple[str, bool]: + async def format_userid_log_message(cls, msg: Message, token: Token) -> t.Tuple[str, bool]: """ Format the portion of the log message that includes details about the detected user ID. @@ -128,7 +129,7 @@ class TokenRemover(Cog): Returns a tuple of (log_message, mention_everyone) """ user_id = cls.extract_user_id(token.user_id) - user = msg.guild.get_member(user_id) + user = await get_or_fetch_member(msg.guild, user_id) if user: return KNOWN_USER_LOG_MESSAGE.format( diff --git a/bot/exts/filters/webhook_remover.py b/bot/exts/filters/webhook_remover.py index f11fc8912..96334317c 100644 --- a/bot/exts/filters/webhook_remover.py +++ b/bot/exts/filters/webhook_remover.py @@ -1,4 +1,3 @@ -import logging import re from discord import Colour, Message, NotFound @@ -7,18 +6,22 @@ from discord.ext.commands import Cog from bot.bot import Bot from bot.constants import Channels, Colours, Event, Icons from bot.exts.moderation.modlog import ModLog +from bot.log import get_logger from bot.utils.messages import format_user -WEBHOOK_URL_RE = re.compile(r"((?:https?://)?discord(?:app)?\.com/api/webhooks/\d+/)\S+/?", re.IGNORECASE) +WEBHOOK_URL_RE = re.compile( + r"((?:https?:\/\/)?(?:ptb\.|canary\.)?discord(?:app)?\.com\/api\/webhooks\/\d+\/)\S+\/?", + re.IGNORECASE +) ALERT_MESSAGE_TEMPLATE = ( "{user}, looks like you posted a Discord webhook URL. Therefore, your " - "message has been removed. Your webhook may have been **compromised** so " - "please re-create the webhook **immediately**. If you believe this was a " + "message has been removed, and your webhook has been deleted. " + "You can re-create it if you wish to. If you believe this was a " "mistake, please let us know." ) -log = logging.getLogger(__name__) +log = get_logger(__name__) class WebhookRemover(Cog): @@ -32,7 +35,7 @@ class WebhookRemover(Cog): """Get current instance of `ModLog`.""" return self.bot.get_cog("ModLog") - async def delete_and_respond(self, msg: Message, redacted_url: str) -> None: + async def delete_and_respond(self, msg: Message, redacted_url: str, *, webhook_deleted: bool) -> None: """Delete `msg` and send a warning that it contained the Discord webhook `redacted_url`.""" # Don't log this, due internal delete, not by user. Will make different entry. self.mod_log.ignore(Event.message_delete, msg.id) @@ -44,9 +47,12 @@ class WebhookRemover(Cog): return await msg.channel.send(ALERT_MESSAGE_TEMPLATE.format(user=msg.author.mention)) - + if webhook_deleted: + delete_state = "The webhook was successfully deleted." + else: + delete_state = "There was an error when deleting the webhook, it might have already been removed." message = ( - f"{format_user(msg.author)} posted a Discord webhook URL to {msg.channel.mention}. " + f"{format_user(msg.author)} posted a Discord webhook URL to {msg.channel.mention}. {delete_state} " f"Webhook URL was `{redacted_url}`" ) log.debug(message) @@ -57,7 +63,7 @@ class WebhookRemover(Cog): colour=Colour(Colours.soft_red), title="Discord webhook URL removed!", text=message, - thumbnail=msg.author.avatar_url_as(static_format="png"), + thumbnail=msg.author.display_avatar.url, channel_id=Channels.mod_alerts ) @@ -72,7 +78,10 @@ class WebhookRemover(Cog): matches = WEBHOOK_URL_RE.search(msg.content) if matches: - await self.delete_and_respond(msg, matches[1] + "xxx") + async with self.bot.http_session.delete(matches[0]) as resp: + # The Discord API Returns a 204 NO CONTENT response on success. + deleted_successfully = resp.status == 204 + await self.delete_and_respond(msg, matches[1] + "xxx", webhook_deleted=deleted_successfully) @Cog.listener() async def on_message_edit(self, before: Message, after: Message) -> None: diff --git a/bot/exts/fun/duck_pond.py b/bot/exts/fun/duck_pond.py index c78b9c141..c51656343 100644 --- a/bot/exts/fun/duck_pond.py +++ b/bot/exts/fun/duck_pond.py @@ -1,18 +1,20 @@ import asyncio -import logging from typing import Union import discord -from discord import Color, Embed, Member, Message, RawReactionActionEvent, TextChannel, User, errors +from discord import Color, Embed, Message, RawReactionActionEvent, TextChannel, errors from discord.ext.commands import Cog, Context, command from bot import constants from bot.bot import Bot +from bot.converters import MemberOrUser +from bot.log import get_logger +from bot.utils import scheduling from bot.utils.checks import has_any_role from bot.utils.messages import count_unique_users_reaction, send_attachments from bot.utils.webhooks import send_webhook -log = logging.getLogger(__name__) +log = get_logger(__name__) class DuckPond(Cog): @@ -23,7 +25,7 @@ class DuckPond(Cog): self.webhook_id = constants.Webhooks.duck_pond self.webhook = None self.ducked_messages = [] - self.bot.loop.create_task(self.fetch_webhook()) + scheduling.create_task(self.fetch_webhook(), event_loop=self.bot.loop) self.relay_lock = None async def fetch_webhook(self) -> None: @@ -36,7 +38,7 @@ class DuckPond(Cog): log.exception(f"Failed to fetch webhook with id `{self.webhook_id}`") @staticmethod - def is_staff(member: Union[User, Member]) -> bool: + def is_staff(member: MemberOrUser) -> bool: """Check if a specific member or user is staff.""" if hasattr(member, "roles"): for role in member.roles: @@ -92,7 +94,7 @@ class DuckPond(Cog): webhook=self.webhook, content=message.clean_content, username=message.author.display_name, - avatar_url=message.author.avatar_url + avatar_url=message.author.display_avatar.url ) if message.attachments: @@ -107,7 +109,7 @@ class DuckPond(Cog): webhook=self.webhook, embed=e, username=message.author.display_name, - avatar_url=message.author.avatar_url + avatar_url=message.author.display_avatar.url ) except discord.HTTPException: log.exception("Failed to send an attachment to the webhook") @@ -171,8 +173,14 @@ class DuckPond(Cog): if not self.is_helper_viewable(channel): return - message = await channel.fetch_message(payload.message_id) + try: + message = await channel.fetch_message(payload.message_id) + except discord.NotFound: + return # Message was deleted. + member = discord.utils.get(message.guild.members, id=payload.user_id) + if not member: + return # Member left or wasn't in the cache. # Was the message sent by a human staff member? if not self.is_staff(message.author) or message.author.bot: diff --git a/bot/exts/fun/off_topic_names.py b/bot/exts/fun/off_topic_names.py index 845b8175c..7df1d172d 100644 --- a/bot/exts/fun/off_topic_names.py +++ b/bot/exts/fun/off_topic_names.py @@ -1,7 +1,7 @@ import difflib -import logging -from datetime import datetime, timedelta +from datetime import timedelta +import arrow from discord import Colour, Embed from discord.ext.commands import Cog, Context, group, has_any_role from discord.utils import sleep_until @@ -10,10 +10,12 @@ from bot.api import ResponseCodeError from bot.bot import Bot from bot.constants import Channels, MODERATION_ROLES from bot.converters import OffTopicName +from bot.log import get_logger from bot.pagination import LinePaginator +from bot.utils import scheduling CHANNELS = (Channels.off_topic_0, Channels.off_topic_1, Channels.off_topic_2) -log = logging.getLogger(__name__) +log = get_logger(__name__) async def update_names(bot: Bot) -> None: @@ -21,9 +23,9 @@ async def update_names(bot: Bot) -> None: while True: # Since we truncate the compute timedelta to seconds, we add one second to ensure # we go past midnight in the `seconds_to_sleep` set below. - today_at_midnight = datetime.utcnow().replace(microsecond=0, second=0, minute=0, hour=0) + today_at_midnight = arrow.utcnow().replace(microsecond=0, second=0, minute=0, hour=0) next_midnight = today_at_midnight + timedelta(days=1) - await sleep_until(next_midnight) + await sleep_until(next_midnight.datetime) try: channel_0_name, channel_1_name, channel_2_name = await bot.api_client.get( @@ -50,7 +52,7 @@ class OffTopicNames(Cog): self.bot = bot self.updater_task = None - self.bot.loop.create_task(self.init_offtopic_updater()) + scheduling.create_task(self.init_offtopic_updater(), event_loop=self.bot.loop) def cog_unload(self) -> None: """Cancel any running updater tasks on cog unload.""" @@ -62,7 +64,7 @@ class OffTopicNames(Cog): await self.bot.wait_until_guild_available() if self.updater_task is None: coro = update_names(self.bot) - self.updater_task = self.bot.loop.create_task(coro) + self.updater_task = scheduling.create_task(coro, event_loop=self.bot.loop) @group(name='otname', aliases=('otnames', 'otn'), invoke_without_command=True) @has_any_role(*MODERATION_ROLES) diff --git a/bot/exts/help_channels/__init__.py b/bot/exts/help_channels/__init__.py index 781f40449..beba18aa6 100644 --- a/bot/exts/help_channels/__init__.py +++ b/bot/exts/help_channels/__init__.py @@ -1,10 +1,9 @@ -import logging - from bot import constants from bot.bot import Bot from bot.exts.help_channels._channel import MAX_CHANNELS_PER_CATEGORY +from bot.log import get_logger -log = logging.getLogger(__name__) +log = get_logger(__name__) def validate_config() -> None: diff --git a/bot/exts/help_channels/_channel.py b/bot/exts/help_channels/_channel.py index 0846b28c8..ff9e6a347 100644 --- a/bot/exts/help_channels/_channel.py +++ b/bot/exts/help_channels/_channel.py @@ -1,4 +1,4 @@ -import logging +import re import typing as t from datetime import timedelta from enum import Enum @@ -10,12 +10,14 @@ from arrow import Arrow import bot from bot import constants from bot.exts.help_channels import _caches, _message -from bot.utils.channel import try_get_channel +from bot.log import get_logger +from bot.utils.channel import get_or_fetch_channel -log = logging.getLogger(__name__) +log = get_logger(__name__) MAX_CHANNELS_PER_CATEGORY = 50 EXCLUDED_CHANNELS = (constants.Channels.cooldown,) +CLAIMED_BY_RE = re.compile(r"Channel claimed by <@!?(?P<user_id>\d{17,20})>\.$") class ClosingReason(Enum): @@ -133,7 +135,7 @@ async def move_to_bottom(channel: discord.TextChannel, category_id: int, **optio options should be avoided, as it may interfere with the category move we perform. """ # Get a fresh copy of the category from the bot to avoid the cache mismatch issue we had. - category = await try_get_channel(category_id) + category = await get_or_fetch_channel(category_id) payload = [{"id": c.id, "position": c.position} for c in category.channels] @@ -157,3 +159,35 @@ async def move_to_bottom(channel: discord.TextChannel, category_id: int, **optio # Now that the channel is moved, we can edit the other attributes if options: await channel.edit(**options) + + +async def ensure_cached_claimant(channel: discord.TextChannel) -> None: + """ + Ensure there is a claimant cached for each help channel. + + Check the redis cache first, return early if there is already a claimant cached. + If there isn't an entry in redis, search for the "Claimed by X." embed in channel history. + Stopping early if we discover a dormant message first. + + If a claimant could not be found, send a warning to #helpers and set the claimant to the bot. + """ + if await _caches.claimants.get(channel.id): + return + + async for message in channel.history(limit=1000): + if message.author.id != bot.instance.user.id: + # We only care about bot messages + continue + if message.embeds: + if _message._match_bot_embed(message, _message.DORMANT_MSG): + log.info("Hit the dormant message embed before finding a claimant in %s (%d).", channel, channel.id) + break + user_id = CLAIMED_BY_RE.match(message.embeds[0].description).group("user_id") + await _caches.claimants.set(channel.id, int(user_id)) + return + + await bot.instance.get_channel(constants.Channels.helpers).send( + f"I couldn't find a claimant for {channel.mention} in that last 1000 messages. " + "Please use your helper powers to close the channel if/when appropriate." + ) + await _caches.claimants.set(channel.id, bot.instance.user.id) diff --git a/bot/exts/help_channels/_cog.py b/bot/exts/help_channels/_cog.py index 35658d117..f276a7993 100644 --- a/bot/exts/help_channels/_cog.py +++ b/bot/exts/help_channels/_cog.py @@ -1,5 +1,4 @@ import asyncio -import logging import random import typing as t from datetime import timedelta @@ -14,9 +13,10 @@ from bot import constants from bot.bot import Bot from bot.constants import Channels, RedirectOutput from bot.exts.help_channels import _caches, _channel, _message, _name, _stats -from bot.utils import channel as channel_utils, lock, scheduling +from bot.log import get_logger +from bot.utils import channel as channel_utils, lock, members, scheduling -log = logging.getLogger(__name__) +log = get_logger(__name__) NAMESPACE = "help" HELP_CHANNEL_TOPIC = """ @@ -66,6 +66,9 @@ class HelpChannels(commands.Cog): self.bot = bot self.scheduler = scheduling.Scheduler(self.__class__.__name__) + self.guild: discord.Guild = None + self.cooldown_role: discord.Role = None + # Categories self.available_category: discord.CategoryChannel = None self.in_use_category: discord.CategoryChannel = None @@ -82,7 +85,7 @@ class HelpChannels(commands.Cog): # Asyncio stuff self.queue_tasks: t.List[asyncio.Task] = [] - self.init_task = self.bot.loop.create_task(self.init_cog()) + self.init_task = scheduling.create_task(self.init_cog(), event_loop=self.bot.loop) def cog_unload(self) -> None: """Cancel the init task and scheduled tasks when the cog unloads.""" @@ -95,24 +98,6 @@ class HelpChannels(commands.Cog): self.scheduler.cancel_all() - async def _handle_role_change(self, member: discord.Member, coro: t.Callable[..., t.Coroutine]) -> None: - """ - Change `member`'s cooldown role via awaiting `coro` and handle errors. - - `coro` is intended to be `discord.Member.add_roles` or `discord.Member.remove_roles`. - """ - try: - await coro(self.bot.get_guild(constants.Guild.id).get_role(constants.Roles.help_cooldown)) - except discord.NotFound: - log.debug(f"Failed to change role for {member} ({member.id}): member not found") - except discord.Forbidden: - log.debug( - f"Forbidden to change role for {member} ({member.id}); " - f"possibly due to role hierarchy" - ) - except discord.HTTPException as e: - log.error(f"Failed to change role for {member} ({member.id}): {e.status} {e.code}") - @lock.lock_arg(NAMESPACE, "message", attrgetter("channel.id")) @lock.lock_arg(NAMESPACE, "message", attrgetter("author.id")) @lock.lock_arg(f"{NAMESPACE}.unclaim", "message", attrgetter("author.id"), wait=True) @@ -120,19 +105,49 @@ class HelpChannels(commands.Cog): """ Claim the channel in which the question `message` was sent. - Move the channel to the In Use category and pin the `message`. Add a cooldown to the - claimant to prevent them from asking another question. Lastly, make a new channel available. + Send an embed stating the claimant, move the channel to the In Use category, and pin the `message`. + Add a cooldown to the claimant to prevent them from asking another question. + Lastly, make a new channel available. """ log.info(f"Channel #{message.channel} was claimed by `{message.author.id}`.") - await self.move_to_in_use(message.channel) - await self._handle_role_change(message.author, message.author.add_roles) - - await _message.pin(message) try: - await _message.dm_on_open(message) - except Exception as e: - log.warning("Error occurred while sending DM:", exc_info=e) + await self.move_to_in_use(message.channel) + except discord.DiscordServerError: + try: + await message.channel.send( + "The bot encountered a Discord API error while trying to move this channel, please try again later." + ) + except Exception as e: + log.warning("Error occurred while sending fail claim message:", exc_info=e) + log.info( + "500 error from Discord when moving #%s (%d) to in-use for %s (%d). Cancelling claim.", + message.channel.name, + message.channel.id, + message.author.name, + message.author.id, + ) + self.bot.stats.incr("help.failed_claims.500_on_move") + return + + embed = discord.Embed( + description=f"Channel claimed by {message.author.mention}.", + color=constants.Colours.bright_green, + ) + await message.channel.send(embed=embed) + + # Handle odd edge case of `message.author` not being a `discord.Member` (see bot#1839) + if not isinstance(message.author, discord.Member): + log.debug(f"{message.author} ({message.author.id}) isn't a member. Not giving cooldown role or sending DM.") + else: + await members.handle_role_change(message.author, message.author.add_roles, self.cooldown_role) + + try: + await _message.dm_on_open(message) + except Exception as e: + log.warning("Error occurred while sending DM:", exc_info=e) + + await _message.pin(message) # Add user with channel for dormant check. await _caches.claimants.set(message.channel.id, message.author.id) @@ -267,6 +282,8 @@ class HelpChannels(commands.Cog): for channel in channels[:abs(missing)]: await self.unclaim_channel(channel, closed_on=_channel.ClosingReason.CLEANUP) + self.available_help_channels = set(_channel.get_category_channels(self.available_category)) + # Getting channels that need to be included in the dynamic message. await self.update_available_help_channels() log.trace("Dynamic available help message updated.") @@ -276,13 +293,13 @@ class HelpChannels(commands.Cog): log.trace("Getting the CategoryChannel objects for the help categories.") try: - self.available_category = await channel_utils.try_get_channel( + self.available_category = await channel_utils.get_or_fetch_channel( constants.Categories.help_available ) - self.in_use_category = await channel_utils.try_get_channel( + self.in_use_category = await channel_utils.get_or_fetch_channel( constants.Categories.help_in_use ) - self.dormant_category = await channel_utils.try_get_channel( + self.dormant_category = await channel_utils.get_or_fetch_channel( constants.Categories.help_dormant ) except discord.HTTPException: @@ -295,6 +312,9 @@ class HelpChannels(commands.Cog): await self.bot.wait_until_guild_available() log.trace("Initialising the cog.") + self.guild = self.bot.get_guild(constants.Guild.id) + self.cooldown_role = self.guild.get_role(constants.Roles.help_cooldown) + await self.init_categories() self.channel_queue = self.create_channel_queue() @@ -306,6 +326,7 @@ class HelpChannels(commands.Cog): log.trace("Moving or rescheduling in-use channels.") for channel in _channel.get_category_channels(self.in_use_category): + await _channel.ensure_cached_claimant(channel) await self.move_idle_channel(channel, has_task=False) # Prevent the command from being used until ready. @@ -367,6 +388,12 @@ class HelpChannels(commands.Cog): log.trace(f"Moving #{channel} ({channel.id}) to the Available category.") + # Unpin any previously stuck pins + log.trace(f"Looking for pins stuck in #{channel} ({channel.id}).") + for message in await channel.pins(): + await _message.pin_wrapper(message.id, channel, pin=False) + log.debug(f"Removed a stuck pin from #{channel} ({channel.id}). ID: {message.id}") + await _channel.move_to_bottom( channel=channel, category_id=constants.Categories.help_available, @@ -387,7 +414,12 @@ class HelpChannels(commands.Cog): ) log.trace(f"Sending dormant message for #{channel} ({channel.id}).") - embed = discord.Embed(description=_message.DORMANT_MSG) + embed = discord.Embed( + description=_message.DORMANT_MSG.format( + dormant=self.dormant_category.name, + available=self.available_category.name, + ) + ) await channel.send(embed=embed) log.trace(f"Pushing #{channel} ({channel.id}) into the channel queue.") @@ -420,18 +452,21 @@ class HelpChannels(commands.Cog): async def _unclaim_channel( self, channel: discord.TextChannel, - claimant_id: int, + claimant_id: t.Optional[int], closed_on: _channel.ClosingReason ) -> None: """Actual implementation of `unclaim_channel`. See that for full documentation.""" await _caches.claimants.delete(channel.id) await _caches.session_participants.delete(channel.id) - claimant = self.bot.get_guild(constants.Guild.id).get_member(claimant_id) - if claimant is None: - log.info(f"{claimant_id} left the guild during their help session; the cooldown role won't be removed") + if not claimant_id: + log.info("No claimant given when un-claiming %s (%d). Skipping role removal.", channel, channel.id) else: - await self._handle_role_change(claimant, claimant.remove_roles) + claimant = await members.get_or_fetch_member(self.guild, claimant_id) + if claimant is None: + log.info(f"{claimant_id} left the guild during their help session; the cooldown role won't be removed") + else: + await members.handle_role_change(claimant, claimant.remove_roles, self.cooldown_role) await _message.unpin(channel) await _stats.report_complete_session(channel.id, closed_on) @@ -500,7 +535,7 @@ class HelpChannels(commands.Cog): """Wait for a dormant channel to become available in the queue and return it.""" log.trace("Waiting for a dormant channel.") - task = asyncio.create_task(self.channel_queue.get()) + task = scheduling.create_task(self.channel_queue.get()) self.queue_tasks.append(task) channel = await task @@ -511,11 +546,6 @@ class HelpChannels(commands.Cog): async def update_available_help_channels(self) -> None: """Updates the dynamic message within #how-to-get-help for available help channels.""" - if not self.available_help_channels: - self.available_help_channels = set( - c for c in self.available_category.channels if not _channel.is_excluded_channel(c) - ) - available_channels = AVAILABLE_HELP_CHANNELS.format( available=", ".join( c.mention for c in sorted(self.available_help_channels, key=attrgetter("position")) @@ -577,7 +607,7 @@ class HelpChannels(commands.Cog): embed = discord.Embed( title="Currently Helping", description=f"You're currently helping in {message.channel.mention}", - color=constants.Colours.soft_green, + color=constants.Colours.bright_green, timestamp=message.created_at ) embed.add_field(name="Conversation", value=f"[Jump to message]({message.jump_url})") diff --git a/bot/exts/help_channels/_message.py b/bot/exts/help_channels/_message.py index befacd263..241dd606c 100644 --- a/bot/exts/help_channels/_message.py +++ b/bot/exts/help_channels/_message.py @@ -1,4 +1,3 @@ -import logging import textwrap import typing as t @@ -9,8 +8,9 @@ from arrow import Arrow import bot from bot import constants from bot.exts.help_channels import _caches +from bot.log import get_logger -log = logging.getLogger(__name__) +log = get_logger(__name__) ASKING_GUIDE_URL = "https://pythondiscord.com/pages/asking-good-questions/" @@ -30,12 +30,12 @@ AVAILABLE_TITLE = "Available help channel" AVAILABLE_FOOTER = "Closes after a period of inactivity, or when you send !close." DORMANT_MSG = f""" -This help channel has been marked as **dormant**, and has been moved into the **Help: Dormant** \ +This help channel has been marked as **dormant**, and has been moved into the **{{dormant}}** \ category at the bottom of the channel list. It is no longer possible to send messages in this \ channel until it becomes available again. If your question wasn't answered yet, you can claim a new help channel from the \ -**Help: Available** category by simply asking your question again. Consider rephrasing the \ +**{{available}}** category by simply asking your question again. Consider rephrasing the \ question to maximize your chance of getting a good answer. If you're not sure how, have a look \ through our guide for **[asking a good question]({ASKING_GUIDE_URL})**. """ @@ -174,7 +174,7 @@ async def notify(channel: discord.TextChannel, last_notification: t.Optional[Arr async def pin(message: discord.Message) -> None: """Pin an initial question `message` and store it in a cache.""" - if await _pin_wrapper(message.id, message.channel, pin=True): + if await pin_wrapper(message.id, message.channel, pin=True): await _caches.question_messages.set(message.channel.id, message.id) @@ -205,7 +205,7 @@ async def unpin(channel: discord.TextChannel) -> None: if msg_id is None: log.debug(f"#{channel} ({channel.id}) doesn't have a message pinned.") else: - await _pin_wrapper(msg_id, channel, pin=False) + await pin_wrapper(msg_id, channel, pin=False) def _match_bot_embed(message: t.Optional[discord.Message], description: str) -> bool: @@ -220,7 +220,7 @@ def _match_bot_embed(message: t.Optional[discord.Message], description: str) -> return message.author == bot.instance.user and bot_msg_desc.strip() == description.strip() -async def _pin_wrapper(msg_id: int, channel: discord.TextChannel, *, pin: bool) -> bool: +async def pin_wrapper(msg_id: int, channel: discord.TextChannel, *, pin: bool) -> bool: """ Pin message `msg_id` in `channel` if `pin` is True or unpin if it's False. diff --git a/bot/exts/help_channels/_name.py b/bot/exts/help_channels/_name.py index 061f855ae..a9d9b2df1 100644 --- a/bot/exts/help_channels/_name.py +++ b/bot/exts/help_channels/_name.py @@ -1,5 +1,4 @@ import json -import logging import typing as t from collections import deque from pathlib import Path @@ -8,8 +7,9 @@ import discord from bot import constants from bot.exts.help_channels._channel import MAX_CHANNELS_PER_CATEGORY, get_category_channels +from bot.log import get_logger -log = logging.getLogger(__name__) +log = get_logger(__name__) def create_name_queue(*categories: discord.CategoryChannel) -> deque: diff --git a/bot/exts/help_channels/_stats.py b/bot/exts/help_channels/_stats.py index eb34e75e1..4698c26de 100644 --- a/bot/exts/help_channels/_stats.py +++ b/bot/exts/help_channels/_stats.py @@ -1,12 +1,11 @@ -import logging - from more_itertools import ilen import bot from bot import constants from bot.exts.help_channels import _caches, _channel +from bot.log import get_logger -log = logging.getLogger(__name__) +log = get_logger(__name__) def report_counts() -> None: diff --git a/bot/exts/info/code_snippets.py b/bot/exts/info/code_snippets.py index 24a9ae28a..07b1b8a2d 100644 --- a/bot/exts/info/code_snippets.py +++ b/bot/exts/info/code_snippets.py @@ -4,15 +4,16 @@ import textwrap from typing import Any from urllib.parse import quote_plus +import discord from aiohttp import ClientResponseError -from discord import Message from discord.ext.commands import Cog from bot.bot import Bot from bot.constants import Channels +from bot.log import get_logger from bot.utils.messages import wait_for_deletion -log = logging.getLogger(__name__) +log = get_logger(__name__) GITHUB_RE = re.compile( r'https://github\.com/(?P<repo>[a-zA-Z0-9-]+/[\w.-]+)/blob/' @@ -45,6 +46,17 @@ class CodeSnippets(Cog): Matches each message against a regex and prints the contents of all matched snippets. """ + def __init__(self, bot: Bot): + """Initializes the cog's bot.""" + self.bot = bot + + self.pattern_handlers = [ + (GITHUB_RE, self._fetch_github_snippet), + (GITHUB_GIST_RE, self._fetch_github_gist_snippet), + (GITLAB_RE, self._fetch_gitlab_snippet), + (BITBUCKET_RE, self._fetch_bitbucket_snippet) + ] + async def _fetch_response(self, url: str, response_format: str, **kwargs) -> Any: """Makes http requests using aiohttp.""" async with self.bot.http_session.get(url, raise_for_status=True, **kwargs) as response: @@ -208,56 +220,56 @@ class CodeSnippets(Cog): # Returns an empty codeblock if the snippet is empty return f'{ret}``` ```' - def __init__(self, bot: Bot): - """Initializes the cog's bot.""" - self.bot = bot + async def _parse_snippets(self, content: str) -> str: + """Parse message content and return a string with a code block for each URL found.""" + all_snippets = [] + + for pattern, handler in self.pattern_handlers: + for match in pattern.finditer(content): + try: + snippet = await handler(**match.groupdict()) + all_snippets.append((match.start(), snippet)) + except ClientResponseError as error: + error_message = error.message # noqa: B306 + log.log( + logging.DEBUG if error.status == 404 else logging.ERROR, + f'Failed to fetch code snippet from {match[0]!r}: {error.status} ' + f'{error_message} for GET {error.request_info.real_url.human_repr()}' + ) - self.pattern_handlers = [ - (GITHUB_RE, self._fetch_github_snippet), - (GITHUB_GIST_RE, self._fetch_github_gist_snippet), - (GITLAB_RE, self._fetch_gitlab_snippet), - (BITBUCKET_RE, self._fetch_bitbucket_snippet) - ] + # Sorts the list of snippets by their match index and joins them into a single message + return '\n'.join(map(lambda x: x[1], sorted(all_snippets))) @Cog.listener() - async def on_message(self, message: Message) -> None: + async def on_message(self, message: discord.Message) -> None: """Checks if the message has a snippet link, removes the embed, then sends the snippet contents.""" - if not message.author.bot: - all_snippets = [] - - for pattern, handler in self.pattern_handlers: - for match in pattern.finditer(message.content): - try: - snippet = await handler(**match.groupdict()) - all_snippets.append((match.start(), snippet)) - except ClientResponseError as error: - error_message = error.message # noqa: B306 - log.log( - logging.DEBUG if error.status == 404 else logging.ERROR, - f'Failed to fetch code snippet from {match[0]!r}: {error.status} ' - f'{error_message} for GET {error.request_info.real_url.human_repr()}' - ) - - # Sorts the list of snippets by their match index and joins them into a single message - message_to_send = '\n'.join(map(lambda x: x[1], sorted(all_snippets))) - - if 0 < len(message_to_send) <= 2000 and message_to_send.count('\n') <= 15: + if message.author.bot: + return + + message_to_send = await self._parse_snippets(message.content) + destination = message.channel + + if 0 < len(message_to_send) <= 2000 and message_to_send.count('\n') <= 15: + try: await message.edit(suppress=True) - if len(message_to_send) > 1000 and message.channel.id != Channels.bot_commands: - # Redirects to #bot-commands if the snippet contents are too long - await self.bot.wait_until_guild_available() - await message.channel.send(('The snippet you tried to send was too long. Please ' - f'see <#{Channels.bot_commands}> for the full snippet.')) - bot_commands_channel = self.bot.get_channel(Channels.bot_commands) - await wait_for_deletion( - await bot_commands_channel.send(message_to_send), - (message.author.id,) - ) - else: - await wait_for_deletion( - await message.channel.send(message_to_send), - (message.author.id,) - ) + except discord.NotFound: + # Don't send snippets if the original message was deleted. + return + + if len(message_to_send) > 1000 and message.channel.id != Channels.bot_commands: + # Redirects to #bot-commands if the snippet contents are too long + await self.bot.wait_until_guild_available() + destination = self.bot.get_channel(Channels.bot_commands) + + await message.channel.send( + 'The snippet you tried to send was too long. ' + f'Please see {destination.mention} for the full snippet.' + ) + + await wait_for_deletion( + await destination.send(message_to_send), + (message.author.id,) + ) def setup(bot: Bot) -> None: diff --git a/bot/exts/info/codeblock/_cog.py b/bot/exts/info/codeblock/_cog.py index 9094d9d15..a859d8cef 100644 --- a/bot/exts/info/codeblock/_cog.py +++ b/bot/exts/info/codeblock/_cog.py @@ -1,4 +1,3 @@ -import logging import time from typing import Optional @@ -11,11 +10,12 @@ from bot.bot import Bot from bot.exts.filters.token_remover import TokenRemover from bot.exts.filters.webhook_remover import WEBHOOK_URL_RE from bot.exts.info.codeblock._instructions import get_instructions -from bot.utils import has_lines +from bot.log import get_logger +from bot.utils import has_lines, scheduling from bot.utils.channel import is_help_channel from bot.utils.messages import wait_for_deletion -log = logging.getLogger(__name__) +log = get_logger(__name__) class CodeBlockCog(Cog, name="Code Block"): @@ -114,7 +114,7 @@ class CodeBlockCog(Cog, name="Code Block"): bot_message = await message.channel.send(f"Hey {message.author.mention}!", embed=embed) self.codeblock_message_ids[message.id] = bot_message.id - self.bot.loop.create_task(wait_for_deletion(bot_message, (message.author.id,))) + scheduling.create_task(wait_for_deletion(bot_message, (message.author.id,)), event_loop=self.bot.loop) # Increase amount of codeblock correction in stats self.bot.stats.incr("codeblock_corrections") @@ -177,10 +177,13 @@ class CodeBlockCog(Cog, name="Code Block"): if not bot_message: return - if not instructions: - log.info("User's incorrect code block has been fixed. Removing instructions message.") - await bot_message.delete() - del self.codeblock_message_ids[payload.message_id] - else: - log.info("Message edited but still has invalid code blocks; editing the instructions.") - await bot_message.edit(embed=self.create_embed(instructions)) + try: + if not instructions: + log.info("User's incorrect code block was fixed. Removing instructions message.") + await bot_message.delete() + del self.codeblock_message_ids[payload.message_id] + else: + log.info("Message edited but still has invalid code blocks; editing instructions.") + await bot_message.edit(embed=self.create_embed(instructions)) + except discord.NotFound: + log.debug("Could not find instructions message; it was probably deleted.") diff --git a/bot/exts/info/codeblock/_instructions.py b/bot/exts/info/codeblock/_instructions.py index dadb5e1ef..8fcadeec2 100644 --- a/bot/exts/info/codeblock/_instructions.py +++ b/bot/exts/info/codeblock/_instructions.py @@ -1,11 +1,11 @@ """This module generates and formats instructional messages about fixing Markdown code blocks.""" -import logging from typing import Optional from bot.exts.info.codeblock import _parsing +from bot.log import get_logger -log = logging.getLogger(__name__) +log = get_logger(__name__) _EXAMPLE_PY = "{lang}\nprint('Hello, world!')" # Make sure to escape any Markdown symbols here. _EXAMPLE_CODE_BLOCKS = ( diff --git a/bot/exts/info/codeblock/_parsing.py b/bot/exts/info/codeblock/_parsing.py index 73fd11b94..3c193d6c5 100644 --- a/bot/exts/info/codeblock/_parsing.py +++ b/bot/exts/info/codeblock/_parsing.py @@ -1,15 +1,15 @@ """This module provides functions for parsing Markdown code blocks.""" import ast -import logging import re import textwrap from typing import NamedTuple, Optional, Sequence from bot import constants +from bot.log import get_logger from bot.utils import has_lines -log = logging.getLogger(__name__) +log = get_logger(__name__) BACKTICK = "`" PY_LANG_CODES = ("python-repl", "python", "pycon", "py") # Order is important; "py" is last cause it's a subset. diff --git a/bot/exts/info/doc/__init__.py b/bot/exts/info/doc/__init__.py index 38a8975c0..facdf4d0b 100644 --- a/bot/exts/info/doc/__init__.py +++ b/bot/exts/info/doc/__init__.py @@ -1,4 +1,5 @@ from bot.bot import Bot + from ._redis_cache import DocRedisCache MAX_SIGNATURE_AMOUNT = 3 diff --git a/bot/exts/info/doc/_batch_parser.py b/bot/exts/info/doc/_batch_parser.py index 369bb462c..c27f28eac 100644 --- a/bot/exts/info/doc/_batch_parser.py +++ b/bot/exts/info/doc/_batch_parser.py @@ -2,7 +2,6 @@ from __future__ import annotations import asyncio import collections -import logging from collections import defaultdict from contextlib import suppress from operator import attrgetter @@ -13,20 +12,26 @@ from bs4 import BeautifulSoup import bot from bot.constants import Channels +from bot.log import get_logger from bot.utils import scheduling + from . import _cog, doc_cache from ._parsing import get_symbol_markdown +from ._redis_cache import StaleItemCounter -log = logging.getLogger(__name__) +log = get_logger(__name__) class StaleInventoryNotifier: """Handle sending notifications about stale inventories through `DocItem`s to dev log.""" + symbol_counter = StaleItemCounter() + def __init__(self): - self._init_task = bot.instance.loop.create_task( + self._init_task = scheduling.create_task( self._init_channel(), - name="StaleInventoryNotifier channel init" + name="StaleInventoryNotifier channel init", + event_loop=bot.instance.loop, ) self._warned_urls = set() @@ -38,13 +43,16 @@ class StaleInventoryNotifier: async def send_warning(self, doc_item: _cog.DocItem) -> None: """Send a warning to dev log if one wasn't already sent for `item`'s url.""" if doc_item.url not in self._warned_urls: - self._warned_urls.add(doc_item.url) - await self._init_task - embed = discord.Embed( - description=f"Doc item `{doc_item.symbol_id=}` present in loaded documentation inventories " - f"not found on [site]({doc_item.url}), inventories may need to be refreshed." - ) - await self._dev_log.send(embed=embed) + # Only warn if the item got less than 3 warnings + # or if it has been more than 3 weeks since the last warning + if await self.symbol_counter.increment_for(doc_item) < 3: + self._warned_urls.add(doc_item.url) + await self._init_task + embed = discord.Embed( + description=f"Doc item `{doc_item.symbol_id=}` present in loaded documentation inventories " + f"not found on [site]({doc_item.url}), inventories may need to be refreshed." + ) + await self._dev_log.send(embed=embed) class QueueItem(NamedTuple): @@ -101,7 +109,7 @@ class BatchParser: if doc_item not in self._item_futures and doc_item not in self._queue: self._item_futures[doc_item].user_requested = True - async with bot.instance.http_session.get(doc_item.url) as response: + async with bot.instance.http_session.get(doc_item.url, raise_for_status=True) as response: soup = await bot.instance.loop.run_in_executor( None, BeautifulSoup, diff --git a/bot/exts/info/doc/_cog.py b/bot/exts/info/doc/_cog.py index c54a3ee1c..4dc5276d9 100644 --- a/bot/exts/info/doc/_cog.py +++ b/bot/exts/info/doc/_cog.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio -import logging import sys import textwrap from collections import defaultdict @@ -13,17 +12,21 @@ import aiohttp import discord from discord.ext import commands +from bot.api import ResponseCodeError from bot.bot import Bot from bot.constants import MODERATION_ROLES, RedirectOutput from bot.converters import Inventory, PackageName, ValidURL, allowed_strings +from bot.log import get_logger from bot.pagination import LinePaginator +from bot.utils import scheduling from bot.utils.lock import SharedEvent, lock from bot.utils.messages import send_denial, wait_for_deletion from bot.utils.scheduling import Scheduler + from . import NAMESPACE, PRIORITY_PACKAGES, _batch_parser, doc_cache -from ._inventory_parser import InventoryDict, fetch_inventory +from ._inventory_parser import InvalidHeaderError, InventoryDict, fetch_inventory -log = logging.getLogger(__name__) +log = get_logger(__name__) # symbols with a group contained here will get the group prefixed on duplicates FORCE_PREFIX_GROUPS = ( @@ -75,9 +78,10 @@ class DocCog(commands.Cog): self.refresh_event.set() self.symbol_get_event = SharedEvent() - self.init_refresh_task = self.bot.loop.create_task( + self.init_refresh_task = scheduling.create_task( self.init_refresh_inventory(), - name="Doc inventory init" + name="Doc inventory init", + event_loop=self.bot.loop, ) @lock(NAMESPACE, COMMAND_LOCK_SINGLETON, raise_error=True) @@ -135,7 +139,12 @@ class DocCog(commands.Cog): The first attempt is rescheduled to execute in `FETCH_RESCHEDULE_DELAY.first` minutes, the subsequent attempts in `FETCH_RESCHEDULE_DELAY.repeated` minutes. """ - package = await fetch_inventory(inventory_url) + try: + package = await fetch_inventory(inventory_url) + except InvalidHeaderError as e: + # Do not reschedule if the header is invalid, as the request went through but the contents are invalid. + log.warning(f"Invalid inventory header at {inventory_url}. Reason: {e}") + return if not package: if api_package_name in self.inventory_scheduler: @@ -150,6 +159,8 @@ class DocCog(commands.Cog): self.update_or_reschedule_inventory(api_package_name, base_url, inventory_url), ) else: + if not base_url: + base_url = self.base_url_from_inventory_url(inventory_url) self.update_single(api_package_name, base_url, package) def ensure_unique_symbol_name(self, package_name: str, group_name: str, symbol_name: str) -> str: @@ -341,14 +352,22 @@ class DocCog(commands.Cog): if doc_embed is None: error_message = await send_denial(ctx, "No documentation found for the requested symbol.") await wait_for_deletion(error_message, (ctx.author.id,), timeout=NOT_FOUND_DELETE_DELAY) - with suppress(discord.NotFound): - await ctx.message.delete() - with suppress(discord.NotFound): - await error_message.delete() + + # Make sure that we won't cause a ghost-ping by deleting the message + if not (ctx.message.mentions or ctx.message.role_mentions): + with suppress(discord.NotFound): + await ctx.message.delete() + await error_message.delete() + else: msg = await ctx.send(embed=doc_embed) await wait_for_deletion(msg, (ctx.author.id,)) + @staticmethod + def base_url_from_inventory_url(inventory_url: str) -> str: + """Get a base url from the url to an objects inventory by removing the last path segment.""" + return inventory_url.removesuffix("/").rsplit("/", maxsplit=1)[0] + "/" + @docs_group.command(name="setdoc", aliases=("s",)) @commands.has_any_role(*MODERATION_ROLES) @lock(NAMESPACE, COMMAND_LOCK_SINGLETON, raise_error=True) @@ -356,21 +375,21 @@ class DocCog(commands.Cog): self, ctx: commands.Context, package_name: PackageName, - base_url: ValidURL, inventory: Inventory, + base_url: ValidURL = "", ) -> None: """ Adds a new documentation metadata object to the site's database. The database will update the object, should an existing item with the specified `package_name` already exist. + If the base url is not specified, a default created by removing the last segment of the inventory url is used. Example: !docs setdoc \ python \ - https://docs.python.org/3/ \ https://docs.python.org/3/objects.inv """ - if not base_url.endswith("/"): + if base_url and not base_url.endswith("/"): raise commands.BadArgument("The base url must end with a slash.") inventory_url, inventory_dict = inventory body = { @@ -378,13 +397,22 @@ class DocCog(commands.Cog): "base_url": base_url, "inventory_url": inventory_url } - await self.bot.api_client.post("bot/documentation-links", json=body) + try: + await self.bot.api_client.post("bot/documentation-links", json=body) + except ResponseCodeError as err: + if err.status == 400 and "already exists" in err.response_json.get("package", [""])[0]: + log.info(f"Ignoring HTTP 400 as package {package_name} has already been added.") + await ctx.send(f"Package {package_name} has already been added.") + return + raise log.info( f"User @{ctx.author} ({ctx.author.id}) added a new documentation package:\n" + "\n".join(f"{key}: {value}" for key, value in body.items()) ) + if not base_url: + base_url = self.base_url_from_inventory_url(inventory_url) self.update_single(package_name, base_url, inventory_dict) await ctx.send(f"Added the package `{package_name}` to the database and updated the inventories.") @@ -436,6 +464,7 @@ class DocCog(commands.Cog): ) -> None: """Clear the persistent redis cache for `package`.""" if await doc_cache.delete(package_name): + await self.item_fetcher.stale_inventory_notifier.symbol_counter.delete(package_name) await ctx.send(f"Successfully cleared the cache for `{package_name}`.") else: await ctx.send("No keys matching the package found.") @@ -444,4 +473,4 @@ class DocCog(commands.Cog): """Clear scheduled inventories, queued symbols and cleanup task on cog unload.""" self.inventory_scheduler.cancel_all() self.init_refresh_task.cancel() - asyncio.create_task(self.item_fetcher.clear(), name="DocCog.item_fetcher unload clear") + scheduling.create_task(self.item_fetcher.clear(), name="DocCog.item_fetcher unload clear") diff --git a/bot/exts/info/doc/_html.py b/bot/exts/info/doc/_html.py index 94efd81b7..ca0a0ac4a 100644 --- a/bot/exts/info/doc/_html.py +++ b/bot/exts/info/doc/_html.py @@ -1,4 +1,3 @@ -import logging import re from functools import partial from typing import Callable, Container, Iterable, List, Union @@ -6,9 +5,11 @@ from typing import Callable, Container, Iterable, List, Union from bs4 import BeautifulSoup from bs4.element import NavigableString, PageElement, SoupStrainer, Tag +from bot.log import get_logger + from . import MAX_SIGNATURE_AMOUNT -log = logging.getLogger(__name__) +log = get_logger(__name__) _UNWANTED_SIGNATURE_SYMBOLS_RE = re.compile(r"\[source]|\\\\|¶") _SEARCH_END_TAG_ATTRS = ( diff --git a/bot/exts/info/doc/_inventory_parser.py b/bot/exts/info/doc/_inventory_parser.py index 80d5841a0..e69246d47 100644 --- a/bot/exts/info/doc/_inventory_parser.py +++ b/bot/exts/info/doc/_inventory_parser.py @@ -1,4 +1,3 @@ -import logging import re import zlib from collections import defaultdict @@ -7,8 +6,9 @@ from typing import AsyncIterator, DefaultDict, List, Optional, Tuple import aiohttp import bot +from bot.log import get_logger -log = logging.getLogger(__name__) +log = get_logger(__name__) FAILED_REQUEST_ATTEMPTS = 3 _V2_LINE_RE = re.compile(r'(?x)(.+?)\s+(\S*:\S*)\s+(-?\d+)\s+?(\S*)\s+(.*)') @@ -16,6 +16,10 @@ _V2_LINE_RE = re.compile(r'(?x)(.+?)\s+(\S*:\S*)\s+(-?\d+)\s+?(\S*)\s+(.*)') InventoryDict = DefaultDict[str, List[Tuple[str, str]]] +class InvalidHeaderError(Exception): + """Raised when an inventory file has an invalid header.""" + + class ZlibStreamReader: """Class used for decoding zlib data of a stream line by line.""" @@ -80,19 +84,25 @@ async def _fetch_inventory(url: str) -> InventoryDict: stream = response.content inventory_header = (await stream.readline()).decode().rstrip() - inventory_version = int(inventory_header[-1:]) - await stream.readline() # skip project name - await stream.readline() # skip project version + try: + inventory_version = int(inventory_header[-1:]) + except ValueError: + raise InvalidHeaderError("Unable to convert inventory version header.") + + has_project_header = (await stream.readline()).startswith(b"# Project") + has_version_header = (await stream.readline()).startswith(b"# Version") + if not (has_project_header and has_version_header): + raise InvalidHeaderError("Inventory missing project or version header.") if inventory_version == 1: return await _load_v1(stream) elif inventory_version == 2: if b"zlib" not in await stream.readline(): - raise ValueError(f"Invalid inventory file at url {url}.") + raise InvalidHeaderError("'zlib' not found in header of compressed inventory.") return await _load_v2(stream) - raise ValueError(f"Invalid inventory file at url {url}.") + raise InvalidHeaderError("Incompatible inventory version.") async def fetch_inventory(url: str) -> Optional[InventoryDict]: @@ -115,6 +125,8 @@ async def fetch_inventory(url: str) -> Optional[InventoryDict]: f"Failed to get inventory from {url}; " f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." ) + except InvalidHeaderError: + raise except Exception: log.exception( f"An unexpected error has occurred during fetching of {url}; " diff --git a/bot/exts/info/doc/_parsing.py b/bot/exts/info/doc/_parsing.py index bf840b96f..6ab38eb3d 100644 --- a/bot/exts/info/doc/_parsing.py +++ b/bot/exts/info/doc/_parsing.py @@ -1,6 +1,5 @@ from __future__ import annotations -import logging import re import string import textwrap @@ -10,14 +9,17 @@ from typing import Collection, Iterable, Iterator, List, Optional, TYPE_CHECKING from bs4 import BeautifulSoup from bs4.element import NavigableString, Tag +from bot.log import get_logger from bot.utils.helpers import find_nth_occurrence + from . import MAX_SIGNATURE_AMOUNT from ._html import get_dd_description, get_general_description, get_signatures from ._markdown import DocMarkdownConverter + if TYPE_CHECKING: from ._cog import DocItem -log = logging.getLogger(__name__) +log = get_logger(__name__) _WHITESPACE_AFTER_NEWLINES_RE = re.compile(r"(?<=\n\n)(\s+)") _PARAMETERS_RE = re.compile(r"\((.+)\)") @@ -34,7 +36,7 @@ _EMBED_CODE_BLOCK_LINE_LENGTH = 61 # _MAX_SIGNATURE_AMOUNT code block wrapped lines with py syntax highlight _MAX_SIGNATURES_LENGTH = (_EMBED_CODE_BLOCK_LINE_LENGTH + 8) * MAX_SIGNATURE_AMOUNT # Maximum embed description length - signatures on top -_MAX_DESCRIPTION_LENGTH = 2048 - _MAX_SIGNATURES_LENGTH +_MAX_DESCRIPTION_LENGTH = 4096 - _MAX_SIGNATURES_LENGTH _TRUNCATE_STRIP_CHARACTERS = "!?:;." + string.whitespace BracketPair = namedtuple("BracketPair", ["opening_bracket", "closing_bracket"]) diff --git a/bot/exts/info/doc/_redis_cache.py b/bot/exts/info/doc/_redis_cache.py index ad764816f..107f2344f 100644 --- a/bot/exts/info/doc/_redis_cache.py +++ b/bot/exts/info/doc/_redis_cache.py @@ -4,6 +4,7 @@ import datetime from typing import Optional, TYPE_CHECKING from async_rediscache.types.base import RedisObject, namespace_lock + if TYPE_CHECKING: from ._cog import DocItem @@ -24,8 +25,7 @@ class DocRedisCache(RedisObject): All keys from a single page are stored together, expiring a week after the first set. """ - url_key = remove_suffix(item.relative_url_path, ".html") - redis_key = f"{self.namespace}:{item.package}:{url_key}" + redis_key = f"{self.namespace}:{item_key(item)}" needs_expire = False with await self._get_pool_connection() as connection: @@ -43,10 +43,36 @@ class DocRedisCache(RedisObject): @namespace_lock async def get(self, item: DocItem) -> Optional[str]: """Return the Markdown content of the symbol `item` if it exists.""" - url_key = remove_suffix(item.relative_url_path, ".html") + with await self._get_pool_connection() as connection: + return await connection.hget(f"{self.namespace}:{item_key(item)}", item.symbol_id, encoding="utf8") + + @namespace_lock + async def delete(self, package: str) -> bool: + """Remove all values for `package`; return True if at least one key was deleted, False otherwise.""" + with await self._get_pool_connection() as connection: + package_keys = [ + package_key async for package_key in connection.iscan(match=f"{self.namespace}:{package}:*") + ] + if package_keys: + await connection.delete(*package_keys) + return True + return False + +class StaleItemCounter(RedisObject): + """Manage increment counters for stale `DocItem`s.""" + + @namespace_lock + async def increment_for(self, item: DocItem) -> int: + """ + Increment the counter for `item` by 1, set it to expire in 3 weeks and return the new value. + + If the counter didn't exist, initialize it with 1. + """ + key = f"{self.namespace}:{item_key(item)}:{item.symbol_id}" with await self._get_pool_connection() as connection: - return await connection.hget(f"{self.namespace}:{item.package}:{url_key}", item.symbol_id, encoding="utf8") + await connection.expire(key, WEEK_SECONDS * 3) + return int(await connection.incr(key)) @namespace_lock async def delete(self, package: str) -> bool: @@ -61,10 +87,6 @@ class DocRedisCache(RedisObject): return False -def remove_suffix(string: str, suffix: str) -> str: - """Remove `suffix` from end of `string`.""" - # TODO replace usages with str.removesuffix on 3.9 - if string.endswith(suffix): - return string[:-len(suffix)] - else: - return string +def item_key(item: DocItem) -> str: + """Get the redis redis key string from `item`.""" + return f"{item.package}:{item.relative_url_path.removesuffix('.html')}" diff --git a/bot/exts/info/help.py b/bot/exts/info/help.py index 0235bbaf3..06799fb71 100644 --- a/bot/exts/info/help.py +++ b/bot/exts/info/help.py @@ -1,21 +1,24 @@ +from __future__ import annotations + import itertools -import logging +import re from collections import namedtuple from contextlib import suppress -from typing import List, Union +from typing import List, Optional, Union -from discord import Colour, Embed +from discord import ButtonStyle, Colour, Embed, Emoji, Interaction, PartialEmoji, ui from discord.ext.commands import Bot, Cog, Command, CommandError, Context, DisabledCommand, Group, HelpCommand from rapidfuzz import fuzz, process from rapidfuzz.utils import default_process from bot import constants -from bot.constants import Channels, STAFF_ROLES +from bot.constants import Channels, STAFF_PARTNERS_COMMUNITY_ROLES from bot.decorators import redirect_output +from bot.log import get_logger from bot.pagination import LinePaginator from bot.utils.messages import wait_for_deletion -log = logging.getLogger(__name__) +log = get_logger(__name__) COMMANDS_PER_PAGE = 8 PREFIX = constants.Bot.prefix @@ -25,6 +28,119 @@ NOT_ALLOWED_TO_RUN_MESSAGE = "***You cannot run this command.***\n\n" Category = namedtuple("Category", ["name", "description", "cogs"]) +class SubcommandButton(ui.Button): + """ + A button shown in a group's help embed. + + The button represents a subcommand, and pressing it will edit the help embed to that of the subcommand. + """ + + def __init__( + self, + help_command: CustomHelpCommand, + command: Command, + *, + style: ButtonStyle = ButtonStyle.primary, + label: Optional[str] = None, + disabled: bool = False, + custom_id: Optional[str] = None, + url: Optional[str] = None, + emoji: Optional[Union[str, Emoji, PartialEmoji]] = None, + row: Optional[int] = None + ): + super().__init__( + style=style, label=label, disabled=disabled, custom_id=custom_id, url=url, emoji=emoji, row=row + ) + + self.help_command = help_command + self.command = command + + async def callback(self, interaction: Interaction) -> None: + """Edits the help embed to that of the subcommand.""" + message = interaction.message + if not message: + return + + subcommand = self.command + if isinstance(subcommand, Group): + embed, subcommand_view = await self.help_command.format_group_help(subcommand) + else: + embed, subcommand_view = await self.help_command.command_formatting(subcommand) + await message.edit(embed=embed, view=subcommand_view) + + +class GroupButton(ui.Button): + """ + A button shown in a subcommand's help embed. + + The button represents the parent command, and pressing it will edit the help embed to that of the parent. + """ + + def __init__( + self, + help_command: CustomHelpCommand, + command: Command, + *, + style: ButtonStyle = ButtonStyle.secondary, + label: Optional[str] = None, + disabled: bool = False, + custom_id: Optional[str] = None, + url: Optional[str] = None, + emoji: Optional[Union[str, Emoji, PartialEmoji]] = None, + row: Optional[int] = None + ): + super().__init__( + style=style, label=label, disabled=disabled, custom_id=custom_id, url=url, emoji=emoji, row=row + ) + + self.help_command = help_command + self.command = command + + async def callback(self, interaction: Interaction) -> None: + """Edits the help embed to that of the parent.""" + message = interaction.message + if not message: + return + + embed, group_view = await self.help_command.format_group_help(self.command.parent) + await message.edit(embed=embed, view=group_view) + + +class CommandView(ui.View): + """ + The view added to any command's help embed. + + If the command has a parent, a button is added to the view to show that parent's help embed. + """ + + def __init__(self, help_command: CustomHelpCommand, command: Command): + super().__init__() + + if command.parent: + self.children.append(GroupButton(help_command, command, emoji="↩️")) + + +class GroupView(CommandView): + """ + The view added to a group's help embed. + + The view generates a SubcommandButton for every subcommand the group has. + """ + + MAX_BUTTONS_IN_ROW = 5 + MAX_ROWS = 5 + + def __init__(self, help_command: CustomHelpCommand, group: Group, subcommands: list[Command]): + super().__init__(help_command, group) + # Don't add buttons if only a portion of the subcommands can be shown. + if len(subcommands) + len(self.children) > self.MAX_ROWS * self.MAX_BUTTONS_IN_ROW: + log.trace(f"Attempted to add navigation buttons for `{group.qualified_name}`, but there was no space.") + return + + for subcommand in subcommands: + self.add_item(SubcommandButton(help_command, subcommand, label=subcommand.name)) + + class HelpQueryNotFound(ValueError): """ Raised when a HelpSession Query doesn't match a command or cog. @@ -54,7 +170,7 @@ class CustomHelpCommand(HelpCommand): def __init__(self): super().__init__(command_attrs={"help": "Shows help for bot commands"}) - @redirect_output(destination_channel=Channels.bot_commands, bypass_roles=STAFF_ROLES) + @redirect_output(destination_channel=Channels.bot_commands, bypass_roles=STAFF_PARTNERS_COMMUNITY_ROLES) async def command_callback(self, ctx: Context, *, command: str = None) -> None: """Attempts to match the provided query with a valid command or cog.""" # the only reason we need to tamper with this is because d.py does not support "categories", @@ -147,7 +263,7 @@ class CustomHelpCommand(HelpCommand): await self.context.send(embed=embed) - async def command_formatting(self, command: Command) -> Embed: + async def command_formatting(self, command: Command) -> tuple[Embed, Optional[CommandView]]: """ Takes a command and turns it into an embed. @@ -179,15 +295,20 @@ class CustomHelpCommand(HelpCommand): except CommandError: command_details += NOT_ALLOWED_TO_RUN_MESSAGE - command_details += f"*{command.help or 'No details provided.'}*\n" + # Remove line breaks from docstrings, if not used to separate paragraphs. + # Allow overriding this behaviour via putting \u2003 at the start of a line. + formatted_doc = re.sub("(?<!\n)\n(?![\n\u2003])", " ", command.help) + command_details += f"*{formatted_doc or 'No details provided.'}*\n" embed.description = command_details - return embed + # If the help is invoked in the context of an error, don't show subcommand navigation. + view = CommandView(self, command) if not self.context.command_failed else None + return embed, view async def send_command_help(self, command: Command) -> None: """Send help for a single command.""" - embed = await self.command_formatting(command) - message = await self.context.send(embed=embed) + embed, view = await self.command_formatting(command) + message = await self.context.send(embed=embed, view=view) await wait_for_deletion(message, (self.context.author.id,)) @staticmethod @@ -208,25 +329,31 @@ class CustomHelpCommand(HelpCommand): else: return "".join(details) - async def send_group_help(self, group: Group) -> None: - """Sends help for a group command.""" + async def format_group_help(self, group: Group) -> tuple[Embed, Optional[CommandView]]: + """Formats help for a group command.""" subcommands = group.commands if len(subcommands) == 0: # no subcommands, just treat it like a regular command - await self.send_command_help(group) - return + return await self.command_formatting(group) # remove commands that the user can't run and are hidden, and sort by name commands_ = await self.filter_commands(subcommands, sort=True) - embed = await self.command_formatting(group) + embed, _ = await self.command_formatting(group) command_details = self.get_commands_brief_details(commands_) if command_details: embed.description += f"\n**Subcommands:**\n{command_details}" - message = await self.context.send(embed=embed) + # If the help is invoked in the context of an error, don't show subcommand navigation. + view = GroupView(self, group, commands_) if not self.context.command_failed else None + return embed, view + + async def send_group_help(self, group: Group) -> None: + """Sends help for a group command.""" + embed, view = await self.format_group_help(group) + message = await self.context.send(embed=embed, view=view) await wait_for_deletion(message, (self.context.author.id,)) async def send_cog_help(self, cog: Cog) -> None: diff --git a/bot/exts/info/information.py b/bot/exts/info/information.py index fc3c2c61e..e616b9208 100644 --- a/bot/exts/info/information.py +++ b/bot/exts/info/information.py @@ -1,25 +1,29 @@ import colorsys -import logging import pprint import textwrap from collections import defaultdict -from typing import Any, DefaultDict, Dict, Mapping, Optional, Tuple, Union +from textwrap import shorten +from typing import Any, DefaultDict, Mapping, Optional, Tuple, Union import rapidfuzz from discord import AllowedMentions, Colour, Embed, Guild, Message, Role -from discord.ext.commands import BucketType, Cog, Context, Paginator, command, group, has_any_role +from discord.ext.commands import BucketType, Cog, Context, Greedy, Paginator, command, group, has_any_role +from discord.utils import escape_markdown from bot import constants from bot.api import ResponseCodeError from bot.bot import Bot -from bot.converters import FetchedMember +from bot.converters import MemberOrUser from bot.decorators import in_whitelist +from bot.errors import NonExistentRoleError +from bot.log import get_logger from bot.pagination import LinePaginator +from bot.utils import time from bot.utils.channel import is_mod_channel, is_staff_channel from bot.utils.checks import cooldown_with_role_bypass, has_no_roles_check, in_whitelist_check -from bot.utils.time import humanize_delta, time_since +from bot.utils.members import get_or_fetch_member -log = logging.getLogger(__name__) +log = get_logger(__name__) class Information(Cog): @@ -42,21 +46,36 @@ class Information(Cog): return channel_counter @staticmethod - def get_member_counts(guild: Guild) -> Dict[str, int]: + def join_role_stats(role_ids: list[int], guild: Guild, name: Optional[str] = None) -> dict[str, int]: + """Return a dictionary with the number of `members` of each role given, and the `name` for this joined group.""" + member_count = 0 + for role_id in role_ids: + if (role := guild.get_role(role_id)) is not None: + member_count += len(role.members) + else: + raise NonExistentRoleError(role_id) + return {name or role.name.title(): member_count} + + @staticmethod + def get_member_counts(guild: Guild) -> dict[str, int]: """Return the total number of members for certain roles in `guild`.""" - roles = ( - guild.get_role(role_id) for role_id in ( - constants.Roles.helpers, constants.Roles.moderators, constants.Roles.admins, - constants.Roles.owners, constants.Roles.contributors, - ) + role_ids = [constants.Roles.helpers, constants.Roles.mod_team, constants.Roles.admins, + constants.Roles.owners, constants.Roles.contributors] + + role_stats = {} + for role_id in role_ids: + role_stats.update(Information.join_role_stats([role_id], guild)) + role_stats.update( + Information.join_role_stats([constants.Roles.project_leads, constants.Roles.domain_leads], guild, "Leads") ) - return {role.name.title(): len(role.members) for role in roles} + return role_stats def get_extended_server_info(self, ctx: Context) -> str: """Return additional server info only visible in moderation channels.""" talentpool_info = "" if cog := self.bot.get_cog("Talentpool"): - talentpool_info = f"Nominated: {len(cog.watched_users)}\n" + num_nominated = len(cog.cache) if cog.cache else "-" + talentpool_info = f"Nominated: {num_nominated}\n" bb_info = "" if cog := self.bot.get_cog("Big Brother"): @@ -64,7 +83,7 @@ class Information(Cog): defcon_info = "" if cog := self.bot.get_cog("Defcon"): - threshold = humanize_delta(cog.threshold) if cog.threshold else "-" + threshold = time.humanize_delta(cog.threshold) if cog.threshold else "-" defcon_info = f"Defcon threshold: {threshold}\n" verification = f"Verification level: {ctx.guild.verification_level.name}\n" @@ -79,7 +98,7 @@ class Information(Cog): {python_general.mention} cooldown: {python_general.slowmode_delay}s """) - @has_any_role(*constants.STAFF_ROLES) + @has_any_role(*constants.STAFF_PARTNERS_COMMUNITY_ROLES) @command(name="roles") async def roles_info(self, ctx: Context) -> None: """Returns a list of all roles and their corresponding IDs.""" @@ -94,12 +113,12 @@ class Information(Cog): # Build an embed embed = Embed( title=f"Role information (Total {len(roles)} role{'s' * (len(role_list) > 1)})", - colour=Colour.blurple() + colour=Colour.og_blurple() ) await LinePaginator.paginate(role_list, ctx, embed, empty=False) - @has_any_role(*constants.STAFF_ROLES) + @has_any_role(*constants.STAFF_PARTNERS_COMMUNITY_ROLES) @command(name="role") async def role_info(self, ctx: Context, *roles: Union[Role, str]) -> None: """ @@ -152,15 +171,15 @@ class Information(Cog): @command(name="server", aliases=["server_info", "guild", "guild_info"]) async def server_info(self, ctx: Context) -> None: """Returns an embed full of server information.""" - embed = Embed(colour=Colour.blurple(), title="Server Information") + embed = Embed(colour=Colour.og_blurple(), title="Server Information") - created = time_since(ctx.guild.created_at, precision="days") - region = ctx.guild.region + created = time.format_relative(ctx.guild.created_at) num_roles = len(ctx.guild.roles) - 1 # Exclude @everyone # Server Features are only useful in certain channels if ctx.channel.id in ( - *constants.MODERATION_CHANNELS, constants.Channels.dev_core, constants.Channels.dev_contrib + *constants.MODERATION_CHANNELS, + constants.Channels.dev_core, ): features = f"\nFeatures: {', '.join(ctx.guild.features)}" else: @@ -171,21 +190,20 @@ class Information(Cog): online_presences = py_invite.approximate_presence_count offline_presences = py_invite.approximate_member_count - online_presences member_status = ( - f"{constants.Emojis.status_online} {online_presences} " - f"{constants.Emojis.status_offline} {offline_presences}" + f"{constants.Emojis.status_online} {online_presences:,} " + f"{constants.Emojis.status_offline} {offline_presences:,}" ) - embed.description = textwrap.dedent(f""" - Created: {created} - Voice region: {region}\ - {features} - Roles: {num_roles} - Member status: {member_status} - """) - embed.set_thumbnail(url=ctx.guild.icon_url) + embed.description = ( + f"Created: {created}" + f"{features}" + f"\nRoles: {num_roles}" + f"\nMember status: {member_status}" + ) + embed.set_thumbnail(url=ctx.guild.icon.url) # Members - total_members = ctx.guild.member_count + total_members = f"{ctx.guild.member_count:,}" member_counts = self.get_member_counts(ctx.guild) member_info = "\n".join(f"{role}: {count}" for role, count in member_counts.items()) embed.add_field(name=f"Members: {total_members}", value=member_info) @@ -205,8 +223,13 @@ class Information(Cog): await ctx.send(embed=embed) @command(name="user", aliases=["user_info", "member", "member_info", "u"]) - async def user_info(self, ctx: Context, user: FetchedMember = None) -> None: + async def user_info(self, ctx: Context, user_or_message: Union[MemberOrUser, Message] = None) -> None: """Returns info about a user.""" + if passed_as_message := isinstance(user_or_message, Message): + user = user_or_message.author + else: + user = user_or_message + if user is None: user = ctx.author @@ -216,19 +239,23 @@ class Information(Cog): return # Will redirect to #bot-commands if it fails. - if in_whitelist_check(ctx, roles=constants.STAFF_ROLES): - embed = await self.create_user_embed(ctx, user) + if in_whitelist_check(ctx, roles=constants.STAFF_PARTNERS_COMMUNITY_ROLES): + embed = await self.create_user_embed(ctx, user, passed_as_message) await ctx.send(embed=embed) - async def create_user_embed(self, ctx: Context, user: FetchedMember) -> Embed: + async def create_user_embed(self, ctx: Context, user: MemberOrUser, passed_as_message: bool) -> Embed: """Creates an embed containing information on the `user`.""" - on_server = bool(ctx.guild.get_member(user.id)) + on_server = bool(await get_or_fetch_member(ctx.guild, user.id)) - created = time_since(user.created_at, max_units=3) + created = time.format_relative(user.created_at) name = str(user) if on_server and user.nick: name = f"{user.nick} ({name})" + name = escape_markdown(name) + + if passed_as_message: + name += " - From Message" if user.public_flags.verified_bot: name += f" {constants.Emojis.verified_bot}" @@ -242,15 +269,20 @@ class Information(Cog): badges.append(emoji) if on_server: - joined = time_since(user.joined_at, max_units=3) - roles = ", ".join(role.mention for role in user.roles[1:]) + if user.joined_at: + joined = time.format_relative(user.joined_at) + else: + joined = "Unable to get join date" + + # The 0 is for excluding the default @everyone role, + # and the -1 is for reversing the order of the roles to highest to lowest in hierarchy. + roles = ", ".join(role.mention for role in user.roles[:0:-1]) membership = {"Joined": joined, "Verified": not user.pending, "Roles": roles or None} if not is_mod_channel(ctx.channel): membership.pop("Verified") membership = textwrap.dedent("\n".join([f"{key}: {value}" for key, value in membership.items()])) else: - roles = None membership = "The user is not a member of the server" fields = [ @@ -266,11 +298,11 @@ class Information(Cog): "Member information", membership ), + await self.user_messages(user), ] # Show more verbose output in moderation channels for infractions and nominations if is_mod_channel(ctx.channel): - fields.append(await self.user_messages(user)) fields.append(await self.expanded_user_infraction_counts(user)) fields.append(await self.user_nomination_counts(user)) else: @@ -285,12 +317,12 @@ class Information(Cog): for field_name, field_content in fields: embed.add_field(name=field_name, value=field_content, inline=False) - embed.set_thumbnail(url=user.avatar_url_as(static_format="png")) - embed.colour = user.colour if user.colour != Colour.default() else Colour.blurple() + embed.set_thumbnail(url=user.display_avatar.url) + embed.colour = user.colour if user.colour != Colour.default() else Colour.og_blurple() return embed - async def basic_user_infraction_counts(self, user: FetchedMember) -> Tuple[str, str]: + async def basic_user_infraction_counts(self, user: MemberOrUser) -> Tuple[str, str]: """Gets the total and active infraction counts for the given `member`.""" infractions = await self.bot.api_client.get( 'bot/infractions', @@ -307,7 +339,7 @@ class Information(Cog): return "Infractions", infraction_output - async def expanded_user_infraction_counts(self, user: FetchedMember) -> Tuple[str, str]: + async def expanded_user_infraction_counts(self, user: MemberOrUser) -> Tuple[str, str]: """ Gets expanded infraction counts for the given `member`. @@ -348,7 +380,7 @@ class Information(Cog): return "Infractions", "\n".join(infraction_output) - async def user_nomination_counts(self, user: FetchedMember) -> Tuple[str, str]: + async def user_nomination_counts(self, user: MemberOrUser) -> Tuple[str, str]: """Gets the active and historical nomination counts for the given `member`.""" nominations = await self.bot.api_client.get( 'bot/nominations', @@ -373,7 +405,7 @@ class Information(Cog): return "Nominations", "\n".join(output) - async def user_messages(self, user: FetchedMember) -> Tuple[Union[bool, str], Tuple[str, str]]: + async def user_messages(self, user: MemberOrUser) -> Tuple[Union[bool, str], Tuple[str, str]]: """ Gets the amount of messages for `member`. @@ -388,8 +420,8 @@ class Information(Cog): if e.status == 404: activity_output = "No activity" else: - activity_output.append(user_activity["total_messages"] or "No messages") - activity_output.append(user_activity["activity_blocks"] or "No activity") + activity_output.append(f"{user_activity['total_messages']:,}" or "No messages") + activity_output.append(f"{user_activity['activity_blocks']:,}" or "No activity") activity_output = "\n".join( f"{name}: {metric}" for name, metric in zip(["Messages", "Activity blocks"], activity_output) @@ -432,12 +464,13 @@ class Information(Cog): # remove trailing whitespace return out.rstrip() - @cooldown_with_role_bypass(2, 60 * 3, BucketType.member, bypass_roles=constants.STAFF_ROLES) - @group(invoke_without_command=True) - @in_whitelist(channels=(constants.Channels.bot_commands,), roles=constants.STAFF_ROLES) - async def raw(self, ctx: Context, *, message: Message, json: bool = False) -> None: - """Shows information about the raw API response.""" - if ctx.author not in message.channel.members: + async def send_raw_content(self, ctx: Context, message: Message, json: bool = False) -> None: + """ + Send information about the raw API response for a `discord.Message`. + + If `json` is True, send the information in a copy-pasteable Python format. + """ + if not message.channel.permissions_for(ctx.author).read_messages: await ctx.send(":x: You do not have permissions to see the channel this message is in.") return @@ -472,10 +505,51 @@ class Information(Cog): for page in paginator.pages: await ctx.send(page, allowed_mentions=AllowedMentions.none()) + @cooldown_with_role_bypass(2, 60 * 3, BucketType.member, bypass_roles=constants.STAFF_PARTNERS_COMMUNITY_ROLES) + @group(invoke_without_command=True) + @in_whitelist(channels=(constants.Channels.bot_commands,), roles=constants.STAFF_PARTNERS_COMMUNITY_ROLES) + async def raw(self, ctx: Context, message: Message) -> None: + """Shows information about the raw API response.""" + await self.send_raw_content(ctx, message) + @raw.command() async def json(self, ctx: Context, message: Message) -> None: """Shows information about the raw API response in a copy-pasteable Python format.""" - await ctx.invoke(self.raw, message=message, json=True) + await self.send_raw_content(ctx, message, json=True) + + @command(aliases=("rule",)) + async def rules(self, ctx: Context, rules: Greedy[int]) -> None: + """Provides a link to all rules or, if specified, displays specific rule(s).""" + rules_embed = Embed(title="Rules", color=Colour.og_blurple(), url="https://www.pythondiscord.com/pages/rules") + + if not rules: + # Rules were not submitted. Return the default description. + rules_embed.description = ( + "The rules and guidelines that apply to this community can be found on" + " our [rules page](https://www.pythondiscord.com/pages/rules). We expect" + " all members of the community to have read and understood these." + ) + + await ctx.send(embed=rules_embed) + return + + full_rules = await self.bot.api_client.get("rules", params={"link_format": "md"}) + + # Remove duplicates and sort the rule indices + rules = sorted(set(rules)) + + invalid = ", ".join(str(index) for index in rules if index < 1 or index > len(full_rules)) + + if invalid: + await ctx.send(shorten(":x: Invalid rule indices: " + invalid, 75, placeholder=" ...")) + return + + for rule in rules: + self.bot.stats.incr(f"rule_uses.{rule}") + + final_rules = tuple(f"**{pick}.** {full_rules[pick - 1]}" for pick in rules) + + await LinePaginator.paginate(final_rules, ctx, rules_embed, max_lines=3) def setup(bot: Bot) -> None: diff --git a/bot/exts/info/pep.py b/bot/exts/info/pep.py index 8ac96bbdb..67866620b 100644 --- a/bot/exts/info/pep.py +++ b/bot/exts/info/pep.py @@ -1,4 +1,3 @@ -import logging from datetime import datetime, timedelta from email.parser import HeaderParser from io import StringIO @@ -9,13 +8,15 @@ from discord.ext.commands import Cog, Context, command from bot.bot import Bot from bot.constants import Keys -from bot.utils.cache import AsyncCache +from bot.log import get_logger +from bot.utils import scheduling +from bot.utils.caching import AsyncCache -log = logging.getLogger(__name__) +log = get_logger(__name__) ICON_URL = "https://www.python.org/static/opengraph-icon-200x200.png" BASE_PEP_URL = "http://www.python.org/dev/peps/pep-" -PEPS_LISTING_API_URL = "https://api.github.com/repos/python/peps/contents?ref=master" +PEPS_LISTING_API_URL = "https://api.github.com/repos/python/peps/contents?ref=main" pep_cache = AsyncCache() @@ -32,7 +33,7 @@ class PythonEnhancementProposals(Cog): self.peps: Dict[int, str] = {} # To avoid situations where we don't have last datetime, set this to now. self.last_refreshed_peps: datetime = datetime.now() - self.bot.loop.create_task(self.refresh_peps_urls()) + scheduling.create_task(self.refresh_peps_urls(), event_loop=self.bot.loop) async def refresh_peps_urls(self) -> None: """Refresh PEP URLs listing in every 3 hours.""" @@ -96,9 +97,12 @@ class PythonEnhancementProposals(Cog): def generate_pep_embed(self, pep_header: Dict, pep_nr: int) -> Embed: """Generate PEP embed based on PEP headers data.""" + # the parsed header can be wrapped to multiple lines, so we need to make sure that is removed + # for an example of a pep with this issue, see pep 500 + title = " ".join(pep_header["Title"].split()) # Assemble the embed pep_embed = Embed( - title=f"**PEP {pep_nr} - {pep_header['Title']}**", + title=f"**PEP {pep_nr} - {title}**", description=f"[Link]({BASE_PEP_URL}{pep_nr:04})", ) diff --git a/bot/exts/info/pypi.py b/bot/exts/info/pypi.py index 2e42e7d6b..dacf7bc12 100644 --- a/bot/exts/info/pypi.py +++ b/bot/exts/info/pypi.py @@ -1,14 +1,16 @@ import itertools -import logging import random import re +from contextlib import suppress -from discord import Embed +from discord import Embed, NotFound from discord.ext.commands import Cog, Context, command from discord.utils import escape_markdown from bot.bot import Bot from bot.constants import Colours, NEGATIVE_REPLIES, RedirectOutput +from bot.log import get_logger +from bot.utils.messages import wait_for_deletion URL = "https://pypi.org/pypi/{package}/json" PYPI_ICON = "https://cdn.discordapp.com/emojis/766274397257334814.png" @@ -18,7 +20,7 @@ PYPI_COLOURS = itertools.cycle((Colours.yellow, Colours.blue, Colours.white)) ILLEGAL_CHARACTERS = re.compile(r"[^-_.a-zA-Z0-9]+") INVALID_INPUT_DELETE_DELAY = RedirectOutput.delete_delay -log = logging.getLogger(__name__) +log = get_logger(__name__) class PyPi(Cog): @@ -27,7 +29,7 @@ class PyPi(Cog): def __init__(self, bot: Bot): self.bot = bot - @command(name="pypi", aliases=("package", "pack")) + @command(name="pypi", aliases=("package", "pack", "pip")) async def get_package_info(self, ctx: Context, package: str) -> None: """Provide information about a specific package from PyPI.""" embed = Embed(title=random.choice(NEGATIVE_REPLIES), colour=Colours.soft_red) @@ -67,8 +69,15 @@ class PyPi(Cog): log.trace(f"Error when fetching PyPi package: {response.status}.") if error: - await ctx.send(embed=embed, delete_after=INVALID_INPUT_DELETE_DELAY) - await ctx.message.delete(delay=INVALID_INPUT_DELETE_DELAY) + error_message = await ctx.send(embed=embed) + await wait_for_deletion(error_message, (ctx.author.id,), timeout=INVALID_INPUT_DELETE_DELAY) + + # Make sure that we won't cause a ghost-ping by deleting the message + if not (ctx.message.mentions or ctx.message.role_mentions): + with suppress(NotFound): + await ctx.message.delete() + await error_message.delete() + else: await ctx.send(embed=embed) diff --git a/bot/exts/info/python_news.py b/bot/exts/info/python_news.py index 0ab5738a4..2fad9d2ab 100644 --- a/bot/exts/info/python_news.py +++ b/bot/exts/info/python_news.py @@ -1,4 +1,4 @@ -import logging +import re import typing as t from datetime import date, datetime @@ -10,6 +10,8 @@ from discord.ext.tasks import loop from bot import constants from bot.bot import Bot +from bot.log import get_logger +from bot.utils import scheduling from bot.utils.webhooks import send_webhook PEPS_RSS_URL = "https://www.python.org/dev/peps/peps.rss/" @@ -21,7 +23,15 @@ THREAD_URL = "https://mail.python.org/archives/list/{list}@python.org/thread/{id AVATAR_URL = "https://www.python.org/static/opengraph-icon-200x200.png" -log = logging.getLogger(__name__) +# By first matching everything within a codeblock, +# when matching markdown it won't be within a codeblock +MARKDOWN_REGEX = re.compile( + r"(?P<codeblock>`.*?`)" # matches everything within a codeblock + r"|(?P<markdown>(?<!\\)[_|])", # matches unescaped `_` and `|` + re.DOTALL # required to support multi-line codeblocks +) + +log = get_logger(__name__) class PythonNews(Cog): @@ -32,8 +42,8 @@ class PythonNews(Cog): self.webhook_names = {} self.webhook: t.Optional[discord.Webhook] = None - self.bot.loop.create_task(self.get_webhook_names()) - self.bot.loop.create_task(self.get_webhook_and_channel()) + scheduling.create_task(self.get_webhook_names(), event_loop=self.bot.loop) + scheduling.create_task(self.get_webhook_and_channel(), event_loop=self.bot.loop) async def start_tasks(self) -> None: """Start the tasks for fetching new PEPs and mailing list messages.""" @@ -72,6 +82,14 @@ class PythonNews(Cog): if mail["name"].split("@")[0] in constants.PythonNews.mail_lists: self.webhook_names[mail["name"].split("@")[0]] = mail["display_name"] + @staticmethod + def escape_markdown(content: str) -> str: + """Escape the markdown underlines and spoilers that aren't in codeblocks.""" + return MARKDOWN_REGEX.sub( + lambda match: match.group("codeblock") or "\\" + match.group("markdown"), + content + ) + async def post_pep_news(self) -> None: """Fetch new PEPs and when they don't have announcement in #python-news, create it.""" # Wait until everything is ready and http_session available @@ -102,8 +120,8 @@ class PythonNews(Cog): # Build an embed and send a webhook embed = discord.Embed( - title=new["title"], - description=new["summary"], + title=self.escape_markdown(new["title"]), + description=self.escape_markdown(new["summary"]), timestamp=new_datetime, url=new["link"], colour=constants.Colours.soft_green @@ -122,7 +140,7 @@ class PythonNews(Cog): self.bot.stats.incr("python_news.posted.pep") if msg.channel.is_news(): - log.trace("Publishing PEP annnouncement because it was in a news channel") + log.trace("Publishing PEP announcement because it was in a news channel") await msg.publish() # Apply new sent news to DB to avoid duplicate sending @@ -167,13 +185,13 @@ class PythonNews(Cog): ): continue - content = email_information["content"] + content = self.escape_markdown(email_information["content"]) link = THREAD_URL.format(id=thread["href"].split("/")[-2], list=maillist) # Build an embed and send a message to the webhook embed = discord.Embed( - title=thread_information["subject"], - description=content[:500] + f"... [continue reading]({link})" if len(content) > 500 else content, + title=self.escape_markdown(thread_information["subject"]), + description=content[:1000] + f"... [continue reading]({link})" if len(content) > 1000 else content, timestamp=new_date, url=link, colour=constants.Colours.soft_green diff --git a/bot/exts/info/site.py b/bot/exts/info/site.py deleted file mode 100644 index fb5b99086..000000000 --- a/bot/exts/info/site.py +++ /dev/null @@ -1,143 +0,0 @@ -import logging - -from discord import Colour, Embed -from discord.ext.commands import Cog, Context, Greedy, group - -from bot.bot import Bot -from bot.constants import URLs -from bot.pagination import LinePaginator - -log = logging.getLogger(__name__) - -PAGES_URL = f"{URLs.site_schema}{URLs.site}/pages" - - -class Site(Cog): - """Commands for linking to different parts of the site.""" - - def __init__(self, bot: Bot): - self.bot = bot - - @group(name="site", aliases=("s",), invoke_without_command=True) - async def site_group(self, ctx: Context) -> None: - """Commands for getting info about our website.""" - await ctx.send_help(ctx.command) - - @site_group.command(name="home", aliases=("about",), root_aliases=("home",)) - async def site_main(self, ctx: Context) -> None: - """Info about the website itself.""" - url = f"{URLs.site_schema}{URLs.site}/" - - embed = Embed(title="Python Discord website") - embed.set_footer(text=url) - embed.colour = Colour.blurple() - embed.description = ( - f"[Our official website]({url}) is an open-source community project " - "created with Python and Django. It contains information about the server " - "itself, lets you sign up for upcoming events, has its own wiki, contains " - "a list of valuable learning resources, and much more." - ) - - await ctx.send(embed=embed) - - @site_group.command(name="resources", root_aliases=("resources", "resource")) - async def site_resources(self, ctx: Context) -> None: - """Info about the site's Resources page.""" - learning_url = f"{PAGES_URL}/resources" - - embed = Embed(title="Resources") - embed.set_footer(text=f"{learning_url}") - embed.colour = Colour.blurple() - embed.description = ( - f"The [Resources page]({learning_url}) on our website contains a " - "list of hand-selected learning resources that we regularly recommend " - f"to both beginners and experts." - ) - - await ctx.send(embed=embed) - - @site_group.command(name="tools", root_aliases=("tools",)) - async def site_tools(self, ctx: Context) -> None: - """Info about the site's Tools page.""" - tools_url = f"{PAGES_URL}/resources/tools" - - embed = Embed(title="Tools") - embed.set_footer(text=f"{tools_url}") - embed.colour = Colour.blurple() - embed.description = ( - f"The [Tools page]({tools_url}) on our website contains a " - f"couple of the most popular tools for programming in Python." - ) - - await ctx.send(embed=embed) - - @site_group.command(name="help") - async def site_help(self, ctx: Context) -> None: - """Info about the site's Getting Help page.""" - url = f"{PAGES_URL}/resources/guides/asking-good-questions" - - embed = Embed(title="Asking Good Questions") - embed.set_footer(text=url) - embed.colour = Colour.blurple() - embed.description = ( - "Asking the right question about something that's new to you can sometimes be tricky. " - f"To help with this, we've created a [guide to asking good questions]({url}) on our website. " - "It contains everything you need to get the very best help from our community." - ) - - await ctx.send(embed=embed) - - @site_group.command(name="faq", root_aliases=("faq",)) - async def site_faq(self, ctx: Context) -> None: - """Info about the site's FAQ page.""" - url = f"{PAGES_URL}/frequently-asked-questions" - - embed = Embed(title="FAQ") - embed.set_footer(text=url) - embed.colour = Colour.blurple() - embed.description = ( - "As the largest Python community on Discord, we get hundreds of questions every day. " - "Many of these questions have been asked before. We've compiled a list of the most " - "frequently asked questions along with their answers, which can be found on " - f"our [FAQ page]({url})." - ) - - await ctx.send(embed=embed) - - @site_group.command(name="rules", aliases=("r", "rule"), root_aliases=("rules", "rule")) - async def site_rules(self, ctx: Context, rules: Greedy[int]) -> None: - """Provides a link to all rules or, if specified, displays specific rule(s).""" - rules_embed = Embed(title='Rules', color=Colour.blurple(), url=f'{PAGES_URL}/rules') - - if not rules: - # Rules were not submitted. Return the default description. - rules_embed.description = ( - "The rules and guidelines that apply to this community can be found on" - f" our [rules page]({PAGES_URL}/rules). We expect" - " all members of the community to have read and understood these." - ) - - await ctx.send(embed=rules_embed) - return - - full_rules = await self.bot.api_client.get('rules', params={'link_format': 'md'}) - - # Remove duplicates and sort the rule indices - rules = sorted(set(rules)) - invalid = ', '.join(str(index) for index in rules if index < 1 or index > len(full_rules)) - - if invalid: - await ctx.send(f":x: Invalid rule indices: {invalid}") - return - - for rule in rules: - self.bot.stats.incr(f"rule_uses.{rule}") - - final_rules = tuple(f"**{pick}.** {full_rules[pick - 1]}" for pick in rules) - - await LinePaginator.paginate(final_rules, ctx, rules_embed, max_lines=3) - - -def setup(bot: Bot) -> None: - """Load the Site cog.""" - bot.add_cog(Site(bot)) diff --git a/bot/exts/info/source.py b/bot/exts/info/source.py index ef07c77a1..e3e7029ca 100644 --- a/bot/exts/info/source.py +++ b/bot/exts/info/source.py @@ -2,45 +2,15 @@ import inspect from pathlib import Path from typing import Optional, Tuple, Union -from discord import Embed, utils +from discord import Embed from discord.ext import commands from bot.bot import Bot from bot.constants import URLs +from bot.converters import SourceConverter +from bot.exts.info.tags import TagIdentifier -SourceType = Union[commands.HelpCommand, commands.Command, commands.Cog, str, commands.ExtensionNotLoaded] - - -class SourceConverter(commands.Converter): - """Convert an argument into a help command, tag, command, or cog.""" - - @staticmethod - async def convert(ctx: commands.Context, argument: str) -> SourceType: - """Convert argument into source object.""" - if argument.lower() == "help": - return ctx.bot.help_command - - cog = ctx.bot.get_cog(argument) - if cog: - return cog - - cmd = ctx.bot.get_command(argument) - if cmd: - return cmd - - tags_cog = ctx.bot.get_cog("Tags") - show_tag = True - - if not tags_cog: - show_tag = False - elif argument.lower() in tags_cog._cache: - return argument.lower() - - escaped_arg = utils.escape_markdown(argument) - - raise commands.BadArgument( - f"Unable to convert '{escaped_arg}' to valid command{', tag,' if show_tag else ''} or Cog." - ) +SourceType = Union[commands.HelpCommand, commands.Command, commands.Cog, TagIdentifier, commands.ExtensionNotLoaded] class BotSource(commands.Cog): @@ -72,9 +42,9 @@ class BotSource(commands.Cog): source_item = inspect.unwrap(source_item.callback) src = source_item.__code__ filename = src.co_filename - elif isinstance(source_item, str): + elif isinstance(source_item, TagIdentifier): tags_cog = self.bot.get_cog("Tags") - filename = tags_cog._cache[source_item]["location"] + filename = tags_cog.tags[source_item].file_path else: src = type(source_item) try: @@ -82,7 +52,7 @@ class BotSource(commands.Cog): except TypeError: raise commands.BadArgument("Cannot get source for a dynamically-created object.") - if not isinstance(source_item, str): + if not isinstance(source_item, TagIdentifier): try: lines, first_line_no = inspect.getsourcelines(src) except OSError: @@ -95,7 +65,7 @@ class BotSource(commands.Cog): # Handle tag file location differently than others to avoid errors in some cases if not first_line_no: - file_location = Path(filename).relative_to("/bot/") + file_location = Path(filename).relative_to("bot/") else: file_location = Path(filename).relative_to(Path.cwd()).as_posix() @@ -113,7 +83,7 @@ class BotSource(commands.Cog): elif isinstance(source_object, commands.Command): description = source_object.short_doc title = f"Command: {source_object.qualified_name}" - elif isinstance(source_object, str): + elif isinstance(source_object, TagIdentifier): title = f"Tag: {source_object}" description = "" else: diff --git a/bot/exts/info/subscribe.py b/bot/exts/info/subscribe.py new file mode 100644 index 000000000..eff0c13b8 --- /dev/null +++ b/bot/exts/info/subscribe.py @@ -0,0 +1,201 @@ +import calendar +import operator +import typing as t +from dataclasses import dataclass + +import arrow +import discord +from discord.ext import commands +from discord.interactions import Interaction + +from bot import constants +from bot.bot import Bot +from bot.decorators import redirect_output +from bot.log import get_logger +from bot.utils import members, scheduling + + +@dataclass(frozen=True) +class AssignableRole: + """ + A role that can be assigned to a user. + + months_available is a tuple that signifies what months the role should be + self-assignable, using None for when it should always be available. + """ + + role_id: int + months_available: t.Optional[tuple[int]] + name: t.Optional[str] = None # This gets populated within Subscribe.init_cog() + + def is_currently_available(self) -> bool: + """Check if the role is available for the current month.""" + if self.months_available is None: + return True + return arrow.utcnow().month in self.months_available + + def get_readable_available_months(self) -> str: + """Get a readable string of the months the role is available.""" + if self.months_available is None: + return f"{self.name} is always available." + + # Join the months together with comma separators, but use "and" for the final seperator. + month_names = [calendar.month_name[month] for month in self.months_available] + available_months_str = ", ".join(month_names[:-1]) + f" and {month_names[-1]}" + return f"{self.name} can only be assigned during {available_months_str}." + + +ASSIGNABLE_ROLES = ( + AssignableRole(constants.Roles.announcements, None), + AssignableRole(constants.Roles.pyweek_announcements, None), + AssignableRole(constants.Roles.lovefest, (1, 2)), + AssignableRole(constants.Roles.advent_of_code, (11, 12)), +) + +ITEMS_PER_ROW = 3 +DELETE_MESSAGE_AFTER = 300 # Seconds + +log = get_logger(__name__) + + +class RoleButtonView(discord.ui.View): + """A list of SingleRoleButtons to show to the member.""" + + def __init__(self, member: discord.Member): + super().__init__() + self.interaction_owner = member + + async def interaction_check(self, interaction: Interaction) -> bool: + """Ensure that the user clicking the button is the member who invoked the command.""" + if interaction.user != self.interaction_owner: + await interaction.response.send_message( + ":x: This is not your command to react to!", + ephemeral=True + ) + return False + return True + + +class SingleRoleButton(discord.ui.Button): + """A button that adds or removes a role from the member depending on it's current state.""" + + ADD_STYLE = discord.ButtonStyle.success + REMOVE_STYLE = discord.ButtonStyle.red + UNAVAILABLE_STYLE = discord.ButtonStyle.secondary + LABEL_FORMAT = "{action} role {role_name}." + CUSTOM_ID_FORMAT = "subscribe-{role_id}" + + def __init__(self, role: AssignableRole, assigned: bool, row: int): + if role.is_currently_available(): + style = self.REMOVE_STYLE if assigned else self.ADD_STYLE + label = self.LABEL_FORMAT.format(action="Remove" if assigned else "Add", role_name=role.name) + else: + style = self.UNAVAILABLE_STYLE + label = f"🔒 {role.name}" + + super().__init__( + style=style, + label=label, + custom_id=self.CUSTOM_ID_FORMAT.format(role_id=role.role_id), + row=row, + ) + self.role = role + self.assigned = assigned + + async def callback(self, interaction: Interaction) -> None: + """Update the member's role and change button text to reflect current text.""" + if isinstance(interaction.user, discord.User): + log.trace("User %s is not a member", interaction.user) + await interaction.message.delete() + self.view.stop() + return + + if not self.role.is_currently_available(): + await interaction.response.send_message(self.role.get_readable_available_months(), ephemeral=True) + return + + await members.handle_role_change( + interaction.user, + interaction.user.remove_roles if self.assigned else interaction.user.add_roles, + discord.Object(self.role.role_id), + ) + + self.assigned = not self.assigned + await self.update_view(interaction) + await interaction.response.send_message( + self.LABEL_FORMAT.format(action="Added" if self.assigned else "Removed", role_name=self.role.name), + ephemeral=True, + ) + + async def update_view(self, interaction: Interaction) -> None: + """Updates the original interaction message with a new view object with the updated buttons.""" + self.style = self.REMOVE_STYLE if self.assigned else self.ADD_STYLE + self.label = self.LABEL_FORMAT.format(action="Remove" if self.assigned else "Add", role_name=self.role.name) + try: + await interaction.message.edit(view=self.view) + except discord.NotFound: + log.debug("Subscribe message for %s removed before buttons could be updated", interaction.user) + self.view.stop() + + +class Subscribe(commands.Cog): + """Cog to allow user to self-assign & remove the roles present in ASSIGNABLE_ROLES.""" + + def __init__(self, bot: Bot): + self.bot = bot + self.init_task = scheduling.create_task(self.init_cog(), event_loop=self.bot.loop) + self.assignable_roles: list[AssignableRole] = [] + self.guild: discord.Guild = None + + async def init_cog(self) -> None: + """Initialise the cog by resolving the role IDs in ASSIGNABLE_ROLES to role names.""" + await self.bot.wait_until_guild_available() + + self.guild = self.bot.get_guild(constants.Guild.id) + + for role in ASSIGNABLE_ROLES: + discord_role = self.guild.get_role(role.role_id) + if discord_role is None: + log.warning("Could not resolve %d to a role in the guild, skipping.", role.role_id) + continue + self.assignable_roles.append( + AssignableRole( + role_id=role.role_id, + months_available=role.months_available, + name=discord_role.name, + ) + ) + + # Sort by role name, then shift unavailable roles to the end of the list + self.assignable_roles.sort(key=operator.attrgetter("name")) + self.assignable_roles.sort(key=operator.methodcaller("is_currently_available"), reverse=True) + + @commands.cooldown(1, 10, commands.BucketType.member) + @commands.command(name="subscribe", aliases=("unsubscribe",)) + @redirect_output( + destination_channel=constants.Channels.bot_commands, + bypass_roles=constants.STAFF_PARTNERS_COMMUNITY_ROLES, + ) + async def subscribe_command(self, ctx: commands.Context, *_) -> None: # We don't actually care about the args + """Display the member's current state for each role, and allow them to add/remove the roles.""" + await self.init_task + + button_view = RoleButtonView(ctx.author) + author_roles = [role.id for role in ctx.author.roles] + for index, role in enumerate(self.assignable_roles): + row = index // ITEMS_PER_ROW + button_view.add_item(SingleRoleButton(role, role.role_id in author_roles, row)) + + await ctx.send( + "Click the buttons below to add or remove your roles!", + view=button_view, + delete_after=DELETE_MESSAGE_AFTER, + ) + + +def setup(bot: Bot) -> None: + """Load the Subscribe cog.""" + if len(ASSIGNABLE_ROLES) > ITEMS_PER_ROW*5: # Discord limits views to 5 rows of buttons. + log.error("Too many roles for 5 rows, not loading the Subscribe cog.") + else: + bot.add_cog(Subscribe(bot)) diff --git a/bot/exts/info/tags.py b/bot/exts/info/tags.py index bb91a8563..f66237c8e 100644 --- a/bot/exts/info/tags.py +++ b/bot/exts/info/tags.py @@ -1,19 +1,23 @@ -import logging +from __future__ import annotations + +import enum import re import time from pathlib import Path -from typing import Callable, Dict, Iterable, List, Optional +from typing import Callable, Iterable, Literal, NamedTuple, Optional, Union -from discord import Colour, Embed, Member +import discord +import frontmatter +from discord import Embed, Member from discord.ext.commands import Cog, Context, group from bot import constants from bot.bot import Bot -from bot.converters import TagNameConverter +from bot.log import get_logger from bot.pagination import LinePaginator from bot.utils.messages import wait_for_deletion -log = logging.getLogger(__name__) +log = get_logger(__name__) TEST_CHANNELS = ( constants.Channels.bot_commands, @@ -24,99 +28,168 @@ REGEX_NON_ALPHABET = re.compile(r"[^a-z]", re.MULTILINE & re.IGNORECASE) FOOTER_TEXT = f"To show a tag, type {constants.Bot.prefix}tags <tagname>." +class COOLDOWN(enum.Enum): + """Sentinel value to signal that a tag is on cooldown.""" + + obj = object() + + +class TagIdentifier(NamedTuple): + """Stores the group and name used as an identifier for a tag.""" + + group: Optional[str] + name: str + + def get_fuzzy_score(self, fuzz_tag_identifier: TagIdentifier) -> float: + """Get fuzzy score, using `fuzz_tag_identifier` as the identifier to fuzzy match with.""" + if (self.group is None) != (fuzz_tag_identifier.group is None): + # Ignore tags without groups if the identifier has a group and vice versa + return .0 + if self.group == fuzz_tag_identifier.group: + # Completely identical, or both None + group_score = 1 + else: + group_score = _fuzzy_search(fuzz_tag_identifier.group, self.group) + + fuzzy_score = group_score * _fuzzy_search(fuzz_tag_identifier.name, self.name) * 100 + if fuzzy_score: + log.trace(f"Fuzzy score {fuzzy_score:=06.2f} for tag {self!r} with fuzz {fuzz_tag_identifier!r}") + return fuzzy_score + + def __str__(self) -> str: + if self.group is not None: + return f"{self.group} {self.name}" + else: + return self.name + + @classmethod + def from_string(cls, string: str) -> TagIdentifier: + """Create a `TagIdentifier` instance from the beginning of `string`.""" + split_string = string.removeprefix(constants.Bot.prefix).split(" ", maxsplit=2) + if len(split_string) == 1: + return cls(None, split_string[0]) + else: + return cls(split_string[0], split_string[1]) + + +class Tag: + """Provide an interface to a tag from resources with `file_content`.""" + + def __init__(self, content_path: Path): + post = frontmatter.loads(content_path.read_text("utf8")) + self.file_path = content_path + self.content = post.content + self.metadata = post.metadata + self._restricted_to: set[int] = set(self.metadata.get("restricted_to", ())) + self._cooldowns: dict[discord.TextChannel, float] = {} + + @property + def embed(self) -> Embed: + """Create an embed for the tag.""" + embed = Embed.from_dict(self.metadata.get("embed", {})) + embed.description = self.content + return embed + + def accessible_by(self, member: discord.Member) -> bool: + """Check whether `member` can access the tag.""" + return bool( + not self._restricted_to + or self._restricted_to & {role.id for role in member.roles} + ) + + def on_cooldown_in(self, channel: discord.TextChannel) -> bool: + """Check whether the tag is on cooldown in `channel`.""" + return self._cooldowns.get(channel, float("-inf")) > time.time() + + def set_cooldown_for(self, channel: discord.TextChannel) -> None: + """Set the tag to be on cooldown in `channel` for `constants.Cooldowns.tags` seconds.""" + self._cooldowns[channel] = time.time() + constants.Cooldowns.tags + + +def _fuzzy_search(search: str, target: str) -> float: + """A simple scoring algorithm based on how many letters are found / total, with order in mind.""" + _search = REGEX_NON_ALPHABET.sub("", search.lower()) + if not _search: + return 0 + + _targets = iter(REGEX_NON_ALPHABET.split(target.lower())) + + current = 0 + for _target in _targets: + index = 0 + try: + while index < len(_target) and _search[current] == _target[index]: + current += 1 + index += 1 + except IndexError: + # Exit when _search runs out + break + + return current / len(_search) + + class Tags(Cog): - """Save new tags and fetch existing tags.""" + """Fetch tags by name or content.""" + + PAGINATOR_DEFAULTS = dict(max_lines=15, empty=False, footer_text=FOOTER_TEXT) def __init__(self, bot: Bot): self.bot = bot - self.tag_cooldowns = {} - self._cache = self.get_tags() - - @staticmethod - def get_tags() -> dict: - """Get all tags.""" - cache = {} + self.tags: dict[TagIdentifier, Tag] = {} + self.initialize_tags() + def initialize_tags(self) -> None: + """Load all tags from resources into `self.tags`.""" base_path = Path("bot", "resources", "tags") + for file in base_path.glob("**/*"): if file.is_file(): - tag_title = file.stem - tag = { - "title": tag_title, - "embed": { - "description": file.read_text(encoding="utf8"), - }, - "restricted_to": None, - "location": f"/bot/{file}" - } - - # Convert to a list to allow negative indexing. - parents = list(file.relative_to(base_path).parents) - if len(parents) > 1: - # -1 would be '.' hence -2 is used as the index. - tag["restricted_to"] = parents[-2].name - - cache[tag_title] = tag - - return cache - - @staticmethod - def check_accessibility(user: Member, tag: dict) -> bool: - """Check if user can access a tag.""" - return not tag["restricted_to"] or tag["restricted_to"].lower() in [role.name.lower() for role in user.roles] - - @staticmethod - def _fuzzy_search(search: str, target: str) -> float: - """A simple scoring algorithm based on how many letters are found / total, with order in mind.""" - current, index = 0, 0 - _search = REGEX_NON_ALPHABET.sub('', search.lower()) - _targets = iter(REGEX_NON_ALPHABET.split(target.lower())) - _target = next(_targets) - try: - while True: - while index < len(_target) and _search[current] == _target[index]: - current += 1 - index += 1 - index, _target = 0, next(_targets) - except (StopIteration, IndexError): - pass - return current / len(_search) * 100 - - def _get_suggestions(self, tag_name: str, thresholds: Optional[List[int]] = None) -> List[str]: - """Return a list of suggested tags.""" - scores: Dict[str, int] = { - tag_title: Tags._fuzzy_search(tag_name, tag['title']) - for tag_title, tag in self._cache.items() - } - - thresholds = thresholds or [100, 90, 80, 70, 60] - - for threshold in thresholds: + parent_dir = file.relative_to(base_path).parent + tag_name = file.stem + # Files directly under `base_path` have an empty string as the parent directory name + tag_group = parent_dir.name or None + + self.tags[TagIdentifier(tag_group, tag_name)] = Tag(file) + + def _get_suggestions(self, tag_identifier: TagIdentifier) -> list[tuple[TagIdentifier, Tag]]: + """Return a list of suggested tags for `tag_identifier`.""" + for threshold in [100, 90, 80, 70, 60]: suggestions = [ - self._cache[tag_title] - for tag_title, matching_score in scores.items() - if matching_score >= threshold + (identifier, tag) + for identifier, tag in self.tags.items() + if identifier.get_fuzzy_score(tag_identifier) >= threshold ] if suggestions: return suggestions return [] - def _get_tag(self, tag_name: str) -> list: - """Get a specific tag.""" - found = [self._cache.get(tag_name.lower(), None)] - if not found[0]: - return self._get_suggestions(tag_name) - return found + def get_fuzzy_matches(self, tag_identifier: TagIdentifier) -> list[tuple[TagIdentifier, Tag]]: + """Get tags with identifiers similar to `tag_identifier`.""" + suggestions = [] + + if tag_identifier.group is not None and len(tag_identifier.group) >= 2: + # Try fuzzy matching with only a name first + suggestions += self._get_suggestions(TagIdentifier(None, tag_identifier.group)) + + if len(tag_identifier.name) >= 2: + suggestions += self._get_suggestions(tag_identifier) - def _get_tags_via_content(self, check: Callable[[Iterable], bool], keywords: str, user: Member) -> list: + return suggestions + + def _get_tags_via_content( + self, + check: Callable[[Iterable], bool], + keywords: str, + user: Member, + ) -> list[tuple[TagIdentifier, Tag]]: """ Search for tags via contents. `predicate` will be the built-in any, all, or a custom callable. Must return a bool. """ - keywords_processed: List[str] = [] - for keyword in keywords.split(','): + keywords_processed = [] + for keyword in keywords.split(","): keyword_sanitized = keyword.strip().casefold() if not keyword_sanitized: # this happens when there are leading / trailing / consecutive comma. @@ -124,45 +197,48 @@ class Tags(Cog): keywords_processed.append(keyword_sanitized) if not keywords_processed: - # after sanitizing, we can end up with an empty list, for example when keywords is ',' + # after sanitizing, we can end up with an empty list, for example when keywords is "," # in that case, we simply want to search for such keywords directly instead. keywords_processed = [keywords] matching_tags = [] - for tag in self._cache.values(): - matches = (query in tag['embed']['description'].casefold() for query in keywords_processed) - if self.check_accessibility(user, tag) and check(matches): - matching_tags.append(tag) + for identifier, tag in self.tags.items(): + matches = (query in tag.content.casefold() for query in keywords_processed) + if tag.accessible_by(user) and check(matches): + matching_tags.append((identifier, tag)) return matching_tags - async def _send_matching_tags(self, ctx: Context, keywords: str, matching_tags: list) -> None: + async def _send_matching_tags( + self, + ctx: Context, + keywords: str, + matching_tags: list[tuple[TagIdentifier, Tag]], + ) -> None: """Send the result of matching tags to user.""" - if not matching_tags: - pass - elif len(matching_tags) == 1: - await ctx.send(embed=Embed().from_dict(matching_tags[0]['embed'])) - else: - is_plural = keywords.strip().count(' ') > 0 or keywords.strip().count(',') > 0 + if len(matching_tags) == 1: + await ctx.send(embed=matching_tags[0][1].embed) + elif matching_tags: + is_plural = keywords.strip().count(" ") > 0 or keywords.strip().count(",") > 0 embed = Embed( title=f"Here are the tags containing the given keyword{'s' * is_plural}:", - description='\n'.join(tag['title'] for tag in matching_tags[:10]) ) await LinePaginator.paginate( - sorted(f"**»** {tag['title']}" for tag in matching_tags), + sorted( + f"**\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}** {identifier.name}" + for identifier, _ in matching_tags + ), ctx, embed, - footer_text=FOOTER_TEXT, - empty=False, - max_lines=15 + **self.PAGINATOR_DEFAULTS, ) - @group(name='tags', aliases=('tag', 't'), invoke_without_command=True) - async def tags_group(self, ctx: Context, *, tag_name: TagNameConverter = None) -> None: + @group(name="tags", aliases=("tag", "t"), invoke_without_command=True, usage="[tag_group] [tag_name]") + async def tags_group(self, ctx: Context, *, argument_string: Optional[str]) -> None: """Show all known tags, a single tag, or run a subcommand.""" - await self.get_command(ctx, tag_name=tag_name) + await self.get_command(ctx, argument_string=argument_string) - @tags_group.group(name='search', invoke_without_command=True) + @tags_group.group(name="search", invoke_without_command=True) async def search_tag_content(self, ctx: Context, *, keywords: str) -> None: """ Search inside tags' contents for tags. Allow searching for multiple keywords separated by comma. @@ -172,123 +248,151 @@ class Tags(Cog): matching_tags = self._get_tags_via_content(all, keywords, ctx.author) await self._send_matching_tags(ctx, keywords, matching_tags) - @search_tag_content.command(name='any') - async def search_tag_content_any_keyword(self, ctx: Context, *, keywords: Optional[str] = 'any') -> None: + @search_tag_content.command(name="any") + async def search_tag_content_any_keyword(self, ctx: Context, *, keywords: Optional[str] = "any") -> None: """ Search inside tags' contents for tags. Allow searching for multiple keywords separated by comma. Search for tags that has ANY of the keywords. """ - matching_tags = self._get_tags_via_content(any, keywords or 'any', ctx.author) + matching_tags = self._get_tags_via_content(any, keywords or "any", ctx.author) await self._send_matching_tags(ctx, keywords, matching_tags) - async def display_tag(self, ctx: Context, tag_name: str = None) -> bool: + async def get_tag_embed( + self, + ctx: Context, + tag_identifier: TagIdentifier, + ) -> Optional[Union[Embed, Literal[COOLDOWN.obj]]]: """ - If a tag is not found, display similar tag names as suggestions. - - If a tag is not specified, display a paginated embed of all tags. + Generate an embed of the requested tag or of suggestions if the tag doesn't exist/isn't accessible by the user. - Tags are on cooldowns on a per-tag, per-channel basis. If a tag is on cooldown, display - nothing and return True. + If the requested tag is on cooldown return `COOLDOWN.obj`, otherwise if no suggestions were found return None. """ - def _command_on_cooldown(tag_name: str) -> bool: - """ - Check if the command is currently on cooldown, on a per-tag, per-channel basis. - - The cooldown duration is set in constants.py. - """ - now = time.time() - - cooldown_conditions = ( - tag_name - and tag_name in self.tag_cooldowns - and (now - self.tag_cooldowns[tag_name]["time"]) < constants.Cooldowns.tags - and self.tag_cooldowns[tag_name]["channel"] == ctx.channel.id + filtered_tags = [ + (ident, tag) for ident, tag in + self.get_fuzzy_matches(tag_identifier)[:10] + if tag.accessible_by(ctx.author) + ] + + tag = self.tags.get(tag_identifier) + + if tag is None and tag_identifier.group is not None: + # Try exact match with only the name + tag = self.tags.get(TagIdentifier(None, tag_identifier.group)) + + if tag is None and len(filtered_tags) == 1: + tag_identifier = filtered_tags[0][0] + tag = filtered_tags[0][1] + + if tag is not None: + if tag.on_cooldown_in(ctx.channel): + log.debug(f"Tag {str(tag_identifier)!r} is on cooldown.") + return COOLDOWN.obj + tag.set_cooldown_for(ctx.channel) + + self.bot.stats.incr( + f"tags.usages" + f"{'.' + tag_identifier.group.replace('-', '_') if tag_identifier.group else ''}" + f".{tag_identifier.name.replace('-', '_')}" ) + return tag.embed - if cooldown_conditions: - return True - return False - - if _command_on_cooldown(tag_name): - time_elapsed = time.time() - self.tag_cooldowns[tag_name]["time"] - time_left = constants.Cooldowns.tags - time_elapsed - log.info( - f"{ctx.author} tried to get the '{tag_name}' tag, but the tag is on cooldown. " - f"Cooldown ends in {time_left:.1f} seconds." + else: + if not filtered_tags: + return None + suggested_tags_text = "\n".join( + f"**\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}** {identifier}" + for identifier, tag in filtered_tags + if not tag.on_cooldown_in(ctx.channel) + ) + return Embed( + title="Did you mean ...", + description=suggested_tags_text ) - return True - - if tag_name is not None: - temp_founds = self._get_tag(tag_name) - - founds = [] - - for found_tag in temp_founds: - if self.check_accessibility(ctx.author, found_tag): - founds.append(found_tag) - if len(founds) == 1: - tag = founds[0] - if ctx.channel.id not in TEST_CHANNELS: - self.tag_cooldowns[tag_name] = { - "time": time.time(), - "channel": ctx.channel.id - } + def accessible_tags(self, user: Member) -> list[str]: + """Return a formatted list of tags that are accessible by `user`; groups first, and alphabetically sorted.""" + def tag_sort_key(tag_item: tuple[TagIdentifier, Tag]) -> str: + group, name = tag_item[0] + if group is None: + # Max codepoint character to force tags without a group to the end + group = chr(0x10ffff) + + return group + name + + result_lines = [] + current_group = "" + group_accessible = True + + for identifier, tag in sorted(self.tags.items(), key=tag_sort_key): + + if identifier.group != current_group: + if not group_accessible: + # Remove group separator line if no tags in the previous group were accessible by the user. + result_lines.pop() + # A new group began, add a separator with the group name. + current_group = identifier.group + if current_group is not None: + group_accessible = False + result_lines.append(f"\n\N{BULLET} **{current_group}**") + else: + result_lines.append("\n\N{BULLET}") + + if tag.accessible_by(user): + result_lines.append(f"**\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}** {identifier.name}") + group_accessible = True + + return result_lines + + def accessible_tags_in_group(self, group: str, user: discord.Member) -> list[str]: + """Return a formatted list of tags in `group`, that are accessible by `user`.""" + return sorted( + f"**\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}** {identifier}" + for identifier, tag in self.tags.items() + if identifier.group == group and tag.accessible_by(user) + ) + + @tags_group.command(name="get", aliases=("show", "g"), usage="[tag_group] [tag_name]") + async def get_command(self, ctx: Context, *, argument_string: Optional[str]) -> bool: + """ + If a single argument matching a group name is given, list all accessible tags from that group + Otherwise display the tag if one was found for the given arguments, or try to display suggestions for that name. - self.bot.stats.incr(f"tags.usages.{tag['title'].replace('-', '_')}") + With no arguments, list all accessible tags. - await wait_for_deletion( - await ctx.send(embed=Embed.from_dict(tag['embed'])), - [ctx.author.id], - ) - return True - elif founds and len(tag_name) >= 3: - await wait_for_deletion( - await ctx.send( - embed=Embed( - title='Did you mean ...', - description='\n'.join(tag['title'] for tag in founds[:10]) - ) - ), - [ctx.author.id], + Returns True if a message was sent, or if the tag is on cooldown. + Returns False if no message was sent. + """ # noqa: D205, D415 + if not argument_string: + if self.tags: + await LinePaginator.paginate( + self.accessible_tags(ctx.author), ctx, Embed(title="Available tags"), **self.PAGINATOR_DEFAULTS ) - return True - - else: - tags = self._cache.values() - if not tags: - await ctx.send(embed=Embed( - description="**There are no tags in the database!**", - colour=Colour.red() - )) - return True else: - embed: Embed = Embed(title="**Current tags**") + await ctx.send(embed=Embed(description="**There are no tags!**")) + return True + + identifier = TagIdentifier.from_string(argument_string) + + if identifier.group is None: + # Try to find accessible tags from a group matching the identifier's name. + if group_tags := self.accessible_tags_in_group(identifier.name, ctx.author): await LinePaginator.paginate( - sorted( - f"**»** {tag['title']}" for tag in tags - if self.check_accessibility(ctx.author, tag) - ), - ctx, - embed, - footer_text=FOOTER_TEXT, - empty=False, - max_lines=15 + group_tags, ctx, Embed(title=f"Tags under *{identifier.name}*"), **self.PAGINATOR_DEFAULTS ) return True - return False - - @tags_group.command(name='get', aliases=('show', 'g')) - async def get_command(self, ctx: Context, *, tag_name: TagNameConverter = None) -> bool: - """ - Get a specified tag, or a list of all tags if no tag is specified. + embed = await self.get_tag_embed(ctx, identifier) + if embed is None: + return False - Returns True if something can be sent, or if the tag is on cooldown. - Returns False if no matches are found. - """ - return await self.display_tag(ctx, tag_name) + if embed is not COOLDOWN.obj: + await wait_for_deletion( + await ctx.send(embed=embed), + (ctx.author.id,) + ) + # A valid tag was found and was either sent, or is on cooldown + return True def setup(bot: Bot) -> None: diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py new file mode 100644 index 000000000..cb6836258 --- /dev/null +++ b/bot/exts/moderation/clean.py @@ -0,0 +1,607 @@ +import contextlib +import re +import time +from collections import defaultdict +from contextlib import suppress +from datetime import datetime +from itertools import takewhile +from typing import Callable, Iterable, Literal, Optional, TYPE_CHECKING, Union + +from discord import Colour, Message, NotFound, TextChannel, User, errors +from discord.ext.commands import Cog, Context, Converter, Greedy, group, has_any_role +from discord.ext.commands.converter import TextChannelConverter +from discord.ext.commands.errors import BadArgument + +from bot.bot import Bot +from bot.constants import Channels, CleanMessages, Colours, Emojis, Event, Icons, MODERATION_ROLES +from bot.converters import Age, ISODateTime +from bot.exts.moderation.modlog import ModLog +from bot.log import get_logger +from bot.utils.channel import is_mod_channel + +log = get_logger(__name__) + +# Number of seconds before command invocations and responses are deleted in non-moderation channels. +MESSAGE_DELETE_DELAY = 5 + +# Type alias for checks for whether a message should be deleted. +Predicate = Callable[[Message], bool] +# Type alias for message lookup ranges. +CleanLimit = Union[Message, Age, ISODateTime] + + +class CleanChannels(Converter): + """A converter to turn the string into a list of channels to clean, or the literal `*` for all public channels.""" + + _channel_converter = TextChannelConverter() + + async def convert(self, ctx: Context, argument: str) -> Union[Literal["*"], list[TextChannel]]: + """Converts a string to a list of channels to clean, or the literal `*` for all public channels.""" + if argument == "*": + return "*" + return [await self._channel_converter.convert(ctx, channel) for channel in argument.split()] + + +class Regex(Converter): + """A converter that takes a string in the form `.+` and returns the contents of the inline code compiled.""" + + async def convert(self, ctx: Context, argument: str) -> re.Pattern: + """Strips the backticks from the string and compiles it to a regex pattern.""" + match = re.fullmatch(r"`(.+?)`", argument) + if not match: + raise BadArgument("Regex pattern missing wrapping backticks") + try: + return re.compile(match.group(1), re.IGNORECASE + re.DOTALL) + except re.error as e: + raise BadArgument(f"Regex error: {e.msg}") + + +if TYPE_CHECKING: # Used to allow method resolution in IDEs like in converters.py. + CleanChannels = Union[Literal["*"], list[TextChannel]] # noqa: F811 + Regex = re.Pattern # noqa: F811 + + +class Clean(Cog): + """ + A cog that allows messages to be deleted in bulk while applying various filters. + + You can delete messages sent by a specific user, messages sent by bots, all messages, or messages that match a + specific regular expression. + + The deleted messages are saved and uploaded to the database via an API endpoint, and a URL is returned which can be + used to view the messages in the Discord dark theme style. + """ + + def __init__(self, bot: Bot): + self.bot = bot + self.cleaning = False + + @property + def mod_log(self) -> ModLog: + """Get currently loaded ModLog cog instance.""" + return self.bot.get_cog("ModLog") + + # region: Helper functions + + @staticmethod + def _validate_input( + channels: Optional[CleanChannels], + bots_only: bool, + users: Optional[list[User]], + first_limit: Optional[CleanLimit], + second_limit: Optional[CleanLimit], + ) -> None: + """Raise errors if an argument value or a combination of values is invalid.""" + if first_limit is None: + # This is an optional argument for the sake of the master command, but it's actually required. + raise BadArgument("Missing cleaning limit.") + + if (isinstance(first_limit, Message) or isinstance(second_limit, Message)) and channels: + raise BadArgument("Both a message limit and channels specified.") + + if isinstance(first_limit, Message) and isinstance(second_limit, Message): + # Messages are not in same channel. + if first_limit.channel != second_limit.channel: + raise BadArgument("Message limits are in different channels.") + + if users and bots_only: + raise BadArgument("Marked as bots only, but users were specified.") + + @staticmethod + async def _send_expiring_message(ctx: Context, content: str) -> None: + """Send `content` to the context channel. Automatically delete if it's not a mod channel.""" + delete_after = None if is_mod_channel(ctx.channel) else MESSAGE_DELETE_DELAY + await ctx.send(content, delete_after=delete_after) + + @staticmethod + def _channels_set( + channels: CleanChannels, ctx: Context, first_limit: CleanLimit, second_limit: CleanLimit + ) -> set[TextChannel]: + """Standardize the input `channels` argument to a usable set of text channels.""" + # Default to using the invoking context's channel or the channel of the message limit(s). + if not channels: + # Input was validated - if first_limit is a message, second_limit won't point at a different channel. + if isinstance(first_limit, Message): + channels = {first_limit.channel} + elif isinstance(second_limit, Message): + channels = {second_limit.channel} + else: + channels = {ctx.channel} + else: + if channels == "*": + channels = { + channel for channel in ctx.guild.channels + if isinstance(channel, TextChannel) + # Assume that non-public channels are not needed to optimize for speed. + and channel.permissions_for(ctx.guild.default_role).view_channel + } + else: + channels = set(channels) + + return channels + + @staticmethod + def _build_predicate( + first_limit: datetime, + second_limit: Optional[datetime] = None, + bots_only: bool = False, + users: Optional[list[User]] = None, + regex: Optional[re.Pattern] = None, + ) -> Predicate: + """Return the predicate that decides whether to delete a given message.""" + def predicate_bots_only(message: Message) -> bool: + """Return True if the message was sent by a bot.""" + return message.author.bot + + def predicate_specific_users(message: Message) -> bool: + """Return True if the message was sent by the user provided in the _clean_messages call.""" + return message.author in users + + def predicate_regex(message: Message) -> bool: + """Check if the regex provided in _clean_messages matches the message content or any embed attributes.""" + content = [message.content] + + # Add the content for all embed attributes + for embed in message.embeds: + content.append(embed.title) + content.append(embed.description) + content.append(embed.footer.text) + content.append(embed.author.name) + for field in embed.fields: + content.append(field.name) + content.append(field.value) + + # Get rid of empty attributes and turn it into a string + content = "\n".join(attr for attr in content if attr) + + # Now let's see if there's a regex match + return bool(regex.search(content)) + + def predicate_range(message: Message) -> bool: + """Check if the message age is between the two limits.""" + return first_limit <= message.created_at <= second_limit + + def predicate_after(message: Message) -> bool: + """Check if the message is older than the first limit.""" + return message.created_at >= first_limit + + predicates = [] + # Set up the correct predicate + if second_limit: + predicates.append(predicate_range) # Delete messages in the specified age range + else: + predicates.append(predicate_after) # Delete messages older than the specified age + + if bots_only: + predicates.append(predicate_bots_only) # Delete messages from bots + if users: + predicates.append(predicate_specific_users) # Delete messages from specific user + if regex: + predicates.append(predicate_regex) # Delete messages that match regex + + if len(predicates) == 1: + return predicates[0] + return lambda m: all(pred(m) for pred in predicates) + + async def _delete_invocation(self, ctx: Context) -> None: + """Delete the command invocation if it's not in a mod channel.""" + if not is_mod_channel(ctx.channel): + self.mod_log.ignore(Event.message_delete, ctx.message.id) + try: + await ctx.message.delete() + except errors.NotFound: + # Invocation message has already been deleted + log.info("Tried to delete invocation message, but it was already deleted.") + + def _use_cache(self, limit: datetime) -> bool: + """Tell whether all messages to be cleaned can be found in the cache.""" + return self.bot.cached_messages[0].created_at <= limit + + def _get_messages_from_cache( + self, + channels: set[TextChannel], + to_delete: Predicate, + lower_limit: datetime + ) -> tuple[defaultdict[TextChannel, list], list[int]]: + """Helper function for getting messages from the cache.""" + message_mappings = defaultdict(list) + message_ids = [] + for message in takewhile(lambda m: m.created_at > lower_limit, reversed(self.bot.cached_messages)): + if not self.cleaning: + # Cleaning was canceled + return message_mappings, message_ids + + if message.channel in channels and to_delete(message): + message_mappings[message.channel].append(message) + message_ids.append(message.id) + + return message_mappings, message_ids + + async def _get_messages_from_channels( + self, + channels: Iterable[TextChannel], + to_delete: Predicate, + before: datetime, + after: Optional[datetime] = None + ) -> tuple[defaultdict[TextChannel, list], list]: + message_mappings = defaultdict(list) + message_ids = [] + + for channel in channels: + async for message in channel.history(limit=CleanMessages.message_limit, before=before, after=after): + + if not self.cleaning: + # Cleaning was canceled, return empty containers. + return defaultdict(list), [] + + if to_delete(message): + message_mappings[message.channel].append(message) + message_ids.append(message.id) + + return message_mappings, message_ids + + @staticmethod + def is_older_than_14d(message: Message) -> bool: + """ + Precisely checks if message is older than 14 days, bulk deletion limit. + + Inspired by how purge works internally. + Comparison on message age could possibly be less accurate which in turn would resort in problems + with message deletion if said messages are very close to the 14d mark. + """ + two_weeks_old_snowflake = int((time.time() - 14 * 24 * 60 * 60) * 1000.0 - 1420070400000) << 22 + return message.id < two_weeks_old_snowflake + + async def _delete_messages_individually(self, messages: list[Message]) -> list[Message]: + """Delete each message in the list unless cleaning is cancelled. Return the deleted messages.""" + deleted = [] + for message in messages: + # Ensure that deletion was not canceled + if not self.cleaning: + return deleted + with contextlib.suppress(NotFound): # Message doesn't exist or was already deleted + await message.delete() + deleted.append(message) + return deleted + + async def _delete_found(self, message_mappings: dict[TextChannel, list[Message]]) -> list[Message]: + """ + Delete the detected messages. + + Deletion is made in bulk per channel for messages less than 14d old. + The function returns the deleted messages. + If cleaning was cancelled in the middle, return messages already deleted. + """ + deleted = [] + for channel, messages in message_mappings.items(): + to_delete = [] + + delete_old = False + for current_index, message in enumerate(messages): # noqa: B007 + if not self.cleaning: + # Means that the cleaning was canceled + return deleted + + if self.is_older_than_14d(message): + # Further messages are too old to be deleted in bulk + delete_old = True + break + + to_delete.append(message) + + if len(to_delete) == 100: + # Only up to 100 messages can be deleted in a bulk + await channel.delete_messages(to_delete) + deleted.extend(to_delete) + to_delete.clear() + + if not self.cleaning: + return deleted + if len(to_delete) > 0: + # Deleting any leftover messages if there are any + with suppress(NotFound): + await channel.delete_messages(to_delete) + deleted.extend(to_delete) + + if not self.cleaning: + return deleted + if delete_old: + old_deleted = await self._delete_messages_individually(messages[current_index:]) + deleted.extend(old_deleted) + + return deleted + + async def _modlog_cleaned_messages( + self, + messages: list[Message], + channels: CleanChannels, + ctx: Context + ) -> Optional[str]: + """Log the deleted messages to the modlog, returning the log url if logging was successful.""" + if not messages: + # Can't build an embed, nothing to clean! + await self._send_expiring_message(ctx, ":x: No matching messages could be found.") + return None + + # Reverse the list to have reverse chronological order + log_messages = reversed(messages) + log_url = await self.mod_log.upload_log(log_messages, ctx.author.id) + + # Build the embed and send it + if channels == "*": + target_channels = "all public channels" + else: + target_channels = ", ".join(channel.mention for channel in channels) + + message = ( + f"**{len(messages)}** messages deleted in {target_channels} by " + f"{ctx.author.mention}\n\n" + f"A log of the deleted messages can be found [here]({log_url})." + ) + + await self.mod_log.send_log_message( + icon_url=Icons.message_bulk_delete, + colour=Colour(Colours.soft_red), + title="Bulk message delete", + text=message, + channel_id=Channels.mod_log, + ) + + return log_url + + # endregion + + async def _clean_messages( + self, + ctx: Context, + channels: Optional[CleanChannels], + bots_only: bool = False, + users: Optional[list[User]] = None, + regex: Optional[re.Pattern] = None, + first_limit: Optional[CleanLimit] = None, + second_limit: Optional[CleanLimit] = None, + attempt_delete_invocation: bool = True, + ) -> Optional[str]: + """A helper function that does the actual message cleaning, returns the log url if logging was successful.""" + self._validate_input(channels, bots_only, users, first_limit, second_limit) + + # Are we already performing a clean? + if self.cleaning: + await self._send_expiring_message( + ctx, ":x: Please wait for the currently ongoing clean operation to complete." + ) + return None + self.cleaning = True + + deletion_channels = self._channels_set(channels, ctx, first_limit, second_limit) + + if isinstance(first_limit, Message): + first_limit = first_limit.created_at + if isinstance(second_limit, Message): + second_limit = second_limit.created_at + if first_limit and second_limit: + first_limit, second_limit = sorted([first_limit, second_limit]) + + # Needs to be called after standardizing the input. + predicate = self._build_predicate(first_limit, second_limit, bots_only, users, regex) + + if attempt_delete_invocation: + # Delete the invocation first + await self._delete_invocation(ctx) + + if self._use_cache(first_limit): + log.trace(f"Messages for cleaning by {ctx.author.id} will be searched in the cache.") + message_mappings, message_ids = self._get_messages_from_cache( + channels=deletion_channels, to_delete=predicate, lower_limit=first_limit + ) + else: + log.trace(f"Messages for cleaning by {ctx.author.id} will be searched in channel histories.") + message_mappings, message_ids = await self._get_messages_from_channels( + channels=deletion_channels, + to_delete=predicate, + before=second_limit, + after=first_limit # Remember first is the earlier datetime. + ) + + if not self.cleaning: + # Means that the cleaning was canceled + return None + + # Now let's delete the actual messages with purge. + self.mod_log.ignore(Event.message_delete, *message_ids) + deleted_messages = await self._delete_found(message_mappings) + self.cleaning = False + + if not channels: + channels = deletion_channels + log_url = await self._modlog_cleaned_messages(deleted_messages, channels, ctx) + + success_message = ( + f"{Emojis.ok_hand} Deleted {len(deleted_messages)} messages. " + f"A log of the deleted messages can be found here {log_url}." + ) + if log_url and is_mod_channel(ctx.channel): + await ctx.reply(success_message) + elif log_url: + if mods := self.bot.get_channel(Channels.mods): + await mods.send(f"{ctx.author.mention} {success_message}") + return log_url + + # region: Commands + + @group(invoke_without_command=True, name="clean", aliases=["clear", "purge"]) + async def clean_group( + self, + ctx: Context, + users: Greedy[User] = None, + first_limit: Optional[CleanLimit] = None, + second_limit: Optional[CleanLimit] = None, + regex: Optional[Regex] = None, + bots_only: Optional[bool] = False, + *, + channels: CleanChannels = None # "Optional" with discord.py silently ignores incorrect input. + ) -> None: + """ + Commands for cleaning messages in channels. + + If arguments are provided, will act as a master command from which all subcommands can be derived. + + \u2003• `users`: A series of user mentions, ID's, or names. + \u2003• `first_limit` and `second_limit`: A message, a duration delta, or an ISO datetime. + At least one limit is required. + If a message is provided, cleaning will happen in that channel, and channels cannot be provided. + If only one of them is provided, acts as `clean until`. If both are provided, acts as `clean between`. + \u2003• `regex`: A regex pattern the message must contain to be deleted. + The pattern must be provided enclosed in backticks. + If the pattern contains spaces, it still needs to be enclosed in double quotes on top of that. + \u2003• `bots_only`: Whether to delete only bots. If specified, users cannot be specified. + \u2003• `channels`: A series of channels to delete in, or an asterisk to delete from all public channels. + """ + if not any([users, first_limit, second_limit, regex, channels]): + await ctx.send_help(ctx.command) + return + + await self._clean_messages(ctx, channels, bots_only, users, regex, first_limit, second_limit) + + @clean_group.command(name="user", aliases=["users"]) + async def clean_user( + self, + ctx: Context, + user: User, + message_or_time: CleanLimit, + *, + channels: CleanChannels = None + ) -> None: + """ + Delete messages posted by the provided user, stop cleaning after reaching `message_or_time`. + + `message_or_time` can be either a message to stop at (exclusive), a timedelta for max message age, or an ISO + datetime. + + If a message is specified, `channels` cannot be specified. + """ + await self._clean_messages(ctx, users=[user], channels=channels, first_limit=message_or_time) + + @clean_group.command(name="bots", aliases=["bot"]) + async def clean_bots(self, ctx: Context, message_or_time: CleanLimit, *, channels: CleanChannels = None) -> None: + """ + Delete all messages posted by a bot, stop cleaning after traversing `traverse` messages. + + `message_or_time` can be either a message to stop at (exclusive), a timedelta for max message age, or an ISO + datetime. + + If a message is specified, `channels` cannot be specified. + """ + await self._clean_messages(ctx, bots_only=True, channels=channels, first_limit=message_or_time) + + @clean_group.command(name="regex", aliases=["word", "expression", "pattern"]) + async def clean_regex( + self, + ctx: Context, + regex: Regex, + message_or_time: CleanLimit, + *, + channels: CleanChannels = None + ) -> None: + """ + Delete all messages that match a certain regex, stop cleaning after reaching `message_or_time`. + + `message_or_time` can be either a message to stop at (exclusive), a timedelta for max message age, or an ISO + datetime. + If a message is specified, `channels` cannot be specified. + + The pattern must be provided enclosed in backticks. + If the pattern contains spaces, it still needs to be enclosed in double quotes on top of that. + For example: `[0-9]` + """ + await self._clean_messages(ctx, regex=regex, channels=channels, first_limit=message_or_time) + + @clean_group.command(name="until") + async def clean_until( + self, + ctx: Context, + until: CleanLimit, + channel: TextChannel = None + ) -> None: + """ + Delete all messages until a certain limit. + + A limit can be either a message, and ISO date-time string, or a time delta. + If a message is specified, `channel` cannot be specified. + """ + await self._clean_messages( + ctx, + channels=[channel] if channel else None, + first_limit=until, + ) + + @clean_group.command(name="between", aliases=["after-until", "from-to"]) + async def clean_between( + self, + ctx: Context, + first_limit: CleanLimit, + second_limit: CleanLimit, + channel: TextChannel = None + ) -> None: + """ + Delete all messages within range. + + The range is specified through two limits. + A limit can be either a message, and ISO date-time string, or a time delta. + + If two messages are specified, they both must be in the same channel. + If a message is specified, `channel` cannot be specified. + """ + await self._clean_messages( + ctx, + channels=[channel] if channel else None, + first_limit=first_limit, + second_limit=second_limit, + ) + + @clean_group.command(name="stop", aliases=["cancel", "abort"]) + async def clean_cancel(self, ctx: Context) -> None: + """If there is an ongoing cleaning process, attempt to immediately cancel it.""" + if not self.cleaning: + message = ":question: There's no cleaning going on." + else: + self.cleaning = False + message = f"{Emojis.check_mark} Clean interrupted." + + await self._send_expiring_message(ctx, message) + await self._delete_invocation(ctx) + + # endregion + + async def cog_check(self, ctx: Context) -> bool: + """Only allow moderators to invoke the commands in this cog.""" + return await has_any_role(*MODERATION_ROLES).predicate(ctx) + + async def cog_command_error(self, ctx: Context, error: Exception) -> None: + """Safely end the cleaning operation on unexpected errors.""" + self.cleaning = False + + +def setup(bot: Bot) -> None: + """Load the Clean cog.""" + bot.add_cog(Clean(bot)) diff --git a/bot/exts/moderation/defcon.py b/bot/exts/moderation/defcon.py index dfb1afd19..178be734d 100644 --- a/bot/exts/moderation/defcon.py +++ b/bot/exts/moderation/defcon.py @@ -1,15 +1,14 @@ -import asyncio -import logging import traceback from collections import namedtuple from datetime import datetime from enum import Enum from typing import Optional, Union +import arrow from aioredis import RedisError from async_rediscache import RedisCache from dateutil.relativedelta import relativedelta -from discord import Colour, Embed, Member, User +from discord import Colour, Embed, Forbidden, Member, TextChannel, User from discord.ext import tasks from discord.ext.commands import Cog, Context, group, has_any_role @@ -17,11 +16,12 @@ from bot.bot import Bot from bot.constants import Channels, Colours, Emojis, Event, Icons, MODERATION_ROLES, Roles from bot.converters import DurationDelta, Expiry from bot.exts.moderation.modlog import ModLog +from bot.log import get_logger +from bot.utils import scheduling, time from bot.utils.messages import format_user from bot.utils.scheduling import Scheduler -from bot.utils.time import humanize_delta, parse_duration_string, relativedelta_to_timedelta -log = logging.getLogger(__name__) +log = get_logger(__name__) REJECTION_MESSAGE = """ Hi, {user} - Thanks for your interest in our server! @@ -47,7 +47,7 @@ class Action(Enum): SERVER_OPEN = ActionInfo(Icons.defcon_unshutdown, Emojis.defcon_unshutdown, Colours.soft_green, "") SERVER_SHUTDOWN = ActionInfo(Icons.defcon_shutdown, Emojis.defcon_shutdown, Colours.soft_red, "") DURATION_UPDATE = ActionInfo( - Icons.defcon_update, Emojis.defcon_update, Colour.blurple(), "**Threshold:** {threshold}\n\n" + Icons.defcon_update, Emojis.defcon_update, Colour.og_blurple(), "**Threshold:** {threshold}\n\n" ) @@ -67,7 +67,7 @@ class Defcon(Cog): self.scheduler = Scheduler(self.__class__.__name__) - self.bot.loop.create_task(self._sync_settings()) + scheduling.create_task(self._sync_settings(), event_loop=self.bot.loop) @property def mod_log(self) -> ModLog: @@ -85,7 +85,7 @@ class Defcon(Cog): try: settings = await self.defcon_settings.to_dict() - self.threshold = parse_duration_string(settings["threshold"]) if settings.get("threshold") else None + self.threshold = time.parse_duration_string(settings["threshold"]) if settings.get("threshold") else None self.expiry = datetime.fromisoformat(settings["expiry"]) if settings.get("expiry") else None except RedisError: log.exception("Unable to get DEFCON settings!") @@ -99,7 +99,7 @@ class Defcon(Cog): self.scheduler.schedule_at(self.expiry, 0, self._remove_threshold()) self._update_notifier() - log.info(f"DEFCON synchronized: {humanize_delta(self.threshold) if self.threshold else '-'}") + log.info(f"DEFCON synchronized: {time.humanize_delta(self.threshold) if self.threshold else '-'}") self._update_channel_topic() @@ -107,19 +107,21 @@ class Defcon(Cog): async def on_member_join(self, member: Member) -> None: """Check newly joining users to see if they meet the account age threshold.""" if self.threshold: - now = datetime.utcnow() + now = arrow.utcnow() - if now - member.created_at < relativedelta_to_timedelta(self.threshold): + if now - member.created_at < time.relativedelta_to_timedelta(self.threshold): log.info(f"Rejecting user {member}: Account is too new") message_sent = False try: await member.send(REJECTION_MESSAGE.format(user=member.mention)) - message_sent = True + except Forbidden: + log.debug(f"Cannot send DEFCON rejection DM to {member}: DMs disabled") except Exception: - log.exception(f"Unable to send rejection message to user: {member}") + # Broadly catch exceptions because DM isn't critical, but it's imperative to kick them. + log.exception(f"Error sending DEFCON rejection message to {member}") await member.kick(reason="DEFCON active, user is too new") self.bot.stats.incr("defcon.leaves") @@ -133,7 +135,7 @@ class Defcon(Cog): await self.mod_log.send_log_message( Icons.defcon_denied, Colours.soft_red, "Entry denied", - message, member.avatar_url_as(static_format="png") + message, member.display_avatar.url ) @group(name='defcon', aliases=('dc',), invoke_without_command=True) @@ -146,11 +148,12 @@ class Defcon(Cog): @has_any_role(*MODERATION_ROLES) async def status(self, ctx: Context) -> None: """Check the current status of DEFCON mode.""" + expiry = time.format_relative(self.expiry) if self.expiry else "-" embed = Embed( - colour=Colour.blurple(), title="DEFCON Status", + colour=Colour.og_blurple(), title="DEFCON Status", description=f""" - **Threshold:** {humanize_delta(self.threshold) if self.threshold else "-"} - **Expires in:** {humanize_delta(relativedelta(self.expiry, datetime.utcnow())) if self.expiry else "-"} + **Threshold:** {time.humanize_delta(self.threshold) if self.threshold else "-"} + **Expires:** {expiry} **Verification level:** {ctx.guild.verification_level.name} """ ) @@ -172,7 +175,7 @@ class Defcon(Cog): """ if isinstance(threshold, int): threshold = relativedelta(days=threshold) - await self._update_threshold(ctx.author, threshold=threshold, expiry=expiry) + await self._update_threshold(ctx.author, ctx.channel, threshold, expiry) @defcon_group.command() @has_any_role(Roles.admins) @@ -181,7 +184,12 @@ class Defcon(Cog): role = ctx.guild.default_role permissions = role.permissions - permissions.update(send_messages=False, add_reactions=False, connect=False) + permissions.update( + send_messages=False, + add_reactions=False, + send_messages_in_threads=False, + connect=False + ) await role.edit(reason="DEFCON shutdown", permissions=permissions) await ctx.send(f"{Action.SERVER_SHUTDOWN.value.emoji} Server shut down.") @@ -192,19 +200,31 @@ class Defcon(Cog): role = ctx.guild.default_role permissions = role.permissions - permissions.update(send_messages=True, add_reactions=True, connect=True) + permissions.update( + send_messages=True, + add_reactions=True, + send_messages_in_threads=True, + connect=True + ) await role.edit(reason="DEFCON unshutdown", permissions=permissions) await ctx.send(f"{Action.SERVER_OPEN.value.emoji} Server reopened.") def _update_channel_topic(self) -> None: """Update the #defcon channel topic with the current DEFCON status.""" - new_topic = f"{BASE_CHANNEL_TOPIC}\n(Threshold: {humanize_delta(self.threshold) if self.threshold else '-'})" + threshold = time.humanize_delta(self.threshold) if self.threshold else '-' + new_topic = f"{BASE_CHANNEL_TOPIC}\n(Threshold: {threshold})" self.mod_log.ignore(Event.guild_channel_update, Channels.defcon) - asyncio.create_task(self.channel.edit(topic=new_topic)) + scheduling.create_task(self.channel.edit(topic=new_topic)) @defcon_settings.atomic_transaction - async def _update_threshold(self, author: User, threshold: relativedelta, expiry: Optional[Expiry] = None) -> None: + async def _update_threshold( + self, + author: User, + channel: TextChannel, + threshold: relativedelta, + expiry: Optional[Expiry] = None + ) -> None: """Update the new threshold in the cog, cache, defcon channel, and logs, and additionally schedule expiry.""" self.threshold = threshold if threshold == relativedelta(days=0): # If the threshold is 0, we don't need to schedule anything @@ -234,19 +254,24 @@ class Defcon(Cog): expiry_message = "" if expiry: - expiry_message = f" for the next {humanize_delta(relativedelta(expiry, datetime.utcnow()), max_units=2)}" + formatted_expiry = time.humanize_delta(expiry, max_units=2) + expiry_message = f" for the next {formatted_expiry}" if self.threshold: channel_message = ( - f"updated; accounts must be {humanize_delta(self.threshold)} " + f"updated; accounts must be {time.humanize_delta(self.threshold)} " f"old to join the server{expiry_message}" ) else: channel_message = "removed" - await self.channel.send( - f"{action.value.emoji} DEFCON threshold {channel_message}{error}." - ) + message = f"{action.value.emoji} DEFCON threshold {channel_message}{error}." + await self.channel.send(message) + + # If invoked outside of #defcon send to `ctx.channel` too + if channel != self.channel: + await channel.send(message) + await self._send_defcon_log(action, author) self._update_channel_topic() @@ -254,7 +279,7 @@ class Defcon(Cog): async def _remove_threshold(self) -> None: """Resets the threshold back to 0.""" - await self._update_threshold(self.bot.user, relativedelta(days=0)) + await self._update_threshold(self.bot.user, self.channel, relativedelta(days=0)) @staticmethod def _stringify_relativedelta(delta: relativedelta) -> str: @@ -264,7 +289,7 @@ class Defcon(Cog): def _log_threshold_stat(self, threshold: relativedelta) -> None: """Adds the threshold to the bot stats in days.""" - threshold_days = relativedelta_to_timedelta(threshold).total_seconds() / SECONDS_IN_DAY + threshold_days = time.relativedelta_to_timedelta(threshold).total_seconds() / SECONDS_IN_DAY self.bot.stats.gauge("defcon.threshold", threshold_days) async def _send_defcon_log(self, action: Action, actor: User) -> None: @@ -272,7 +297,7 @@ class Defcon(Cog): info = action.value log_msg: str = ( f"**Staffer:** {actor.mention} {actor} (`{actor.id}`)\n" - f"{info.template.format(threshold=(humanize_delta(self.threshold) if self.threshold else '-'))}" + f"{info.template.format(threshold=(time.humanize_delta(self.threshold) if self.threshold else '-'))}" ) status_msg = f"DEFCON {action.name.lower()}" @@ -291,7 +316,7 @@ class Defcon(Cog): @tasks.loop(hours=1) async def defcon_notifier(self) -> None: """Routinely notify moderators that DEFCON is active.""" - await self.channel.send(f"Defcon is on and is set to {humanize_delta(self.threshold)}.") + await self.channel.send(f"Defcon is on and is set to {time.humanize_delta(self.threshold)}.") def cog_unload(self) -> None: """Cancel the notifer and threshold removal tasks when the cog unloads.""" diff --git a/bot/exts/moderation/dm_relay.py b/bot/exts/moderation/dm_relay.py index 1d2206e27..566422e29 100644 --- a/bot/exts/moderation/dm_relay.py +++ b/bot/exts/moderation/dm_relay.py @@ -1,13 +1,13 @@ -import logging - import discord from discord.ext.commands import Cog, Context, command, has_any_role from bot.bot import Bot from bot.constants import Emojis, MODERATION_ROLES +from bot.log import get_logger +from bot.utils.channel import is_mod_channel from bot.utils.services import send_to_paste_service -log = logging.getLogger(__name__) +log = get_logger(__name__) class DMRelay(Cog): @@ -63,8 +63,9 @@ class DMRelay(Cog): await ctx.send(paste_link) async def cog_check(self, ctx: Context) -> bool: - """Only allow moderators to invoke the commands in this cog.""" - return await has_any_role(*MODERATION_ROLES).predicate(ctx) + """Only allow moderators to invoke the commands in this cog in mod channels.""" + return (await has_any_role(*MODERATION_ROLES).predicate(ctx) + and is_mod_channel(ctx.channel)) def setup(bot: Bot) -> None: diff --git a/bot/exts/moderation/incidents.py b/bot/exts/moderation/incidents.py index 0e479d33f..b579416a6 100644 --- a/bot/exts/moderation/incidents.py +++ b/bot/exts/moderation/incidents.py @@ -1,17 +1,20 @@ import asyncio -import logging -import typing as t +import re from datetime import datetime from enum import Enum +from typing import Optional import discord -from discord.ext.commands import Cog +from async_rediscache import RedisCache +from discord.ext.commands import Cog, Context, MessageConverter, MessageNotFound from bot.bot import Bot -from bot.constants import Channels, Colours, Emojis, Guild, Webhooks -from bot.utils.messages import sub_clyde +from bot.constants import Channels, Colours, Emojis, Guild, Roles, Webhooks +from bot.log import get_logger +from bot.utils import scheduling +from bot.utils.messages import format_user, sub_clyde -log = logging.getLogger(__name__) +log = get_logger(__name__) # Amount of messages for `crawl_task` to process at most on start-up - limited to 50 # as in practice, there should never be this many messages, and if there are, @@ -21,6 +24,12 @@ CRAWL_LIMIT = 50 # Seconds for `crawl_task` to sleep after adding reactions to a message CRAWL_SLEEP = 2 +DISCORD_MESSAGE_LINK_RE = re.compile( + r"(https?:\/\/(?:(ptb|canary|www)\.)?discord(?:app)?\.com\/channels\/" + r"[0-9]{15,20}" + r"\/[0-9]{15,20}\/[0-9]{15,20})" +) + class Signal(Enum): """ @@ -36,17 +45,17 @@ class Signal(Enum): # Reactions from non-mod roles will be removed -ALLOWED_ROLES: t.Set[int] = set(Guild.moderation_roles) +ALLOWED_ROLES: set[int] = set(Guild.moderation_roles) # Message must have all of these emoji to pass the `has_signals` check -ALL_SIGNALS: t.Set[str] = {signal.value for signal in Signal} +ALL_SIGNALS: set[str] = {signal.value for signal in Signal} # An embed coupled with an optional file to be dispatched # If the file is not None, the embed attempts to show it in its body -FileEmbed = t.Tuple[discord.Embed, t.Optional[discord.File]] +FileEmbed = tuple[discord.Embed, Optional[discord.File]] -async def download_file(attachment: discord.Attachment) -> t.Optional[discord.File]: +async def download_file(attachment: discord.Attachment) -> Optional[discord.File]: """ Download & return `attachment` file. @@ -93,7 +102,7 @@ async def make_embed(incident: discord.Message, outcome: Signal, actioned_by: di timestamp=datetime.utcnow(), colour=colour, ) - embed.set_footer(text=footer, icon_url=actioned_by.avatar_url) + embed.set_footer(text=footer, icon_url=actioned_by.display_avatar.url) if incident.attachments: attachment = incident.attachments[0] # User-sent messages can only contain one attachment @@ -104,7 +113,7 @@ async def make_embed(incident: discord.Message, outcome: Signal, actioned_by: di else: embed.set_author(name="[Failed to relay attachment]", url=attachment.proxy_url) # Embed links the file else: - file = None + file = discord.utils.MISSING return embed, file @@ -120,7 +129,7 @@ def is_incident(message: discord.Message) -> bool: return all(conditions) -def own_reactions(message: discord.Message) -> t.Set[str]: +def own_reactions(message: discord.Message) -> set[str]: """Get the set of reactions placed on `message` by the bot itself.""" return {str(reaction.emoji) for reaction in message.reactions if reaction.me} @@ -130,6 +139,109 @@ def has_signals(message: discord.Message) -> bool: return ALL_SIGNALS.issubset(own_reactions(message)) +def shorten_text(text: str) -> str: + """ + Truncate the text if there are over 3 lines or 300 characters, or if it is a single word. + + The maximum length of the string would be 303 characters across 3 lines at maximum. + """ + original_length = len(text) + # Truncate text to a maximum of 300 characters + if len(text) > 300: + text = text[:300] + + # Limit to a maximum of three lines + text = "\n".join(text.split("\n", maxsplit=3)[:3]) + + # If it is a single word, then truncate it to 50 characters + if text.find(" ") == -1: + text = text[:50] + + # Remove extra whitespaces from the `text` + text = text.strip() + + # Add placeholder if the text was shortened + if len(text) < original_length: + text = f"{text}..." + + return text + + +async def make_message_link_embed(ctx: Context, message_link: str) -> Optional[discord.Embed]: + """ + Create an embedded representation of the discord message link contained in the incident report. + + The Embed would contain the following information --> + Author: @Jason Terror ♦ (736234578745884682) + Channel: Special/#bot-commands (814190307980607493) + Content: This is a very important message! + """ + embed = None + + try: + message: discord.Message = await MessageConverter().convert(ctx, message_link) + except MessageNotFound: + mod_logs_channel = ctx.bot.get_channel(Channels.mod_log) + + last_100_logs: list[discord.Message] = await mod_logs_channel.history(limit=100).flatten() + + for log_entry in last_100_logs: + if not log_entry.embeds: + continue + + log_embed: discord.Embed = log_entry.embeds[0] + if ( + log_embed.author.name == "Message deleted" + and f"[Jump to message]({message_link})" in log_embed.description + ): + embed = discord.Embed( + colour=discord.Colour.dark_gold(), + title="Deleted Message Link", + description=( + f"Found <#{Channels.mod_log}> entry for deleted message: " + f"[Jump to message]({log_entry.jump_url})." + ) + ) + if not embed: + embed = discord.Embed( + colour=discord.Colour.red(), + title="Bad Message Link", + description=f"Message {message_link} not found." + ) + except discord.DiscordException as e: + log.exception(f"Failed to make message link embed for '{message_link}', raised exception: {e}") + else: + channel = message.channel + if not channel.permissions_for(channel.guild.get_role(Roles.helpers)).view_channel: + log.info( + f"Helpers don't have read permissions in #{channel.name}," + f" not sending message link embed for {message_link}" + ) + return + + embed = discord.Embed( + colour=discord.Colour.gold(), + description=( + f"**Author:** {format_user(message.author)}\n" + f"**Channel:** {channel.mention} ({channel.category}" + f"{f'/#{channel.parent.name} - ' if isinstance(channel, discord.Thread) else '/#'}" + f"{channel.name})\n" + ), + timestamp=message.created_at + ) + embed.set_author(name=message.author, icon_url=message.author.display_avatar.url) + embed.add_field( + name="Content", + value=shorten_text(message.content) if message.content else "[No Message Content]" + ) + embed.set_footer(text=f"Message ID: {message.id}") + + if message.attachments: + embed.set_image(url=message.attachments[0].url) + + return embed + + async def add_signals(incident: discord.Message) -> None: """ Add `Signal` member emoji to `incident` as reactions. @@ -143,7 +255,14 @@ async def add_signals(incident: discord.Message) -> None: log.trace(f"Skipping emoji as it's already been placed: {signal_emoji}") else: log.trace(f"Adding reaction: {signal_emoji}") - await incident.add_reaction(signal_emoji.value) + try: + await incident.add_reaction(signal_emoji.value) + except discord.NotFound as e: + if e.code != 10008: + raise + + log.trace(f"Couldn't react with signal because message {incident.id} was deleted; skipping incident") + return class Incidents(Cog): @@ -160,6 +279,7 @@ class Incidents(Cog): * See: `crawl_incidents` On message: + * Run message through `extract_message_links` and send them into the channel * Add `Signal` member emoji if message qualifies as an incident * Ignore messages starting with # * Use this if verbal communication is necessary @@ -173,17 +293,34 @@ class Incidents(Cog): * If `Signal.ACTIONED` or `Signal.NOT_ACTIONED` were chosen, attempt to relay the incident message to #incidents-archive * If relay successful, delete original message + * Delete quotation message if cached * See: `on_raw_reaction_add` Please refer to function docstrings for implementation details. """ + # This dictionary maps an incident report message to the message link embed's ID + # RedisCache[discord.Message.id, discord.Message.id] + message_link_embeds_cache = RedisCache() + def __init__(self, bot: Bot) -> None: """Prepare `event_lock` and schedule `crawl_task` on start-up.""" self.bot = bot + self.incidents_webhook = None + + scheduling.create_task(self.fetch_webhook(), event_loop=self.bot.loop) self.event_lock = asyncio.Lock() - self.crawl_task = self.bot.loop.create_task(self.crawl_incidents()) + self.crawl_task = scheduling.create_task(self.crawl_incidents(), event_loop=self.bot.loop) + + async def fetch_webhook(self) -> None: + """Fetch the incidents webhook object, so we can post message link embeds to it.""" + await self.bot.wait_until_guild_available() + + try: + self.incidents_webhook = await self.bot.fetch_webhook(Webhooks.incidents) + except discord.HTTPException: + log.error(f"Failed to fetch incidents webhook with id `{Webhooks.incidents}`.") async def crawl_incidents(self) -> None: """ @@ -245,7 +382,7 @@ class Incidents(Cog): await webhook.send( embed=embed, username=sub_clyde(incident.author.name), - avatar_url=incident.author.avatar_url, + avatar_url=incident.author.display_avatar.url, file=attachment_file, ) except Exception: @@ -268,7 +405,7 @@ class Incidents(Cog): return payload.message_id == incident.id coroutine = self.bot.wait_for(event="raw_message_delete", check=check, timeout=timeout) - return self.bot.loop.create_task(coroutine) + return scheduling.create_task(coroutine, event_loop=self.bot.loop) async def process_event(self, reaction: str, incident: discord.Message, member: discord.Member) -> None: """ @@ -284,18 +421,27 @@ class Incidents(Cog): This ensures that if there is a racing event awaiting the lock, it will fail to find the message, and will abort. There is a `timeout` to ensure that this doesn't hold the lock forever should something go wrong. + + Deletes cache value (`message_link_embeds_cache`) of `incident` if it exists. It then removes the + webhook message for that particular link from the channel. """ - members_roles: t.Set[int] = {role.id for role in member.roles} + members_roles: set[int] = {role.id for role in member.roles} if not members_roles & ALLOWED_ROLES: # Intersection is truthy on at least 1 common element log.debug(f"Removing invalid reaction: user {member} is not permitted to send signals") - await incident.remove_reaction(reaction, member) + try: + await incident.remove_reaction(reaction, member) + except discord.NotFound: + log.trace("Couldn't remove reaction because the reaction or its message was deleted") return try: signal = Signal(reaction) except ValueError: log.debug(f"Removing invalid reaction: emoji {reaction} is not a valid signal") - await incident.remove_reaction(reaction, member) + try: + await incident.remove_reaction(reaction, member) + except discord.NotFound: + log.trace("Couldn't remove reaction because the reaction or its message was deleted") return log.trace(f"Received signal: {signal}") @@ -313,7 +459,10 @@ class Incidents(Cog): confirmation_task = self.make_confirmation_task(incident, timeout) log.trace("Deleting original message") - await incident.delete() + try: + await incident.delete() + except discord.NotFound: + log.trace("Couldn't delete message because it was already deleted") log.trace(f"Awaiting deletion confirmation: {timeout=} seconds") try: @@ -323,7 +472,11 @@ class Incidents(Cog): else: log.trace("Deletion was confirmed") - async def resolve_message(self, message_id: int) -> t.Optional[discord.Message]: + if self.incidents_webhook: + # Deletes the message link embeds found in cache from the channel and cache. + await self.delete_msg_link_embed(incident.id) + + async def resolve_message(self, message_id: int) -> Optional[discord.Message]: """ Get `discord.Message` for `message_id` from cache, or API. @@ -338,7 +491,7 @@ class Incidents(Cog): """ await self.bot.wait_until_guild_available() # First make sure that the cache is ready log.trace(f"Resolving message for: {message_id=}") - message: t.Optional[discord.Message] = self.bot._connection._get_message(message_id) + message: Optional[discord.Message] = self.bot._connection._get_message(message_id) if message is not None: log.trace("Message was found in cache") @@ -402,9 +555,107 @@ class Incidents(Cog): @Cog.listener() async def on_message(self, message: discord.Message) -> None: - """Pass `message` to `add_signals` if and only if it satisfies `is_incident`.""" - if is_incident(message): - await add_signals(message) + """ + Pass `message` to `add_signals` and `extract_message_links` if it satisfies `is_incident`. + + If `message` is an incident report, then run it through `extract_message_links` to get all + the message link embeds (embeds which contain information about that particular link). + These message link embeds are then sent into the channel. + + Also passes the message into `add_signals` if the message is an incident. + """ + if not is_incident(message): + return + + await add_signals(message) + + # Only use this feature if incidents webhook embed is found + if self.incidents_webhook: + if embed_list := await self.extract_message_links(message): + await self.send_message_link_embeds(embed_list, message, self.incidents_webhook) + + @Cog.listener() + async def on_raw_message_delete(self, payload: discord.RawMessageDeleteEvent) -> None: + """ + Delete message link embeds for `payload.message_id`. + + Search through the cache for message, if found delete it from cache and channel. + """ + if self.incidents_webhook: + await self.delete_msg_link_embed(payload.message_id) + + async def extract_message_links(self, message: discord.Message) -> Optional[list[discord.Embed]]: + """ + Check if there's any message links in the text content. + + Then pass the message_link into `make_message_link_embed` to format an + embed for it containing information about the link. + + As Discord only allows a max of 10 embeds in a single webhook, just send the + first 10 embeds and don't care about the rest. + + If no links are found for the message, just log a trace statement. + """ + message_links = DISCORD_MESSAGE_LINK_RE.findall(message.content) + if not message_links: + log.trace( + f"No message links detected on incident message with id {message.id}." + ) + return + + embeds = [] + for message_link in message_links[:10]: + ctx = await self.bot.get_context(message) + embed = await make_message_link_embed(ctx, message_link[0]) + if embed: + embeds.append(embed) + + return embeds + + async def send_message_link_embeds( + self, + webhook_embed_list: list, + message: discord.Message, + webhook: discord.Webhook, + ) -> Optional[int]: + """ + Send message link embeds to #incidents channel. + + Using the `webhook` passed in as a parameter to send + the embeds in the `webhook_embed_list` parameter. + + After sending each embed it maps the `message.id` + to the `webhook_msg_ids` IDs in the async redis-cache. + """ + try: + webhook_msg = await webhook.send( + embeds=[embed for embed in webhook_embed_list if embed], + username=sub_clyde(message.author.name), + avatar_url=message.author.display_avatar.url, + wait=True, + ) + except discord.DiscordException: + log.exception( + f"Failed to send message link embed {message.id} to #incidents." + ) + else: + await self.message_link_embeds_cache.set(message.id, webhook_msg.id) + log.trace("Message link embeds sent successfully to #incidents!") + return webhook_msg.id + + async def delete_msg_link_embed(self, message_id: int) -> None: + """Delete the Discord message link message found in cache for `message_id`.""" + log.trace("Deleting Discord message link's webhook message.") + webhook_msg_id = await self.message_link_embeds_cache.get(int(message_id)) + + if webhook_msg_id: + try: + await self.incidents_webhook.delete_message(webhook_msg_id) + except discord.errors.NotFound: + log.trace(f"Incidents message link embed (`{webhook_msg_id}`) has already been deleted, skipping.") + + await self.message_link_embeds_cache.delete(message_id) + log.trace("Successfully deleted discord links webhook message.") def setup(bot: Bot) -> None: diff --git a/bot/exts/moderation/infraction/_scheduler.py b/bot/exts/moderation/infraction/_scheduler.py index 19402d01d..2fc54856f 100644 --- a/bot/exts/moderation/infraction/_scheduler.py +++ b/bot/exts/moderation/infraction/_scheduler.py @@ -1,10 +1,9 @@ -import logging import textwrap import typing as t from abc import abstractmethod -from datetime import datetime from gettext import ngettext +import arrow import dateutil.parser import discord from discord.ext.commands import Context @@ -13,13 +12,14 @@ from bot import constants from bot.api import ResponseCodeError from bot.bot import Bot from bot.constants import Colours +from bot.converters import MemberOrUser from bot.exts.moderation.infraction import _utils -from bot.exts.moderation.infraction._utils import UserSnowflake from bot.exts.moderation.modlog import ModLog +from bot.log import get_logger from bot.utils import messages, scheduling, time from bot.utils.channel import is_mod_channel -log = logging.getLogger(__name__) +log = get_logger(__name__) class InfractionScheduler: @@ -29,7 +29,7 @@ class InfractionScheduler: self.bot = bot self.scheduler = scheduling.Scheduler(self.__class__.__name__) - self.bot.loop.create_task(self.reschedule_infractions(supported_infractions)) + scheduling.create_task(self.reschedule_infractions(supported_infractions), event_loop=self.bot.loop) def cog_unload(self) -> None: """Cancel scheduled tasks.""" @@ -67,7 +67,7 @@ class InfractionScheduler: # We make sure to fire this if to_schedule: next_reschedule_point = max( - dateutil.parser.isoparse(infr["expires_at"]).replace(tzinfo=None) for infr in to_schedule + dateutil.parser.isoparse(infr["expires_at"]) for infr in to_schedule ) log.trace("Will reschedule remaining infractions at %s", next_reschedule_point) @@ -81,12 +81,16 @@ class InfractionScheduler: apply_coro: t.Optional[t.Awaitable] ) -> None: """Reapply an infraction if it's still active or deactivate it if less than 60 sec left.""" - # Calculate the time remaining, in seconds, for the mute. - expiry = dateutil.parser.isoparse(infraction["expires_at"]).replace(tzinfo=None) - delta = (expiry - datetime.utcnow()).total_seconds() + if infraction["expires_at"] is not None: + # Calculate the time remaining, in seconds, for the mute. + expiry = dateutil.parser.isoparse(infraction["expires_at"]) + delta = (expiry - arrow.utcnow()).total_seconds() + else: + # If the infraction is permanent, it is not possible to get the time remaining. + delta = None - # Mark as inactive if less than a minute remains. - if delta < 60: + # Mark as inactive if the infraction is not permanent and less than a minute remains. + if delta is not None and delta < 60: log.info( "Infraction will be deactivated instead of re-applied " "because less than 1 minute remains." @@ -115,7 +119,7 @@ class InfractionScheduler: self, ctx: Context, infraction: _utils.Infraction, - user: UserSnowflake, + user: MemberOrUser, action_coro: t.Optional[t.Awaitable] = None, user_reason: t.Optional[str] = None, additional_info: str = "", @@ -132,7 +136,7 @@ class InfractionScheduler: infr_type = infraction["type"] icon = _utils.INFRACTION_ICONS[infr_type][0] reason = infraction["reason"] - expiry = time.format_infraction_with_duration(infraction["expires_at"]) + expiry = time.format_with_duration(infraction["expires_at"]) id_ = infraction['id'] if user_reason is None: @@ -161,7 +165,7 @@ class InfractionScheduler: # send DMs to user that it doesn't share a guild with. If we were to # apply kick/ban infractions first, this would mean that we'd make it # impossible for us to deliver a DM. See python-discord/bot#982. - if not infraction["hidden"]: + if not infraction["hidden"] and infr_type in {"ban", "kick"}: if await _utils.notify_infraction(infraction, user, user_reason): dm_result = ":incoming_envelope: " dm_log_text = "\nDM: Sent" @@ -170,13 +174,7 @@ class InfractionScheduler: dm_log_text = "\nDM: **Failed**" end_msg = "" - if infraction["actor"] == self.bot.user.id: - log.trace( - f"Infraction #{id_} actor is bot; including the reason in the confirmation message." - ) - if reason: - end_msg = f" (reason: {textwrap.shorten(reason, width=1500, placeholder='...')})" - elif is_mod_channel(ctx.channel): + if is_mod_channel(ctx.channel): log.trace(f"Fetching total infraction count for {user}.") infractions = await self.bot.api_client.get( @@ -185,6 +183,12 @@ class InfractionScheduler: ) total = len(infractions) end_msg = f" (#{id_} ; {total} infraction{ngettext('', 's', total)} total)" + elif infraction["actor"] == self.bot.user.id: + log.trace( + f"Infraction #{id_} actor is bot; including the reason in the confirmation message." + ) + if reason: + end_msg = f" (reason: {textwrap.shorten(reason, width=1500, placeholder='...')})" purge = infraction.get("purge", "") @@ -216,7 +220,7 @@ class InfractionScheduler: failed = True if failed: - log.trace(f"Deleted infraction {infraction['id']} from database because applying infraction failed.") + log.trace(f"Trying to delete infraction {id_} from database because applying infraction failed.") try: await self.bot.api_client.delete(f"bot/infractions/{id_}") except ResponseCodeError as e: @@ -227,6 +231,15 @@ class InfractionScheduler: else: infr_message = f" **{purge}{' '.join(infr_type.split('_'))}** to {user.mention}{expiry_msg}{end_msg}" + # If we need to DM and haven't already tried to + if not infraction["hidden"] and infr_type not in {"ban", "kick"}: + if await _utils.notify_infraction(infraction, user, user_reason): + dm_result = ":incoming_envelope: " + dm_log_text = "\nDM: Sent" + else: + dm_result = f"{constants.Emojis.failmail} " + dm_log_text = "\nDM: **Failed**" + # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} confirmation message.") await ctx.send(f"{dm_result}{confirm_msg}{infr_message}.") @@ -238,7 +251,7 @@ class InfractionScheduler: icon_url=icon, colour=Colours.soft_red, title=f"Infraction {log_title}: {' '.join(infr_type.split('_'))}", - thumbnail=user.avatar_url_as(static_format="png"), + thumbnail=user.display_avatar.url, text=textwrap.dedent(f""" Member: {messages.format_user(user)} Actor: {ctx.author.mention}{dm_log_text}{expiry_log_text} @@ -246,7 +259,7 @@ class InfractionScheduler: {additional_info} """), content=log_content, - footer=f"ID {infraction['id']}" + footer=f"ID: {id_}" ) log.info(f"Applied {purge}{infr_type} infraction #{id_} to {user}.") @@ -256,14 +269,18 @@ class InfractionScheduler: self, ctx: Context, infr_type: str, - user: UserSnowflake, - send_msg: bool = True + user: MemberOrUser, + *, + send_msg: bool = True, + notify: bool = True ) -> None: """ Prematurely end an infraction for a user and log the action in the mod log. If `send_msg` is True, then a pardoning confirmation message will be sent to - the context channel. Otherwise, no such message will be sent. + the context channel. Otherwise, no such message will be sent. + + If `notify` is True, notify the user of the pardon via DM where applicable. """ log.trace(f"Pardoning {infr_type} infraction for {user}.") @@ -284,7 +301,7 @@ class InfractionScheduler: return # Deactivate the infraction and cancel its scheduled expiration task. - log_text = await self.deactivate_infraction(response[0], send_log=False) + log_text = await self.deactivate_infraction(response[0], send_log=False, notify=notify) log_text["Member"] = messages.format_user(user) log_text["Actor"] = ctx.author.mention @@ -328,7 +345,7 @@ class InfractionScheduler: icon_url=_utils.INFRACTION_ICONS[infr_type][1], colour=Colours.soft_green, title=f"Infraction {log_title}: {' '.join(infr_type.split('_'))}", - thumbnail=user.avatar_url_as(static_format="png"), + thumbnail=user.display_avatar.url, text="\n".join(f"{k}: {v}" for k, v in log_text.items()), footer=footer, content=log_content, @@ -337,7 +354,9 @@ class InfractionScheduler: async def deactivate_infraction( self, infraction: _utils.Infraction, - send_log: bool = True + *, + send_log: bool = True, + notify: bool = True ) -> t.Dict[str, str]: """ Deactivate an active infraction and return a dictionary of lines to send in a mod log. @@ -346,6 +365,8 @@ class InfractionScheduler: expiration task cancelled. If `send_log` is True, a mod log is sent for the deactivation of the infraction. + If `notify` is True, notify the user of the pardon via DM where applicable. + Infractions of unsupported types will raise a ValueError. """ guild = self.bot.get_guild(constants.Guild.id) @@ -354,25 +375,20 @@ class InfractionScheduler: actor = infraction["actor"] type_ = infraction["type"] id_ = infraction["id"] - inserted_at = infraction["inserted_at"] - expiry = infraction["expires_at"] log.info(f"Marking infraction #{id_} as inactive (expired).") - expiry = dateutil.parser.isoparse(expiry).replace(tzinfo=None) if expiry else None - created = time.format_infraction_with_duration(inserted_at, expiry) - log_content = None log_text = { "Member": f"<@{user_id}>", "Actor": f"<@{actor}>", "Reason": infraction["reason"], - "Created": created, + "Created": time.format_with_duration(infraction["inserted_at"], infraction["expires_at"]), } try: log.trace("Awaiting the pardon action coroutine.") - returned_log = await self._pardon_action(infraction) + returned_log = await self._pardon_action(infraction, notify) if returned_log is not None: log_text = {**log_text, **returned_log} # Merge the logs together @@ -441,7 +457,7 @@ class InfractionScheduler: log_title = "expiration failed" if "Failure" in log_text else "expired" user = self.bot.get_user(user_id) - avatar = user.avatar_url_as(static_format="png") if user else None + avatar = user.display_avatar.url if user else None # Move reason to end so when reason is too long, this is not gonna cut out required items. log_text["Reason"] = log_text.pop("Reason") @@ -460,10 +476,15 @@ class InfractionScheduler: return log_text @abstractmethod - async def _pardon_action(self, infraction: _utils.Infraction) -> t.Optional[t.Dict[str, str]]: + async def _pardon_action( + self, + infraction: _utils.Infraction, + notify: bool + ) -> t.Optional[t.Dict[str, str]]: """ Execute deactivation steps specific to the infraction's type and return a log dict. + If `notify` is True, notify the user of the pardon via DM where applicable. If an infraction type is unsupported, return None instead. """ raise NotImplementedError @@ -475,5 +496,5 @@ class InfractionScheduler: At the time of expiration, the infraction is marked as inactive on the website and the expiration task is cancelled. """ - expiry = dateutil.parser.isoparse(infraction["expires_at"]).replace(tzinfo=None) + expiry = dateutil.parser.isoparse(infraction["expires_at"]) self.scheduler.schedule_at(expiry, infraction["id"], self.deactivate_infraction(infraction)) diff --git a/bot/exts/moderation/infraction/_utils.py b/bot/exts/moderation/infraction/_utils.py index a6f180c8c..eec539fee 100644 --- a/bot/exts/moderation/infraction/_utils.py +++ b/bot/exts/moderation/infraction/_utils.py @@ -1,4 +1,3 @@ -import logging import typing as t from datetime import datetime @@ -8,10 +7,12 @@ from discord.ext.commands import Context import bot from bot.api import ResponseCodeError from bot.constants import Colours, Icons +from bot.converters import MemberOrUser from bot.errors import InvalidInfractedUserError +from bot.log import get_logger from bot.utils import time -log = logging.getLogger(__name__) +log = get_logger(__name__) # apply icon, pardon icon INFRACTION_ICONS = { @@ -21,25 +22,25 @@ INFRACTION_ICONS = { "note": (Icons.user_warn, None), "superstar": (Icons.superstarify, Icons.unsuperstarify), "warning": (Icons.user_warn, None), - "voice_ban": (Icons.voice_state_red, Icons.voice_state_green), + "voice_mute": (Icons.voice_state_red, Icons.voice_state_green), } RULES_URL = "https://pythondiscord.com/pages/rules" # Type aliases -UserObject = t.Union[discord.Member, discord.User] -UserSnowflake = t.Union[UserObject, discord.Object] Infraction = t.Dict[str, t.Union[str, int, bool]] -APPEAL_EMAIL = "[email protected]" +APPEAL_SERVER_INVITE = "https://discord.gg/WXrCJxWBnm" INFRACTION_TITLE = "Please review our rules" -INFRACTION_APPEAL_EMAIL_FOOTER = f"To appeal this infraction, send an e-mail to {APPEAL_EMAIL}" +INFRACTION_APPEAL_SERVER_FOOTER = f"\nTo appeal this infraction, join our [appeals server]({APPEAL_SERVER_INVITE})." INFRACTION_APPEAL_MODMAIL_FOOTER = ( - 'If you would like to discuss or appeal this infraction, ' - 'send a message to the ModMail bot' + '\nIf you would like to discuss or appeal this infraction, ' + 'send a message to the ModMail bot.' ) INFRACTION_AUTHOR_NAME = "Infraction information" +LONGEST_EXTRAS = max(len(INFRACTION_APPEAL_SERVER_FOOTER), len(INFRACTION_APPEAL_MODMAIL_FOOTER)) + INFRACTION_DESCRIPTION_TEMPLATE = ( "**Type:** {type}\n" "**Expires:** {expires}\n" @@ -47,7 +48,7 @@ INFRACTION_DESCRIPTION_TEMPLATE = ( ) -async def post_user(ctx: Context, user: UserSnowflake) -> t.Optional[dict]: +async def post_user(ctx: Context, user: MemberOrUser) -> t.Optional[dict]: """ Create a new user in the database. @@ -55,14 +56,11 @@ async def post_user(ctx: Context, user: UserSnowflake) -> t.Optional[dict]: """ log.trace(f"Attempting to add user {user.id} to the database.") - if not isinstance(user, (discord.Member, discord.User)): - log.debug("The user being added to the DB is not a Member or User object.") - payload = { - 'discriminator': int(getattr(user, 'discriminator', 0)), + 'discriminator': int(user.discriminator), 'id': user.id, 'in_guild': False, - 'name': getattr(user, 'name', 'Name unknown'), + 'name': user.name, 'roles': [] } @@ -77,12 +75,13 @@ async def post_user(ctx: Context, user: UserSnowflake) -> t.Optional[dict]: async def post_infraction( ctx: Context, - user: UserSnowflake, + user: MemberOrUser, infr_type: str, reason: str, expires_at: datetime = None, hidden: bool = False, - active: bool = True + active: bool = True, + dm_sent: bool = False, ) -> t.Optional[dict]: """Posts an infraction to the API.""" if isinstance(user, (discord.Member, discord.User)) and user.bot: @@ -97,7 +96,8 @@ async def post_infraction( "reason": reason, "type": infr_type, "user": user.id, - "active": active + "active": active, + "dm_sent": dm_sent } if expires_at: payload['expires_at'] = expires_at.isoformat() @@ -120,7 +120,7 @@ async def post_infraction( async def get_active_infraction( ctx: Context, - user: UserSnowflake, + user: MemberOrUser, infr_type: str, send_msg: bool = True ) -> t.Optional[dict]: @@ -145,34 +145,52 @@ async def get_active_infraction( # Checks to see if the moderator should be told there is an active infraction if send_msg: log.trace(f"{user} has active infractions of type {infr_type}.") - await ctx.send( - f":x: According to my records, this user already has a {infr_type} infraction. " - f"See infraction **#{active_infractions[0]['id']}**." - ) + await send_active_infraction_message(ctx, active_infractions[0]) return active_infractions[0] else: log.trace(f"{user} does not have active infractions of type {infr_type}.") -async def send_infraction_embed( - user: UserObject, - infr_type: str, - expires_at: t.Optional[str] = None, - reason: t.Optional[str] = None, - icon_url: str = Icons.token_removed +async def send_active_infraction_message(ctx: Context, infraction: Infraction) -> None: + """Send a message stating that the given infraction is active.""" + await ctx.send( + f":x: According to my records, this user already has a {infraction['type']} infraction. " + f"See infraction **#{infraction['id']}**." + ) + + +async def notify_infraction( + infraction: Infraction, + user: MemberOrUser, + reason: t.Optional[str] = None ) -> bool: - """DM a user about their new infraction and return True if the DM is successful.""" + """ + DM a user about their new infraction and return True if the DM is successful. + + `reason` can be used to override what is in `infraction`. Otherwise, this data will + be retrieved from `infraction`. + """ + infr_id = infraction["id"] + infr_type = infraction["type"].replace("_", " ").title() + icon_url = INFRACTION_ICONS[infraction["type"]][0] + expires_at = time.format_with_duration(infraction["expires_at"]) + log.trace(f"Sending {user} a DM about their {infr_type} infraction.") + if reason is None: + reason = infraction["reason"] + text = INFRACTION_DESCRIPTION_TEMPLATE.format( type=infr_type.title(), - expires=f"{expires_at} UTC" if expires_at else "N/A", + expires=expires_at or "N/A", reason=reason or "No reason provided." ) # For case when other fields than reason is too long and this reach limit, then force-shorten string - if len(text) > 2048: - text = f"{text[:2045]}..." + if len(text) > 4096 - LONGEST_EXTRAS: + text = f"{text[:4093-LONGEST_EXTRAS]}..." + + text += INFRACTION_APPEAL_SERVER_FOOTER if infraction["type"] == 'ban' else INFRACTION_APPEAL_MODMAIL_FOOTER embed = discord.Embed( description=text, @@ -183,49 +201,19 @@ async def send_infraction_embed( embed.title = INFRACTION_TITLE embed.url = RULES_URL - embed.set_footer( - text=INFRACTION_APPEAL_EMAIL_FOOTER if infr_type == 'Ban' else INFRACTION_APPEAL_MODMAIL_FOOTER - ) - - return await send_private_embed(user, embed) - - -async def notify_infraction( - infraction: Infraction, - user: t.Optional[UserSnowflake] = None, - reason: t.Optional[str] = None -) -> bool: - """ - DM a user about their new infraction and return True if the DM is successful. - - `user` and `reason` can be used to override what is in `infraction`. Otherwise, this data will - be retrieved from `infraction`. - - Also return False if the user needs to be fetched but fails to be fetched. - """ - if user is None: - user = discord.Object(infraction["user"]) - - # Sometimes user is a discord.Object; make it a proper user. - try: - if not isinstance(user, (discord.Member, discord.User)): - user = await bot.instance.fetch_user(user.id) - except discord.HTTPException as e: - log.error(f"Failed to DM {user.id}: could not fetch user (status {e.status})") - return False - - type_ = infraction["type"].replace("_", " ").title() - icon = INFRACTION_ICONS[infraction["type"]][0] - expiry = time.format_infraction_with_duration(infraction["expires_at"]) - - if reason is None: - reason = infraction["reason"] + dm_sent = await send_private_embed(user, embed) + if dm_sent: + await bot.instance.api_client.patch( + f"bot/infractions/{infr_id}", + json={"dm_sent": True} + ) + log.debug(f"Update infraction #{infr_id} dm_sent field to true.") - return await send_infraction_embed(user, type_, expiry, reason, icon) + return dm_sent async def notify_pardon( - user: UserObject, + user: MemberOrUser, title: str, content: str, icon_url: str = Icons.user_verified @@ -243,7 +231,7 @@ async def notify_pardon( return await send_private_embed(user, embed) -async def send_private_embed(user: UserObject, embed: discord.Embed) -> bool: +async def send_private_embed(user: MemberOrUser, embed: discord.Embed) -> bool: """ A helper method for sending an embed to a user's DMs. diff --git a/bot/exts/moderation/infraction/infractions.py b/bot/exts/moderation/infraction/infractions.py index f19323c7c..af42ab1b8 100644 --- a/bot/exts/moderation/infraction/infractions.py +++ b/bot/exts/moderation/infraction/infractions.py @@ -1,4 +1,3 @@ -import logging import textwrap import typing as t @@ -10,14 +9,20 @@ from discord.ext.commands import Context, command from bot import constants from bot.bot import Bot from bot.constants import Event -from bot.converters import Duration, Expiry, FetchedMember +from bot.converters import Age, Duration, Expiry, MemberOrUser, UnambiguousMemberOrUser from bot.decorators import respect_role_hierarchy from bot.exts.moderation.infraction import _utils from bot.exts.moderation.infraction._scheduler import InfractionScheduler -from bot.exts.moderation.infraction._utils import UserSnowflake +from bot.log import get_logger +from bot.utils.members import get_or_fetch_member from bot.utils.messages import format_user -log = logging.getLogger(__name__) +log = get_logger(__name__) + +if t.TYPE_CHECKING: + from bot.exts.moderation.clean import Clean + from bot.exts.moderation.infraction.management import ModManagement + from bot.exts.moderation.watchchannels.bigbrother import BigBrother class Infractions(InfractionScheduler, commands.Cog): @@ -27,7 +32,7 @@ class Infractions(InfractionScheduler, commands.Cog): category_description = "Server moderation tools." def __init__(self, bot: Bot): - super().__init__(bot, supported_infractions={"ban", "kick", "mute", "note", "warning", "voice_ban"}) + super().__init__(bot, supported_infractions={"ban", "kick", "mute", "note", "warning", "voice_mute"}) self.category = "Moderation" self._muted_role = discord.Object(constants.Roles.muted) @@ -54,7 +59,7 @@ class Infractions(InfractionScheduler, commands.Cog): # region: Permanent infractions @command() - async def warn(self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None) -> None: + async def warn(self, ctx: Context, user: UnambiguousMemberOrUser, *, reason: t.Optional[str] = None) -> None: """Warn a user for the given reason.""" if not isinstance(user, Member): await ctx.send(":x: The user doesn't appear to be on the server.") @@ -67,7 +72,7 @@ class Infractions(InfractionScheduler, commands.Cog): await self.apply_infraction(ctx, infraction, user) @command() - async def kick(self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None) -> None: + async def kick(self, ctx: Context, user: UnambiguousMemberOrUser, *, reason: t.Optional[str] = None) -> None: """Kick a user for the given reason.""" if not isinstance(user, Member): await ctx.send(":x: The user doesn't appear to be on the server.") @@ -79,7 +84,7 @@ class Infractions(InfractionScheduler, commands.Cog): async def ban( self, ctx: Context, - user: FetchedMember, + user: UnambiguousMemberOrUser, duration: t.Optional[Expiry] = None, *, reason: t.Optional[str] = None @@ -91,37 +96,85 @@ class Infractions(InfractionScheduler, commands.Cog): """ await self.apply_ban(ctx, user, reason, expires_at=duration) - @command(aliases=('pban',)) - async def purgeban( + @command(aliases=("cban", "purgeban", "pban")) + async def cleanban( self, ctx: Context, - user: FetchedMember, + user: UnambiguousMemberOrUser, duration: t.Optional[Expiry] = None, *, reason: t.Optional[str] = None ) -> None: """ - Same as ban but removes all their messages of the last 24 hours. + Same as ban, but also cleans all their messages from the last hour. If duration is specified, it temporarily bans that user for the given duration. """ - await self.apply_ban(ctx, user, reason, 1, expires_at=duration) + clean_cog: t.Optional[Clean] = self.bot.get_cog("Clean") + if clean_cog is None: + # If we can't get the clean cog, fall back to native purgeban. + await self.apply_ban(ctx, user, reason, purge_days=1, expires_at=duration) + return + + infraction = await self.apply_ban(ctx, user, reason, expires_at=duration) + if not infraction or not infraction.get("id"): + # Ban was unsuccessful, quit early. + await ctx.send(":x: Failed to apply ban.") + log.error("Failed to apply ban to user %d", user.id) + return + + # Calling commands directly skips Discord.py's convertors, so we need to convert args manually. + clean_time = await Age().convert(ctx, "1h") + + log_url = await clean_cog._clean_messages( + ctx, + users=[user], + channels="*", + first_limit=clean_time, + attempt_delete_invocation=False, + ) + if not log_url: + # Cleaning failed, or there were no messages to clean, exit early. + return + + infr_manage_cog: t.Optional[ModManagement] = self.bot.get_cog("ModManagement") + if infr_manage_cog is None: + # If we can't get the mod management cog, don't bother appending the log. + return + + # Overwrite the context's send function so infraction append + # doesn't output the update infraction confirmation message. + async def send(*args, **kwargs) -> None: + pass + ctx.send = send + await infr_manage_cog.infraction_append(ctx, infraction, None, reason=f"[Clean log]({log_url})") + + @command(aliases=("vban",)) + async def voiceban(self, ctx: Context) -> None: + """ + NOT IMPLEMENTED. - @command(aliases=('vban',)) - async def voiceban( + Permanently ban a user from joining voice channels. + + If duration is specified, it temporarily voice bans that user for the given duration. + """ + await ctx.send(":x: This command is not yet implemented. Maybe you meant to use `voicemute`?") + + @command(aliases=("vmute",)) + async def voicemute( self, ctx: Context, - user: FetchedMember, + user: UnambiguousMemberOrUser, duration: t.Optional[Expiry] = None, *, reason: t.Optional[str] ) -> None: """ - Permanently ban user from using voice channels. + Permanently mute user in voice channels. - If duration is specified, it temporarily voice bans that user for the given duration. + If duration is specified, it temporarily voice mutes that user for the given duration. """ - await self.apply_voice_ban(ctx, user, reason, expires_at=duration) + await self.apply_voice_mute(ctx, user, reason, expires_at=duration) # endregion # region: Temporary infractions @@ -129,7 +182,7 @@ class Infractions(InfractionScheduler, commands.Cog): @command(aliases=["mute"]) async def tempmute( self, ctx: Context, - user: FetchedMember, + user: UnambiguousMemberOrUser, duration: t.Optional[Expiry] = None, *, reason: t.Optional[str] = None @@ -163,7 +216,7 @@ class Infractions(InfractionScheduler, commands.Cog): async def tempban( self, ctx: Context, - user: FetchedMember, + user: UnambiguousMemberOrUser, duration: Expiry, *, reason: t.Optional[str] = None @@ -186,16 +239,25 @@ class Infractions(InfractionScheduler, commands.Cog): await self.apply_ban(ctx, user, reason, expires_at=duration) @command(aliases=("tempvban", "tvban")) - async def tempvoiceban( - self, - ctx: Context, - user: FetchedMember, - duration: Expiry, - *, - reason: t.Optional[str] + async def tempvoiceban(self, ctx: Context) -> None: + """ + NOT IMPLEMENTED. + + Temporarily voice bans that user for the given duration. + """ + await ctx.send(":x: This command is not yet implemented. Maybe you meant to use `tempvoicemute`?") + + @command(aliases=("tempvmute", "tvmute")) + async def tempvoicemute( + self, + ctx: Context, + user: UnambiguousMemberOrUser, + duration: Expiry, + *, + reason: t.Optional[str] ) -> None: """ - Temporarily voice ban a user for the given reason and duration. + Temporarily voice mute a user for the given reason and duration. A unit of time should be appended to the duration. Units (∗case-sensitive): @@ -209,13 +271,13 @@ class Infractions(InfractionScheduler, commands.Cog): Alternatively, an ISO 8601 timestamp can be provided for the duration. """ - await self.apply_voice_ban(ctx, user, reason, expires_at=duration) + await self.apply_voice_mute(ctx, user, reason, expires_at=duration) # endregion # region: Permanent shadow infractions @command(hidden=True) - async def note(self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None) -> None: + async def note(self, ctx: Context, user: UnambiguousMemberOrUser, *, reason: t.Optional[str] = None) -> None: """Create a private note for a user with the given reason without notifying the user.""" infraction = await _utils.post_infraction(ctx, user, "note", reason, hidden=True, active=False) if infraction is None: @@ -224,7 +286,7 @@ class Infractions(InfractionScheduler, commands.Cog): await self.apply_infraction(ctx, infraction, user) @command(hidden=True, aliases=['shadowban', 'sban']) - async def shadow_ban(self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None) -> None: + async def shadow_ban(self, ctx: Context, user: UnambiguousMemberOrUser, *, reason: t.Optional[str] = None) -> None: """Permanently ban a user for the given reason without notifying the user.""" await self.apply_ban(ctx, user, reason, hidden=True) @@ -235,7 +297,7 @@ class Infractions(InfractionScheduler, commands.Cog): async def shadow_tempban( self, ctx: Context, - user: FetchedMember, + user: UnambiguousMemberOrUser, duration: Expiry, *, reason: t.Optional[str] = None @@ -261,27 +323,47 @@ class Infractions(InfractionScheduler, commands.Cog): # region: Remove infractions (un- commands) @command() - async def unmute(self, ctx: Context, user: FetchedMember) -> None: + async def unmute(self, ctx: Context, user: UnambiguousMemberOrUser) -> None: """Prematurely end the active mute infraction for the user.""" await self.pardon_infraction(ctx, "mute", user) @command() - async def unban(self, ctx: Context, user: FetchedMember) -> None: + async def unban(self, ctx: Context, user: UnambiguousMemberOrUser) -> None: """Prematurely end the active ban infraction for the user.""" await self.pardon_infraction(ctx, "ban", user) @command(aliases=("uvban",)) - async def unvoiceban(self, ctx: Context, user: FetchedMember) -> None: - """Prematurely end the active voice ban infraction for the user.""" - await self.pardon_infraction(ctx, "voice_ban", user) + async def unvoiceban(self, ctx: Context) -> None: + """ + NOT IMPLEMENTED. + + Temporarily voice bans that user for the given duration. + """ + await ctx.send(":x: This command is not yet implemented. Maybe you meant to use `unvoicemute`?") + + @command(aliases=("uvmute",)) + async def unvoicemute(self, ctx: Context, user: UnambiguousMemberOrUser) -> None: + """Prematurely end the active voice mute infraction for the user.""" + await self.pardon_infraction(ctx, "voice_mute", user) # endregion # region: Base apply functions async def apply_mute(self, ctx: Context, user: Member, reason: t.Optional[str], **kwargs) -> None: """Apply a mute infraction with kwargs passed to `post_infraction`.""" - if await _utils.get_active_infraction(ctx, user, "mute"): - return + if active := await _utils.get_active_infraction(ctx, user, "mute", send_msg=False): + if active["actor"] != self.bot.user.id: + await _utils.send_active_infraction_message(ctx, active) + return + + # Allow the current mute attempt to override an automatically triggered mute. + log_text = await self.deactivate_infraction(active, notify=False) + if "Failure" in log_text: + await ctx.send( + f":x: can't override infraction **mute** for {user.mention}: " + f"failed to deactivate. {log_text['Failure']}" + ) + return infraction = await _utils.post_infraction(ctx, user, "mute", reason, active=True, **kwargs) if infraction is None: @@ -304,6 +386,10 @@ class Infractions(InfractionScheduler, commands.Cog): @respect_role_hierarchy(member_arg=2) async def apply_kick(self, ctx: Context, user: Member, reason: t.Optional[str], **kwargs) -> None: """Apply a kick infraction with kwargs passed to `post_infraction`.""" + if user.top_role >= ctx.me.top_role: + await ctx.send(":x: I can't kick users above or equal to me in the role hierarchy.") + return + infraction = await _utils.post_infraction(ctx, user, "kick", reason, active=False, **kwargs) if infraction is None: return @@ -320,16 +406,20 @@ class Infractions(InfractionScheduler, commands.Cog): async def apply_ban( self, ctx: Context, - user: UserSnowflake, + user: MemberOrUser, reason: t.Optional[str], purge_days: t.Optional[int] = 0, **kwargs - ) -> None: + ) -> t.Optional[dict]: """ Apply a ban infraction with kwargs passed to `post_infraction`. Will also remove the banned user from the Big Brother watch list if applicable. """ + if isinstance(user, Member) and user.top_role >= ctx.me.top_role: + await ctx.send(":x: I can't ban users above or equal to me in the role hierarchy.") + return None + # In the case of a permanent ban, we don't need get_active_infractions to tell us if one is active is_temporary = kwargs.get("expires_at") is not None active_infraction = await _utils.get_active_infraction(ctx, user, "ban", is_temporary) @@ -337,19 +427,19 @@ class Infractions(InfractionScheduler, commands.Cog): if active_infraction: if is_temporary: log.trace("Tempban ignored as it cannot overwrite an active ban.") - return + return None if active_infraction.get('expires_at') is None: log.trace("Permaban already exists, notify.") await ctx.send(f":x: User is already permanently banned (#{active_infraction['id']}).") - return + return None log.trace("Old tempban is being replaced by new permaban.") - await self.pardon_infraction(ctx, "ban", user, is_temporary) + await self.pardon_infraction(ctx, "ban", user, send_msg=is_temporary) infraction = await _utils.post_infraction(ctx, user, "ban", reason, active=True, **kwargs) if infraction is None: - return + return None infraction["purge"] = "purge " if purge_days else "" @@ -361,27 +451,25 @@ class Infractions(InfractionScheduler, commands.Cog): action = ctx.guild.ban(user, reason=reason, delete_message_days=purge_days) await self.apply_infraction(ctx, infraction, user, action) + bb_cog: t.Optional[BigBrother] = self.bot.get_cog("Big Brother") if infraction.get('expires_at') is not None: log.trace(f"Ban isn't permanent; user {user} won't be unwatched by Big Brother.") - return - - bb_cog = self.bot.get_cog("Big Brother") - if not bb_cog: + elif not bb_cog: log.error(f"Big Brother cog not loaded; perma-banned user {user} won't be unwatched.") - return - - log.trace(f"Big Brother cog loaded; attempting to unwatch perma-banned user {user}.") + else: + log.trace(f"Big Brother cog loaded; attempting to unwatch perma-banned user {user}.") + bb_reason = "User has been permanently banned from the server. Automatically removed." + await bb_cog.apply_unwatch(ctx, user, bb_reason, send_message=False) - bb_reason = "User has been permanently banned from the server. Automatically removed." - await bb_cog.apply_unwatch(ctx, user, bb_reason, send_message=False) + return infraction @respect_role_hierarchy(member_arg=2) - async def apply_voice_ban(self, ctx: Context, user: UserSnowflake, reason: t.Optional[str], **kwargs) -> None: - """Apply a voice ban infraction with kwargs passed to `post_infraction`.""" - if await _utils.get_active_infraction(ctx, user, "voice_ban"): + async def apply_voice_mute(self, ctx: Context, user: MemberOrUser, reason: t.Optional[str], **kwargs) -> None: + """Apply a voice mute infraction with kwargs passed to `post_infraction`.""" + if await _utils.get_active_infraction(ctx, user, "voice_mute"): return - infraction = await _utils.post_infraction(ctx, user, "voice_ban", reason, active=True, **kwargs) + infraction = await _utils.post_infraction(ctx, user, "voice_mute", reason, active=True, **kwargs) if infraction is None: return @@ -395,7 +483,7 @@ class Infractions(InfractionScheduler, commands.Cog): if not isinstance(user, Member): return - await user.move_to(None, reason="Disconnected from voice to apply voiceban.") + await user.move_to(None, reason="Disconnected from voice to apply voice mute.") await user.remove_roles(self._voice_verified_role, reason=reason) await self.apply_infraction(ctx, infraction, user, action()) @@ -403,9 +491,16 @@ class Infractions(InfractionScheduler, commands.Cog): # endregion # region: Base pardon functions - async def pardon_mute(self, user_id: int, guild: discord.Guild, reason: t.Optional[str]) -> t.Dict[str, str]: - """Remove a user's muted role, DM them a notification, and return a log dict.""" - user = guild.get_member(user_id) + async def pardon_mute( + self, + user_id: int, + guild: discord.Guild, + reason: t.Optional[str], + *, + notify: bool = True + ) -> t.Dict[str, str]: + """Remove a user's muted role, optionally DM them a notification, and return a log dict.""" + user = await get_or_fetch_member(guild, user_id) log_text = {} if user: @@ -413,16 +508,17 @@ class Infractions(InfractionScheduler, commands.Cog): self.mod_log.ignore(Event.member_update, user.id) await user.remove_roles(self._muted_role, reason=reason) - # DM the user about the expiration. - notified = await _utils.notify_pardon( - user=user, - title="You have been unmuted", - content="You may now send messages in the server.", - icon_url=_utils.INFRACTION_ICONS["mute"][1] - ) + if notify: + # DM the user about the expiration. + notified = await _utils.notify_pardon( + user=user, + title="You have been unmuted", + content="You may now send messages in the server.", + icon_url=_utils.INFRACTION_ICONS["mute"][1] + ) + log_text["DM"] = "Sent" if notified else "**Failed**" log_text["Member"] = format_user(user) - log_text["DM"] = "Sent" if notified else "**Failed**" else: log.info(f"Failed to unmute user {user_id}: user not found") log_text["Failure"] = "User was not found in the guild." @@ -444,31 +540,39 @@ class Infractions(InfractionScheduler, commands.Cog): return log_text - async def pardon_voice_ban(self, user_id: int, guild: discord.Guild, reason: t.Optional[str]) -> t.Dict[str, str]: - """Add Voice Verified role back to user, DM them a notification, and return a log dict.""" - user = guild.get_member(user_id) + async def pardon_voice_mute( + self, + user_id: int, + guild: discord.Guild, + *, + notify: bool = True + ) -> t.Dict[str, str]: + """Optionally DM the user a pardon notification and return a log dict.""" + user = await get_or_fetch_member(guild, user_id) log_text = {} if user: - # DM user about infraction expiration - notified = await _utils.notify_pardon( - user=user, - title="Voice ban ended", - content="You have been unbanned and can verify yourself again in the server.", - icon_url=_utils.INFRACTION_ICONS["voice_ban"][1] - ) + if notify: + # DM user about infraction expiration + notified = await _utils.notify_pardon( + user=user, + title="Voice mute ended", + content="You have been unmuted and can verify yourself again in the server.", + icon_url=_utils.INFRACTION_ICONS["voice_mute"][1] + ) + log_text["DM"] = "Sent" if notified else "**Failed**" log_text["Member"] = format_user(user) - log_text["DM"] = "Sent" if notified else "**Failed**" else: log_text["Info"] = "User was not found in the guild." return log_text - async def _pardon_action(self, infraction: _utils.Infraction) -> t.Optional[t.Dict[str, str]]: + async def _pardon_action(self, infraction: _utils.Infraction, notify: bool) -> t.Optional[t.Dict[str, str]]: """ Execute deactivation steps specific to the infraction's type and return a log dict. + If `notify` is True, notify the user of the pardon via DM where applicable. If an infraction type is unsupported, return None instead. """ guild = self.bot.get_guild(constants.Guild.id) @@ -476,11 +580,11 @@ class Infractions(InfractionScheduler, commands.Cog): reason = f"Infraction #{infraction['id']} expired or was pardoned." if infraction["type"] == "mute": - return await self.pardon_mute(user_id, guild, reason) + return await self.pardon_mute(user_id, guild, reason, notify=notify) elif infraction["type"] == "ban": return await self.pardon_ban(user_id, guild, reason) - elif infraction["type"] == "voice_ban": - return await self.pardon_voice_ban(user_id, guild, reason) + elif infraction["type"] == "voice_mute": + return await self.pardon_voice_mute(user_id, guild, notify=notify) # endregion @@ -493,7 +597,7 @@ class Infractions(InfractionScheduler, commands.Cog): async def cog_command_error(self, ctx: Context, error: Exception) -> None: """Send a notification to the invoking context on a Union failure.""" if isinstance(error, commands.BadUnionArgument): - if discord.User in error.converters or discord.Member in error.converters: + if discord.User in error.converters or Member in error.converters: await ctx.send(str(error.errors[0])) error.handled = True diff --git a/bot/exts/moderation/infraction/management.py b/bot/exts/moderation/infraction/management.py index 08d7e0b6d..c813d1fdc 100644 --- a/bot/exts/moderation/infraction/management.py +++ b/bot/exts/moderation/infraction/management.py @@ -1,7 +1,5 @@ -import logging import textwrap import typing as t -from datetime import datetime import discord from discord.ext import commands @@ -10,15 +8,18 @@ from discord.utils import escape_markdown from bot import constants from bot.bot import Bot -from bot.converters import Expiry, Infraction, Snowflake, UserMention, allowed_strings, proxy_user +from bot.converters import Expiry, Infraction, MemberOrUser, Snowflake, UnambiguousUser, allowed_strings +from bot.errors import InvalidInfraction from bot.exts.moderation.infraction import _utils from bot.exts.moderation.infraction.infractions import Infractions from bot.exts.moderation.modlog import ModLog +from bot.log import get_logger from bot.pagination import LinePaginator from bot.utils import messages, time from bot.utils.channel import is_mod_channel +from bot.utils.members import get_or_fetch_member -log = logging.getLogger(__name__) +log = get_logger(__name__) class ModManagement(commands.Cog): @@ -40,9 +41,22 @@ class ModManagement(commands.Cog): return self.bot.get_cog("Infractions") @commands.group(name='infraction', aliases=('infr', 'infractions', 'inf', 'i'), invoke_without_command=True) - async def infraction_group(self, ctx: Context) -> None: - """Infraction management commands.""" - await ctx.send_help(ctx.command) + async def infraction_group(self, ctx: Context, infraction: Infraction = None) -> None: + """ + Infraction management commands. + + If `infraction` is passed then this command fetches that infraction. The `Infraction` converter + supports 'l', 'last' and 'recent' to get the most recent infraction made by `ctx.author`. + """ + if infraction is None: + await ctx.send_help(ctx.command) + return + + embed = discord.Embed( + title=f"Infraction #{infraction['id']}", + colour=discord.Colour.orange() + ) + await self.send_infraction_list(ctx, embed, [infraction]) @infraction_group.command(name="resend", aliases=("send", "rs", "dm")) async def infraction_resend(self, ctx: Context, infraction: Infraction) -> None: @@ -51,11 +65,16 @@ class ModManagement(commands.Cog): await ctx.send(f"{constants.Emojis.failmail} You may not resend hidden infractions.") return + member = await get_or_fetch_member(ctx.guild, infraction["user"]) + if not member: + await ctx.send(f"{constants.Emojis.failmail} Cannot find member `{infraction['user']}`.") + return + id_ = infraction["id"] reason = infraction["reason"] or "No reason provided." reason += "\n\n**This is a re-sent message for a previously applied infraction which may have been edited.**" - if await _utils.notify_infraction(infraction, reason=reason): + if await _utils.notify_infraction(infraction, member, reason): await ctx.send(f":incoming_envelope: Resent DM for infraction `{id_}`.") else: await ctx.send(f"{constants.Emojis.failmail} Failed to resend DM for infraction `{id_}`.") @@ -95,7 +114,7 @@ class ModManagement(commands.Cog): """ old_reason = infraction["reason"] - if old_reason is not None: + if old_reason is not None and reason is not None: add_period = not old_reason.endswith((".", "!", "?")) reason = old_reason + (". " if add_period else " ") + reason @@ -140,16 +159,17 @@ class ModManagement(commands.Cog): log_text = "" if duration is not None and not infraction['active']: - if reason is None: + if (infr_type := infraction['type']) in ('note', 'warning'): + await ctx.send(f":x: Cannot edit the expiration of a {infr_type}.") + else: await ctx.send(":x: Cannot edit the expiration of an expired infraction.") - return - confirm_messages.append("expiry unchanged (infraction already expired)") + return elif isinstance(duration, str): request_data['expires_at'] = None confirm_messages.append("marked as permanent") elif duration is not None: request_data['expires_at'] = duration.isoformat() - expiry = time.format_infraction_with_duration(request_data['expires_at']) + expiry = time.format_with_duration(duration) confirm_messages.append(f"set to expire on {expiry}") else: confirm_messages.append("expiry unchanged") @@ -174,15 +194,15 @@ class ModManagement(commands.Cog): if 'expires_at' in request_data: # A scheduled task should only exist if the old infraction wasn't permanent if infraction['expires_at']: - self.infractions_cog.scheduler.cancel(new_infraction['id']) + self.infractions_cog.scheduler.cancel(infraction_id) # If the infraction was not marked as permanent, schedule a new expiration task if request_data['expires_at']: self.infractions_cog.schedule_expiration(new_infraction) log_text += f""" - Previous expiry: {infraction['expires_at'] or "Permanent"} - New expiry: {new_infraction['expires_at'] or "Permanent"} + Previous expiry: {time.until_expiration(infraction['expires_at'])} + New expiry: {time.until_expiration(new_infraction['expires_at'])} """.rstrip() changes = ' & '.join(confirm_messages) @@ -190,54 +210,61 @@ class ModManagement(commands.Cog): # Get information about the infraction's user user_id = new_infraction['user'] - user = ctx.guild.get_member(user_id) + user = await get_or_fetch_member(ctx.guild, user_id) if user: user_text = messages.format_user(user) - thumbnail = user.avatar_url_as(static_format="png") + thumbnail = user.display_avatar.url else: user_text = f"<@{user_id}>" thumbnail = None await self.mod_log.send_log_message( icon_url=constants.Icons.pencil, - colour=discord.Colour.blurple(), + colour=discord.Colour.og_blurple(), title="Infraction edited", thumbnail=thumbnail, text=textwrap.dedent(f""" Member: {user_text} Actor: <@{new_infraction['actor']}> Edited by: {ctx.message.author.mention}{log_text} - """) + """), + footer=f"ID: {infraction_id}" ) # endregion # region: Search infractions @infraction_group.group(name="search", aliases=('s',), invoke_without_command=True) - async def infraction_search_group(self, ctx: Context, query: t.Union[UserMention, Snowflake, str]) -> None: + async def infraction_search_group(self, ctx: Context, query: t.Union[UnambiguousUser, Snowflake, str]) -> None: """Searches for infractions in the database.""" if isinstance(query, int): await self.search_user(ctx, discord.Object(query)) - else: + elif isinstance(query, str): await self.search_reason(ctx, query) + else: + await self.search_user(ctx, query) - @infraction_search_group.command(name="user", aliases=("member", "id")) - async def search_user(self, ctx: Context, user: t.Union[discord.User, proxy_user]) -> None: + @infraction_search_group.command(name="user", aliases=("member", "userid")) + async def search_user(self, ctx: Context, user: t.Union[MemberOrUser, discord.Object]) -> None: """Search for infractions by member.""" infraction_list = await self.bot.api_client.get( 'bot/infractions/expanded', params={'user__id': str(user.id)} ) - user = self.bot.get_user(user.id) - if not user and infraction_list: - # Use the user data retrieved from the DB for the username. - user = infraction_list[0]["user"] - user = escape_markdown(user["name"]) + f"#{user['discriminator']:04}" + if isinstance(user, (discord.Member, discord.User)): + user_str = escape_markdown(str(user)) + else: + if infraction_list: + user = infraction_list[0]["user"] + user_str = escape_markdown(user["name"]) + f"#{user['discriminator']:04}" + else: + user_str = str(user.id) + formatted_infraction_count = self.format_infraction_count(len(infraction_list)) embed = discord.Embed( - title=f"Infractions for {user} ({len(infraction_list)} total)", + title=f"Infractions for {user_str} ({formatted_infraction_count} total)", colour=discord.Colour.orange() ) await self.send_infraction_list(ctx, embed, infraction_list) @@ -249,15 +276,70 @@ class ModManagement(commands.Cog): 'bot/infractions/expanded', params={'search': reason} ) + + formatted_infraction_count = self.format_infraction_count(len(infraction_list)) + embed = discord.Embed( + title=f"Infractions matching `{reason}` ({formatted_infraction_count} total)", + colour=discord.Colour.orange() + ) + await self.send_infraction_list(ctx, embed, infraction_list) + + # endregion + # region: Search for infractions by given actor + + @infraction_group.command(name="by", aliases=("b",)) + async def search_by_actor( + self, + ctx: Context, + actor: t.Union[t.Literal["m", "me"], UnambiguousUser], + oldest_first: bool = False + ) -> None: + """ + Search for infractions made by `actor`. + + Use "m" or "me" as the `actor` to get infractions by author. + + Use "1" for `oldest_first` to send oldest infractions first. + """ + if isinstance(actor, str): + actor = ctx.author + + if oldest_first: + ordering = 'inserted_at' # oldest infractions first + else: + ordering = '-inserted_at' # newest infractions first + + infraction_list = await self.bot.api_client.get( + 'bot/infractions/expanded', + params={ + 'actor__id': str(actor.id), + 'ordering': ordering + } + ) + + formatted_infraction_count = self.format_infraction_count(len(infraction_list)) embed = discord.Embed( - title=f"Infractions matching `{reason}` ({len(infraction_list)} total)", + title=f"Infractions by {actor} ({formatted_infraction_count} total)", colour=discord.Colour.orange() ) + await self.send_infraction_list(ctx, embed, infraction_list) # endregion # region: Utility functions + @staticmethod + def format_infraction_count(infraction_count: int) -> str: + """ + Returns a string-formatted infraction count. + + API limits returned infractions to a maximum of 100, so if `infraction_count` + is 100 then we return `"100+"`. Otherwise, return `str(infraction_count)`. + """ + if infraction_count == 100: + return "100+" + return str(infraction_count) + async def send_infraction_list( self, ctx: Context, @@ -288,7 +370,9 @@ class ModManagement(commands.Cog): active = infraction["active"] user = infraction["user"] expires_at = infraction["expires_at"] - created = time.format_infraction(infraction["inserted_at"]) + inserted_at = infraction["inserted_at"] + created = time.discord_timestamp(inserted_at) + dm_sent = infraction["dm_sent"] # Format the user string. if user_obj := self.bot.get_user(user["id"]): @@ -300,25 +384,31 @@ class ModManagement(commands.Cog): user_str = f"<@{user['id']}> ({name}#{user['discriminator']:04})" if active: - remaining = time.until_expiration(expires_at) or "Expired" + remaining = time.until_expiration(expires_at) else: remaining = "Inactive" if expires_at is None: - expires = "*Permanent*" + duration = "*Permanent*" else: - date_from = datetime.strptime(created, time.INFRACTION_FORMAT) - expires = time.format_infraction_with_duration(expires_at, date_from) + duration = time.humanize_delta(inserted_at, expires_at) + + # Format `dm_sent` + if dm_sent is None: + dm_sent_text = "N/A" + else: + dm_sent_text = "Yes" if dm_sent else "No" lines = textwrap.dedent(f""" {"**===============**" if active else "==============="} Status: {"__**Active**__" if active else "Inactive"} User: {user_str} Type: **{infraction["type"]}** + DM Sent: {dm_sent_text} Shadow: {infraction["hidden"]} Created: {created} - Expires: {expires} - Remaining: {remaining} + Expires: {remaining} + Duration: {duration} Actor: <@{infraction["actor"]["id"]}> ID: `{infraction["id"]}` Reason: {infraction["reason"] or "*None*"} @@ -339,13 +429,20 @@ class ModManagement(commands.Cog): return all(checks) # This cannot be static (must have a __func__ attribute). - async def cog_command_error(self, ctx: Context, error: Exception) -> None: - """Send a notification to the invoking context on a Union failure.""" + async def cog_command_error(self, ctx: Context, error: commands.CommandError) -> None: + """Handles errors for commands within this cog.""" if isinstance(error, commands.BadUnionArgument): if discord.User in error.converters: await ctx.send(str(error.errors[0])) error.handled = True + elif isinstance(error, InvalidInfraction): + if error.infraction_arg.isdigit(): + await ctx.send(f":x: Could not find an infraction with id `{error.infraction_arg}`.") + else: + await ctx.send(f":x: `{error.infraction_arg}` is not a valid integer infraction id.") + error.handled = True + def setup(bot: Bot) -> None: """Load the ModManagement cog.""" diff --git a/bot/exts/moderation/infraction/superstarify.py b/bot/exts/moderation/infraction/superstarify.py index 160c1ad19..b91a5edba 100644 --- a/bot/exts/moderation/infraction/superstarify.py +++ b/bot/exts/moderation/infraction/superstarify.py @@ -1,5 +1,4 @@ import json -import logging import random import textwrap import typing as t @@ -14,10 +13,12 @@ from bot.bot import Bot from bot.converters import Duration, Expiry from bot.exts.moderation.infraction import _utils from bot.exts.moderation.infraction._scheduler import InfractionScheduler +from bot.log import get_logger +from bot.utils import time +from bot.utils.members import get_or_fetch_member from bot.utils.messages import format_user -from bot.utils.time import format_infraction -log = logging.getLogger(__name__) +log = get_logger(__name__) NICKNAME_POLICY_URL = "https://pythondiscord.com/pages/rules/#nickname-policy" SUPERSTARIFY_DEFAULT_DURATION = "1h" @@ -56,7 +57,9 @@ class Superstarify(InfractionScheduler, Cog): return infraction = active_superstarifies[0] - forced_nick = self.get_nick(infraction["id"], before.id) + infr_id = infraction["id"] + + forced_nick = self.get_nick(infr_id, before.id) if after.display_name == forced_nick: return # Nick change was triggered by this event. Ignore. @@ -72,7 +75,7 @@ class Superstarify(InfractionScheduler, Cog): ) await after.edit( nick=forced_nick, - reason=f"Superstarified member tried to escape the prison: {infraction['id']}" + reason=f"Superstarified member tried to escape the prison: {infr_id}" ) if not await _utils.notify_infraction(infraction, after, reason): @@ -126,6 +129,10 @@ class Superstarify(InfractionScheduler, Cog): An optional reason can be provided, which would be added to a message stating their old nickname and linking to the nickname policy. """ + if member.top_role >= ctx.me.top_role: + await ctx.send(":x: I can't starify users above or equal to me in the role hierarchy.") + return + if await _utils.get_active_infraction(ctx, member, "superstar"): return @@ -139,7 +146,7 @@ class Superstarify(InfractionScheduler, Cog): id_ = infraction["id"] forced_nick = self.get_nick(id_, member.id) - expiry_str = format_infraction(infraction["expires_at"]) + expiry_str = time.discord_timestamp(infraction["expires_at"]) # Apply the infraction async def action() -> None: @@ -186,13 +193,13 @@ class Superstarify(InfractionScheduler, Cog): """Remove the superstarify infraction and allow the user to change their nickname.""" await self.pardon_infraction(ctx, "superstar", member) - async def _pardon_action(self, infraction: _utils.Infraction) -> t.Optional[t.Dict[str, str]]: - """Pardon a superstar infraction and return a log dict.""" + async def _pardon_action(self, infraction: _utils.Infraction, notify: bool) -> t.Optional[t.Dict[str, str]]: + """Pardon a superstar infraction, optionally notify the user via DM, and return a log dict.""" if infraction["type"] != "superstar": return guild = self.bot.get_guild(constants.Guild.id) - user = guild.get_member(infraction["user"]) + user = await get_or_fetch_member(guild, infraction["user"]) # Don't bother sending a notification if the user left the guild. if not user: @@ -202,18 +209,19 @@ class Superstarify(InfractionScheduler, Cog): ) return {} + log_text = {"Member": format_user(user)} + # DM the user about the expiration. - notified = await _utils.notify_pardon( - user=user, - title="You are no longer superstarified", - content="You may now change your nickname on the server.", - icon_url=_utils.INFRACTION_ICONS["superstar"][1] - ) + if notify: + notified = await _utils.notify_pardon( + user=user, + title="You are no longer superstarified", + content="You may now change your nickname on the server.", + icon_url=_utils.INFRACTION_ICONS["superstar"][1] + ) + log_text["DM"] = "Sent" if notified else "**Failed**" - return { - "Member": format_user(user), - "DM": "Sent" if notified else "**Failed**" - } + return log_text @staticmethod def get_nick(infraction_id: int, member_id: int) -> str: diff --git a/bot/exts/moderation/metabase.py b/bot/exts/moderation/metabase.py index db5f04d83..ce9c220b3 100644 --- a/bot/exts/moderation/metabase.py +++ b/bot/exts/moderation/metabase.py @@ -1,6 +1,5 @@ import csv import json -import logging from datetime import timedelta from io import StringIO from typing import Dict, List, Optional @@ -14,11 +13,12 @@ from discord.ext.commands import Cog, Context, group, has_any_role from bot.bot import Bot from bot.constants import Metabase as MetabaseConfig, Roles from bot.converters import allowed_strings -from bot.utils import send_to_paste_service +from bot.log import get_logger +from bot.utils import scheduling, send_to_paste_service from bot.utils.channel import is_mod_channel from bot.utils.scheduling import Scheduler -log = logging.getLogger(__name__) +log = get_logger(__name__) BASE_HEADERS = { "Content-Type": "application/json" @@ -40,7 +40,26 @@ class Metabase(Cog): self.exports: Dict[int, List[Dict]] = {} # Saves the output of each question, so internal eval can access it - self.init_task = self.bot.loop.create_task(self.init_cog()) + self.init_task = scheduling.create_task(self.init_cog(), event_loop=self.bot.loop) + + async def cog_command_error(self, ctx: Context, error: Exception) -> None: + """Handle ClientResponseError errors locally to invalidate token if needed.""" + if not isinstance(error.original, ClientResponseError): + return + + if error.original.status == 403: + # User doesn't have access to the given question + log.warning(f"Failed to auth with Metabase for {error.original.url}.") + await ctx.send(f":x: {ctx.author.mention} Failed to auth with Metabase for that question.") + elif error.original.status == 404: + await ctx.send(f":x: {ctx.author.mention} That question could not be found.") + else: + # User credentials are invalid, or the refresh failed. + # Delete the expiry time, to force a refresh on next startup. + await self.session_info.delete("session_expiry") + log.exception("Session token is invalid or refresh failed.") + await ctx.send(f":x: {ctx.author.mention} Session token is invalid or refresh failed.") + error.handled = True async def init_cog(self) -> None: """Initialise the metabase session.""" @@ -65,7 +84,7 @@ class Metabase(Cog): "username": MetabaseConfig.username, "password": MetabaseConfig.password } - async with self.bot.http_session.post(f"{MetabaseConfig.url}/session", json=data) as resp: + async with self.bot.http_session.post(f"{MetabaseConfig.base_url}/api/session", json=data) as resp: json_data = await resp.json() self.session_token = json_data.get("id") @@ -86,7 +105,7 @@ class Metabase(Cog): """A group of commands for interacting with metabase.""" await ctx.send_help(ctx.command) - @metabase_group.command(name="extract") + @metabase_group.command(name="extract", aliases=("export",)) async def metabase_extract( self, ctx: Context, @@ -106,48 +125,50 @@ class Metabase(Cog): Valid extensions are: csv and json. """ - async with ctx.typing(): - - # Make sure we have a session token before running anything - await self.init_task - - url = f"{MetabaseConfig.url}/card/{question_id}/query/{extension}" - try: - async with self.bot.http_session.post(url, headers=self.headers, raise_for_status=True) as resp: - if extension == "csv": - out = await resp.text() - # Save the output for use with int e - self.exports[question_id] = list(csv.DictReader(StringIO(out))) - - elif extension == "json": - out = await resp.json() - # Save the output for use with int e - self.exports[question_id] = out - - # Format it nicely for human eyes - out = json.dumps(out, indent=4, sort_keys=True) - except ClientResponseError as e: - if e.status == 403: - # User doesn't have access to the given question - log.warning(f"Failed to auth with Metabase for question {question_id}.") - await ctx.send(f":x: {ctx.author.mention} Failed to auth with Metabase for that question.") - else: - # User credentials are invalid, or the refresh failed. - # Delete the expiry time, to force a refresh on next startup. - await self.session_info.delete("session_expiry") - log.exception("Session token is invalid or refresh failed.") - await ctx.send(f":x: {ctx.author.mention} Session token is invalid or refresh failed.") - return - - paste_link = await send_to_paste_service(out, extension=extension) - if paste_link: - message = f":+1: {ctx.author.mention} Here's your link: {paste_link}" - else: - message = f":x: {ctx.author.mention} Link service is unavailible." - await ctx.send( - f"{message}\nYou can also access this data within internal eval by doing: " - f"`bot.get_cog('Metabase').exports[{question_id}]`" - ) + await ctx.trigger_typing() + + # Make sure we have a session token before running anything + await self.init_task + + url = f"{MetabaseConfig.base_url}/api/card/{question_id}/query/{extension}" + + async with self.bot.http_session.post(url, headers=self.headers, raise_for_status=True) as resp: + if extension == "csv": + out = await resp.text(encoding="utf-8") + # Save the output for use with int e + self.exports[question_id] = list(csv.DictReader(StringIO(out))) + + elif extension == "json": + out = await resp.json(encoding="utf-8") + # Save the output for use with int e + self.exports[question_id] = out + + # Format it nicely for human eyes + out = json.dumps(out, indent=4, sort_keys=True) + + paste_link = await send_to_paste_service(out, extension=extension) + if paste_link: + message = f":+1: {ctx.author.mention} Here's your link: {paste_link}" + else: + message = f":x: {ctx.author.mention} Link service is unavailible." + await ctx.send( + f"{message}\nYou can also access this data within internal eval by doing: " + f"`bot.get_cog('Metabase').exports[{question_id}]`" + ) + + @metabase_group.command(name="publish", aliases=("share",)) + async def metabase_publish(self, ctx: Context, question_id: int) -> None: + """Publically shares the given question and posts the link.""" + await ctx.trigger_typing() + # Make sure we have a session token before running anything + await self.init_task + + url = f"{MetabaseConfig.base_url}/api/card/{question_id}/public_link" + + async with self.bot.http_session.post(url, headers=self.headers, raise_for_status=True) as resp: + response_json = await resp.json(encoding="utf-8") + sharing_url = f"{MetabaseConfig.public_url}/public/question/{response_json['uuid']}" + await ctx.send(f":+1: {ctx.author.mention} Here's your sharing link: {sharing_url}") # This cannot be static (must have a __func__ attribute). async def cog_check(self, ctx: Context) -> bool: diff --git a/bot/exts/moderation/modlog.py b/bot/exts/moderation/modlog.py index be65ade6e..32ea0dc6a 100644 --- a/bot/exts/moderation/modlog.py +++ b/bot/exts/moderation/modlog.py @@ -1,25 +1,25 @@ import asyncio import difflib import itertools -import logging import typing as t -from datetime import datetime +from datetime import datetime, timezone from itertools import zip_longest import discord from dateutil.relativedelta import relativedelta from deepdiff import DeepDiff -from discord import Colour +from discord import Colour, Message, Thread from discord.abc import GuildChannel from discord.ext.commands import Cog, Context from discord.utils import escape_markdown from bot.bot import Bot from bot.constants import Categories, Channels, Colours, Emojis, Event, Guild as GuildConstant, Icons, Roles, URLs +from bot.log import get_logger +from bot.utils import time from bot.utils.messages import format_user -from bot.utils.time import humanize_delta -log = logging.getLogger(__name__) +log = get_logger(__name__) GUILD_CHANNEL = t.Union[discord.CategoryChannel, discord.TextChannel, discord.VoiceChannel] @@ -41,7 +41,6 @@ class ModLog(Cog, name="ModLog"): self.bot = bot self._ignored = {event: [] for event in Event} - self._cached_deletes = [] self._cached_edits = [] async def upload_log( @@ -58,7 +57,7 @@ class ModLog(Cog, name="ModLog"): 'bot/deleted-messages', json={ 'actor': actor_id, - 'creation': datetime.utcnow().isoformat(), + 'creation': datetime.now(timezone.utc).isoformat(), 'deletedmessage_set': [ { 'id': message.id, @@ -97,9 +96,10 @@ class ModLog(Cog, name="ModLog"): footer: t.Optional[str] = None, ) -> Context: """Generate log embed and send to logging channel.""" + await self.bot.wait_until_guild_available() # Truncate string directly here to avoid removing newlines embed = discord.Embed( - description=text[:2045] + "..." if len(text) > 2048 else text + description=text[:4093] + "..." if len(text) > 4096 else text ) if title and icon_url: @@ -116,7 +116,7 @@ class ModLog(Cog, name="ModLog"): if ping_everyone: if content: - content = f"<@&{Roles.moderators}>\n{content}" + content = f"<@&{Roles.moderators}> {content}" else: content = f"<@&{Roles.moderators}>" @@ -251,7 +251,7 @@ class ModLog(Cog, name="ModLog"): message = f"**#{after.name}** (`{after.id}`)\n{message}" await self.send_log_message( - Icons.hash_blurple, Colour.blurple(), + Icons.hash_blurple, Colour.og_blurple(), "Channel updated", message ) @@ -326,7 +326,7 @@ class ModLog(Cog, name="ModLog"): message = f"**{after.name}** (`{after.id}`)\n{message}" await self.send_log_message( - Icons.crown_blurple, Colour.blurple(), + Icons.crown_blurple, Colour.og_blurple(), "Role updated", message ) @@ -376,9 +376,9 @@ class ModLog(Cog, name="ModLog"): message = f"**{after.name}** (`{after.id}`)\n{message}" await self.send_log_message( - Icons.guild_update, Colour.blurple(), + Icons.guild_update, Colour.og_blurple(), "Guild updated", message, - thumbnail=after.icon_url_as(format="png") + thumbnail=after.icon.with_static_format("png") ) @Cog.listener() @@ -394,7 +394,7 @@ class ModLog(Cog, name="ModLog"): await self.send_log_message( Icons.user_ban, Colours.soft_red, "User banned", format_user(member), - thumbnail=member.avatar_url_as(static_format="png"), + thumbnail=member.display_avatar.url, channel_id=Channels.user_log ) @@ -404,10 +404,10 @@ class ModLog(Cog, name="ModLog"): if member.guild.id != GuildConstant.id: return - now = datetime.utcnow() + now = datetime.now(timezone.utc) difference = abs(relativedelta(now, member.created_at)) - message = format_user(member) + "\n\n**Account age:** " + humanize_delta(difference) + message = format_user(member) + "\n\n**Account age:** " + time.humanize_delta(difference) if difference.days < 1 and difference.months < 1 and difference.years < 1: # New user account! message = f"{Emojis.new} {message}" @@ -415,7 +415,7 @@ class ModLog(Cog, name="ModLog"): await self.send_log_message( Icons.sign_in, Colours.soft_green, "User joined", message, - thumbnail=member.avatar_url_as(static_format="png"), + thumbnail=member.display_avatar.url, channel_id=Channels.user_log ) @@ -432,7 +432,7 @@ class ModLog(Cog, name="ModLog"): await self.send_log_message( Icons.sign_out, Colours.soft_red, "User left", format_user(member), - thumbnail=member.avatar_url_as(static_format="png"), + thumbnail=member.display_avatar.url, channel_id=Channels.user_log ) @@ -447,9 +447,9 @@ class ModLog(Cog, name="ModLog"): return await self.send_log_message( - Icons.user_unban, Colour.blurple(), + Icons.user_unban, Colour.og_blurple(), "User unbanned", format_user(member), - thumbnail=member.avatar_url_as(static_format="png"), + thumbnail=member.display_avatar.url, channel_id=Channels.mod_log ) @@ -512,35 +512,62 @@ class ModLog(Cog, name="ModLog"): await self.send_log_message( icon_url=Icons.user_update, - colour=Colour.blurple(), + colour=Colour.og_blurple(), title="Member updated", text=message, - thumbnail=after.avatar_url_as(static_format="png"), + thumbnail=after.display_avatar.url, channel_id=Channels.user_log ) - @Cog.listener() - async def on_message_delete(self, message: discord.Message) -> None: - """Log message delete event to message change log.""" + def is_message_blacklisted(self, message: Message) -> bool: + """Return true if the message is in a blacklisted thread or channel.""" + # Ignore bots or DMs + if message.author.bot or not message.guild: + return True + + return self.is_channel_ignored(message.channel.id) + + def is_channel_ignored(self, channel_id: int) -> bool: + """ + Return true if the channel, or parent channel in the case of threads, passed should be ignored by modlog. + + Currently ignored channels are: + 1. Channels not in the guild we care about (constants.Guild.id). + 2. Channels that mods do not have view permissions to + 3. Channels in constants.Guild.modlog_blacklist + """ + channel = self.bot.get_channel(channel_id) + + # Ignore not found channels, DMs, and messages outside of the main guild. + if not channel or not hasattr(channel, "guild") or channel.guild.id != GuildConstant.id: + return True + + # Look at the parent channel of a thread. + if isinstance(channel, Thread): + channel = channel.parent + + # Mod team doesn't have view permission to the channel. + if not channel.permissions_for(channel.guild.get_role(Roles.mod_team)).view_channel: + return True + + return channel.id in GuildConstant.modlog_blacklist + + async def log_cached_deleted_message(self, message: discord.Message) -> None: + """ + Log the message's details to message change log. + + This is called when a cached message is deleted. + """ channel = message.channel author = message.author - # Ignore DMs. - if not message.guild: + if self.is_message_blacklisted(message): return - if message.guild.id != GuildConstant.id or channel.id in GuildConstant.modlog_blacklist: - return - - self._cached_deletes.append(message.id) - if message.id in self._ignored[Event.message_delete]: self._ignored[Event.message_delete].remove(message.id) return - if author.bot: - return - if channel.category: response = ( f"**Author:** {format_user(author)}\n" @@ -564,7 +591,7 @@ class ModLog(Cog, name="ModLog"): # Shorten the message content if necessary content = message.clean_content - remaining_chars = 2040 - len(response) + remaining_chars = 4090 - len(response) if len(content) > remaining_chars: botlog_url = await self.upload_log(messages=[message], actor_id=message.author.id) @@ -581,17 +608,15 @@ class ModLog(Cog, name="ModLog"): channel_id=Channels.message_log ) - @Cog.listener() - async def on_raw_message_delete(self, event: discord.RawMessageDeleteEvent) -> None: - """Log raw message delete event to message change log.""" - if event.guild_id != GuildConstant.id or event.channel_id in GuildConstant.modlog_blacklist: - return + async def log_uncached_deleted_message(self, event: discord.RawMessageDeleteEvent) -> None: + """ + Log the message's details to message change log. - await asyncio.sleep(1) # Wait here in case the normal event was fired - - if event.message_id in self._cached_deletes: - # It was in the cache and the normal event was fired, so we can just ignore it - self._cached_deletes.remove(event.message_id) + This is called when a message absent from the cache is deleted. + Hence, the message contents aren't logged. + """ + await self.bot.wait_until_guild_available() + if self.is_channel_ignored(event.channel_id): return if event.message_id in self._ignored[Event.message_delete]: @@ -623,14 +648,17 @@ class ModLog(Cog, name="ModLog"): ) @Cog.listener() + async def on_raw_message_delete(self, event: discord.RawMessageDeleteEvent) -> None: + """Log message deletions to message change log.""" + if event.cached_message is not None: + await self.log_cached_deleted_message(event.cached_message) + else: + await self.log_uncached_deleted_message(event) + + @Cog.listener() async def on_message_edit(self, msg_before: discord.Message, msg_after: discord.Message) -> None: """Log message edit event to message change log.""" - if ( - not msg_before.guild - or msg_before.guild.id != GuildConstant.id - or msg_before.channel.id in GuildConstant.modlog_blacklist - or msg_before.author.bot - ): + if self.is_message_blacklisted(msg_before): return self._cached_edits.append(msg_before.id) @@ -685,7 +713,7 @@ class ModLog(Cog, name="ModLog"): # datetime as the baseline and create a human-readable delta between this edit event # and the last time the message was edited timestamp = msg_before.edited_at - delta = humanize_delta(relativedelta(msg_after.edited_at, msg_before.edited_at)) + delta = time.humanize_delta(msg_after.edited_at, msg_before.edited_at) footer = f"Last edited {delta} ago" else: # Message was not previously edited, use the created_at datetime as the baseline, no @@ -694,25 +722,24 @@ class ModLog(Cog, name="ModLog"): footer = None await self.send_log_message( - Icons.message_edit, Colour.blurple(), "Message edited", response, + Icons.message_edit, Colour.og_blurple(), "Message edited", response, channel_id=Channels.message_log, timestamp_override=timestamp, footer=footer ) @Cog.listener() async def on_raw_message_edit(self, event: discord.RawMessageUpdateEvent) -> None: """Log raw message edit event to message change log.""" + if event.guild_id is None: + return # ignore DM edits + + await self.bot.wait_until_guild_available() try: channel = self.bot.get_channel(int(event.data["channel_id"])) message = await channel.fetch_message(event.message_id) except discord.NotFound: # Was deleted before we got the event return - if ( - not message.guild - or message.guild.id != GuildConstant.id - or message.channel.id in GuildConstant.modlog_blacklist - or message.author.bot - ): + if self.is_message_blacklisted(message): return await asyncio.sleep(1) # Wait here in case the normal event was fired @@ -742,16 +769,95 @@ class ModLog(Cog, name="ModLog"): ) await self.send_log_message( - Icons.message_edit, Colour.blurple(), "Message edited (Before)", + Icons.message_edit, Colour.og_blurple(), "Message edited (Before)", before_response, channel_id=Channels.message_log ) await self.send_log_message( - Icons.message_edit, Colour.blurple(), "Message edited (After)", + Icons.message_edit, Colour.og_blurple(), "Message edited (After)", after_response, channel_id=Channels.message_log ) @Cog.listener() + async def on_thread_update(self, before: Thread, after: Thread) -> None: + """Log thread archiving, un-archiving and name edits.""" + if self.is_channel_ignored(after.id): + log.trace("Ignoring update of thread %s (%d)", after.mention, after.id) + return + + if before.name != after.name: + await self.send_log_message( + Icons.hash_blurple, + Colour.og_blurple(), + "Thread name edited", + ( + f"Thread {after.mention} (`{after.id}`) from {after.parent.mention} (`{after.parent.id}`): " + f"`{before.name}` -> `{after.name}`" + ) + ) + return + + if not before.archived and after.archived: + colour = Colours.soft_red + action = "archived" + icon = Icons.hash_red + elif before.archived and not after.archived: + colour = Colours.soft_green + action = "un-archived" + icon = Icons.hash_green + else: + return + + await self.send_log_message( + icon, + colour, + f"Thread {action}", + ( + f"Thread {after.mention} ({after.name}, `{after.id}`) from {after.parent.mention} " + f"(`{after.parent.id}`) was {action}" + ) + ) + + @Cog.listener() + async def on_thread_delete(self, thread: Thread) -> None: + """Log thread deletion.""" + if self.is_channel_ignored(thread.id): + log.trace("Ignoring deletion of thread %s (%d)", thread.mention, thread.id) + return + + await self.send_log_message( + Icons.hash_red, + Colours.soft_red, + "Thread deleted", + ( + f"Thread {thread.mention} ({thread.name}, `{thread.id}`) from {thread.parent.mention} " + f"(`{thread.parent.id}`) deleted" + ) + ) + + @Cog.listener() + async def on_thread_join(self, thread: Thread) -> None: + """Log thread creation.""" + # If we are in the thread already we can most probably assume we already logged it? + # We don't really have a better way of doing this since the API doesn't make any difference between the two + if thread.me: + return + + if self.is_channel_ignored(thread.id): + log.trace("Ignoring creation of thread %s (%d)", thread.mention, thread.id) + return + + await self.send_log_message( + Icons.hash_green, + Colours.soft_green, + "Thread created", + ( + f"Thread {thread.mention} ({thread.name}, `{thread.id}`) from {thread.parent.mention} " + f"(`{thread.parent.id}`) created" + ) + ) + + @Cog.listener() async def on_voice_state_update( self, member: discord.Member, @@ -761,7 +867,8 @@ class ModLog(Cog, name="ModLog"): """Log member voice state changes to the voice log channel.""" if ( member.guild.id != GuildConstant.id - or (before.channel and before.channel.id in GuildConstant.modlog_blacklist) + or (before.channel and self.is_channel_ignored(before.channel.id)) + or (after.channel and self.is_channel_ignored(after.channel.id)) ): return @@ -783,7 +890,7 @@ class ModLog(Cog, name="ModLog"): diff_values = {**diff.get("values_changed", {}), **diff.get("type_changes", {})} icon = Icons.voice_state_blue - colour = Colour.blurple() + colour = Colour.og_blurple() changes = [] for attr, values in diff_values.items(): @@ -820,7 +927,7 @@ class ModLog(Cog, name="ModLog"): colour=colour, title="Voice state updated", text=message, - thumbnail=member.avatar_url_as(static_format="png"), + thumbnail=member.display_avatar.url, channel_id=Channels.voice_log ) diff --git a/bot/exts/moderation/modpings.py b/bot/exts/moderation/modpings.py index 1ad5005de..b5cd29b12 100644 --- a/bot/exts/moderation/modpings.py +++ b/bot/exts/moderation/modpings.py @@ -1,17 +1,22 @@ +import asyncio import datetime -import logging +import arrow from async_rediscache import RedisCache -from dateutil.parser import isoparse +from dateutil.parser import isoparse, parse as dateutil_parse from discord import Embed, Member from discord.ext.commands import Cog, Context, group, has_any_role from bot.bot import Bot from bot.constants import Colours, Emojis, Guild, Icons, MODERATION_ROLES, Roles from bot.converters import Expiry +from bot.log import get_logger +from bot.utils import scheduling, time from bot.utils.scheduling import Scheduler -log = logging.getLogger(__name__) +log = get_logger(__name__) + +MAXIMUM_WORK_LIMIT = 16 class ModPings(Cog): @@ -22,14 +27,28 @@ class ModPings(Cog): # The cache's values are the times when the role should be re-applied to them, stored in ISO format. pings_off_mods = RedisCache() + # RedisCache[discord.Member.id, 'start timestamp|total worktime in seconds'] + # The cache's keys are mod's ID + # The cache's values are their pings on schedule timestamp and the total seconds (work time) until pings off + modpings_schedule = RedisCache() + def __init__(self, bot: Bot): self.bot = bot - self._role_scheduler = Scheduler(self.__class__.__name__) + self._role_scheduler = Scheduler("ModPingsOnOff") + self._modpings_scheduler = Scheduler("ModPingsSchedule") self.guild = None self.moderators_role = None - self.reschedule_task = self.bot.loop.create_task(self.reschedule_roles(), name="mod-pings-reschedule") + self.modpings_schedule_task = scheduling.create_task( + self.reschedule_modpings_schedule(), + event_loop=self.bot.loop + ) + self.reschedule_task = scheduling.create_task( + self.reschedule_roles(), + name="mod-pings-reschedule", + event_loop=self.bot.loop, + ) async def reschedule_roles(self) -> None: """Reschedule moderators role re-apply times.""" @@ -44,7 +63,7 @@ class ModPings(Cog): log.trace("Applying the moderators role to the mod team where necessary.") for mod in mod_team.members: if mod in pings_on: # Make sure that on-duty mods aren't in the cache. - if mod in pings_off: + if mod.id in pings_off: await self.pings_off_mods.delete(mod.id) continue @@ -52,13 +71,61 @@ class ModPings(Cog): if mod.id not in pings_off: await self.reapply_role(mod) else: - expiry = isoparse(pings_off[mod.id]).replace(tzinfo=None) + expiry = isoparse(pings_off[mod.id]) self._role_scheduler.schedule_at(expiry, mod.id, self.reapply_role(mod)) + async def reschedule_modpings_schedule(self) -> None: + """Reschedule moderators schedule ping.""" + await self.bot.wait_until_guild_available() + schedule_cache = await self.modpings_schedule.to_dict() + + log.info("Scheduling modpings schedule for applicable moderators found in cache.") + for mod_id, schedule in schedule_cache.items(): + start_timestamp, work_time = schedule.split("|") + start = datetime.datetime.fromtimestamp(float(start_timestamp)) + + mod = await self.bot.fetch_user(mod_id) + self._modpings_scheduler.schedule_at( + start, + mod_id, + self.add_role_schedule(mod, work_time, start) + ) + + async def remove_role_schedule(self, mod: Member, work_time: int, schedule_start: datetime.datetime) -> None: + """Removes the moderator's role to the given moderator.""" + log.trace(f"Removing moderator role from mod with ID {mod.id}") + await mod.remove_roles(self.moderators_role, reason="Moderator schedule time expired.") + + # Remove the task before scheduling it again + self._modpings_scheduler.cancel(mod.id) + + # Add the task again + log.trace(f"Adding mod pings schedule task again for mod with ID {mod.id}") + schedule_start += datetime.timedelta(days=1) + self._modpings_scheduler.schedule_at( + schedule_start, + mod.id, + self.add_role_schedule(mod, work_time, schedule_start) + ) + + async def add_role_schedule(self, mod: Member, work_time: int, schedule_start: datetime.datetime) -> None: + """Adds the moderator's role to the given moderator.""" + # If the moderator has pings off, then skip adding role + if mod.id in await self.pings_off_mods.to_dict(): + log.trace(f"Skipping adding moderator role to mod with ID {mod.id} - found in pings off cache.") + else: + log.trace(f"Applying moderator role to mod with ID {mod.id}") + await mod.add_roles(self.moderators_role, reason="Moderator scheduled time started!") + + log.trace(f"Sleeping for {work_time} seconds, worktime for mod with ID {mod.id}") + await asyncio.sleep(work_time) + await self.remove_role_schedule(mod, work_time, schedule_start) + async def reapply_role(self, mod: Member) -> None: """Reapply the moderator's role to the given moderator.""" log.trace(f"Re-applying role to mod with ID {mod.id}.") await mod.add_roles(self.moderators_role, reason="Pings off period expired.") + await self.pings_off_mods.delete(mod.id) @group(name='modpings', aliases=('modping',), invoke_without_command=True) @has_any_role(*MODERATION_ROLES) @@ -86,8 +153,7 @@ class ModPings(Cog): The duration cannot be longer than 30 days. """ - duration: datetime.datetime - delta = duration - datetime.datetime.utcnow() + delta = duration - arrow.utcnow() if delta > datetime.timedelta(days=30): await ctx.send(":x: Cannot remove the role for longer than 30 days.") return @@ -126,12 +192,66 @@ class ModPings(Cog): await ctx.send(f"{Emojis.check_mark} Moderators role has been re-applied.") + @modpings_group.group( + name='schedule', + aliases=('s',), + invoke_without_command=True + ) + @has_any_role(*MODERATION_ROLES) + async def schedule_modpings(self, ctx: Context, start: str, end: str) -> None: + """Schedule modpings role to be added at <start> and removed at <end> everyday at UTC time!""" + start, end = dateutil_parse(start), dateutil_parse(end) + + if end < start: + end += datetime.timedelta(days=1) + + if (end - start) > datetime.timedelta(hours=MAXIMUM_WORK_LIMIT): + await ctx.send( + f":x: {ctx.author.mention} You can't have the modpings role for" + f" more than {MAXIMUM_WORK_LIMIT} hours!" + ) + return + + if start < datetime.datetime.utcnow(): + # The datetime has already gone for the day, so make it tomorrow + # otherwise the scheduler would schedule it immediately + start += datetime.timedelta(days=1) + + work_time = (end - start).total_seconds() + + await self.modpings_schedule.set(ctx.author.id, f"{start.timestamp()}|{work_time}") + + if ctx.author.id in self._modpings_scheduler: + self._modpings_scheduler.cancel(ctx.author.id) + + self._modpings_scheduler.schedule_at( + start, + ctx.author.id, + self.add_role_schedule(ctx.author, work_time, start) + ) + + await ctx.send( + f"{Emojis.ok_hand} {ctx.author.mention} Scheduled mod pings from " + f"{time.discord_timestamp(start, time.TimestampFormats.TIME)} to " + f"{time.discord_timestamp(end, time.TimestampFormats.TIME)}!" + ) + + @schedule_modpings.command(name='delete', aliases=('del', 'd')) + async def modpings_schedule_delete(self, ctx: Context) -> None: + """Delete your modpings schedule.""" + self._modpings_scheduler.cancel(ctx.author.id) + await self.modpings_schedule.delete(ctx.author.id) + await ctx.send(f"{Emojis.ok_hand} {ctx.author.mention} Deleted your modpings schedule!") + def cog_unload(self) -> None: """Cancel role tasks when the cog unloads.""" log.trace("Cog unload: canceling role tasks.") self.reschedule_task.cancel() self._role_scheduler.cancel_all() + self.modpings_schedule_task.cancel() + self._modpings_scheduler.cancel_all() + def setup(bot: Bot) -> None: """Load the ModPings cog.""" diff --git a/bot/exts/moderation/silence.py b/bot/exts/moderation/silence.py index 8025f3df6..511520252 100644 --- a/bot/exts/moderation/silence.py +++ b/bot/exts/moderation/silence.py @@ -1,22 +1,24 @@ import json -import logging import typing from contextlib import suppress from datetime import datetime, timedelta, timezone from typing import Optional, OrderedDict, Union from async_rediscache import RedisCache -from discord import Guild, PermissionOverwrite, TextChannel, VoiceChannel +from discord import Guild, PermissionOverwrite, TextChannel, Thread, VoiceChannel from discord.ext import commands, tasks from discord.ext.commands import Context +from discord.utils import MISSING from bot import constants from bot.bot import Bot from bot.converters import HushDurationConverter +from bot.log import get_logger +from bot.utils import scheduling from bot.utils.lock import LockedResourceError, lock, lock_arg from bot.utils.scheduling import Scheduler -log = logging.getLogger(__name__) +log = get_logger(__name__) LOCK_NAMESPACE = "silence" @@ -47,7 +49,16 @@ class SilenceNotifier(tasks.Loop): """Loop notifier for posting notices to `alert_channel` containing added channels.""" def __init__(self, alert_channel: TextChannel): - super().__init__(self._notifier, seconds=1, minutes=0, hours=0, count=None, reconnect=True, loop=None) + super().__init__( + self._notifier, + seconds=1, + minutes=0, + hours=0, + count=None, + reconnect=True, + loop=None, + time=MISSING + ) self._silenced_channels = {} self._alert_channel = alert_channel @@ -104,7 +115,7 @@ class Silence(commands.Cog): self.bot = bot self.scheduler = Scheduler(self.__class__.__name__) - self._init_task = self.bot.loop.create_task(self._async_init()) + self._init_task = scheduling.create_task(self._async_init(), event_loop=self.bot.loop) async def _async_init(self) -> None: """Set instance attributes once the guild is available and reschedule unsilences.""" @@ -172,6 +183,12 @@ class Silence(commands.Cog): channel_info = f"#{channel} ({channel.id})" log.debug(f"{ctx.author} is silencing channel {channel_info}.") + # Since threads don't have specific overrides, we cannot silence them individually. + # The parent channel has to be muted or the thread should be archived. + if isinstance(channel, Thread): + await ctx.send(":x: Threads cannot be silenced.") + return + if not await self._set_silence_overwrites(channel, kick=kick): log.info(f"Tried to silence channel {channel_info} but the channel was already silenced.") await self.send_message(MSG_SILENCE_FAIL, ctx.channel, channel, alert_target=False) @@ -202,8 +219,6 @@ class Silence(commands.Cog): duration: HushDurationConverter ) -> typing.Tuple[TextOrVoiceChannel, Optional[int]]: """Helper method to parse the arguments of the silence command.""" - duration: Optional[int] - if duration_or_channel: if isinstance(duration_or_channel, (TextChannel, VoiceChannel)): channel = duration_or_channel @@ -224,7 +239,13 @@ class Silence(commands.Cog): if isinstance(channel, TextChannel): role = self._everyone_role overwrite = channel.overwrites_for(role) - prev_overwrites = dict(send_messages=overwrite.send_messages, add_reactions=overwrite.add_reactions) + prev_overwrites = dict( + send_messages=overwrite.send_messages, + add_reactions=overwrite.add_reactions, + create_private_threads=overwrite.create_private_threads, + create_public_threads=overwrite.create_public_threads, + send_messages_in_threads=overwrite.send_messages_in_threads + ) else: role = self._verified_voice_role @@ -324,7 +345,15 @@ class Silence(commands.Cog): # Check if old overwrites were not stored if prev_overwrites is None: log.info(f"Missing previous overwrites for #{channel} ({channel.id}); defaulting to None.") - overwrite.update(send_messages=None, add_reactions=None, speak=None, connect=None) + overwrite.update( + send_messages=None, + add_reactions=None, + create_private_threads=None, + create_public_threads=None, + send_messages_in_threads=None, + speak=None, + connect=None + ) else: overwrite.update(**json.loads(prev_overwrites)) diff --git a/bot/exts/moderation/slowmode.py b/bot/exts/moderation/slowmode.py index d8baff76a..b6a771441 100644 --- a/bot/exts/moderation/slowmode.py +++ b/bot/exts/moderation/slowmode.py @@ -1,4 +1,3 @@ -import logging from typing import Optional from dateutil.relativedelta import relativedelta @@ -8,15 +7,16 @@ from discord.ext.commands import Cog, Context, group, has_any_role from bot.bot import Bot from bot.constants import Channels, Emojis, MODERATION_ROLES from bot.converters import DurationDelta +from bot.log import get_logger from bot.utils import time -log = logging.getLogger(__name__) +log = get_logger(__name__) SLOWMODE_MAX_DELAY = 21600 # seconds COMMONLY_SLOWMODED_CHANNELS = { Channels.python_general: "python_general", - Channels.discord_py: "discordpy", + Channels.discord_bots: "discord_bots", Channels.off_topic_0: "ot0", } @@ -39,8 +39,7 @@ class Slowmode(Cog): if channel is None: channel = ctx.channel - delay = relativedelta(seconds=channel.slowmode_delay) - humanized_delay = time.humanize_delta(delay) + humanized_delay = time.humanize_delta(seconds=channel.slowmode_delay) await ctx.send(f'The slowmode delay for {channel.mention} is {humanized_delay}.') diff --git a/bot/exts/moderation/stream.py b/bot/exts/moderation/stream.py index fd856a7f4..985cc6eb1 100644 --- a/bot/exts/moderation/stream.py +++ b/bot/exts/moderation/stream.py @@ -1,4 +1,3 @@ -import logging from datetime import timedelta, timezone from operator import itemgetter @@ -9,13 +8,16 @@ from async_rediscache import RedisCache from discord.ext import commands from bot.bot import Bot -from bot.constants import Colours, Emojis, Guild, MODERATION_ROLES, Roles, STAFF_ROLES, VideoPermission +from bot.constants import ( + Colours, Emojis, Guild, MODERATION_ROLES, Roles, STAFF_PARTNERS_COMMUNITY_ROLES, VideoPermission +) from bot.converters import Expiry +from bot.log import get_logger from bot.pagination import LinePaginator -from bot.utils.scheduling import Scheduler -from bot.utils.time import format_infraction_with_duration +from bot.utils import scheduling, time +from bot.utils.members import get_or_fetch_member -log = logging.getLogger(__name__) +log = get_logger(__name__) class Stream(commands.Cog): @@ -27,8 +29,8 @@ class Stream(commands.Cog): def __init__(self, bot: Bot): self.bot = bot - self.scheduler = Scheduler(self.__class__.__name__) - self.reload_task = self.bot.loop.create_task(self._reload_tasks_from_redis()) + self.scheduler = scheduling.Scheduler(self.__class__.__name__) + self.reload_task = scheduling.create_task(self._reload_tasks_from_redis(), event_loop=self.bot.loop) def cog_unload(self) -> None: """Cancel all scheduled tasks.""" @@ -44,23 +46,17 @@ class Stream(commands.Cog): """Reload outstanding tasks from redis on startup, delete the task if the member has since left the server.""" await self.bot.wait_until_guild_available() items = await self.task_cache.items() + guild = self.bot.get_guild(Guild.id) for key, value in items: - member = self.bot.get_guild(Guild.id).get_member(key) + member = await get_or_fetch_member(guild, key) if not member: - # Member isn't found in the cache - try: - member = await self.bot.get_guild(Guild.id).fetch_member(key) - except discord.errors.NotFound: - log.debug( - f"Member {key} left the guild before we could schedule " - "the revoking of their streaming permissions." - ) - await self.task_cache.delete(key) - continue - except discord.HTTPException: - log.exception(f"Exception while trying to retrieve member {key} from Discord.") - continue + log.debug( + "User with ID %d left the guild before their streaming permissions could be revoked.", + key + ) + await self.task_cache.delete(key) + continue revoke_time = Arrow.utcfromtimestamp(value) log.debug(f"Scheduling {member} ({member.id}) to have streaming permission revoked at {revoke_time}") @@ -134,20 +130,15 @@ class Stream(commands.Cog): await member.add_roles(discord.Object(Roles.video), reason="Temporary streaming access granted") - # Use embed as embed timestamps do timezone conversions. - embed = discord.Embed( - description=f"{Emojis.check_mark} {member.mention} can now stream.", - colour=Colours.soft_green - ) - embed.set_footer(text=f"Streaming permission has been given to {member} until") - embed.timestamp = duration - - # Mention in content as mentions in embeds don't ping - await ctx.send(content=member.mention, embed=embed) + await ctx.send(f"{Emojis.check_mark} {member.mention} can now stream until {time.discord_timestamp(duration)}.") # Convert here for nicer logging - revoke_time = format_infraction_with_duration(str(duration)) - log.debug(f"Successfully gave {member} ({member.id}) permission to stream until {revoke_time}.") + humanized_duration = time.humanize_delta(duration, arrow.utcnow(), max_units=2) + end_time = duration.strftime("%Y-%m-%d %H:%M:%S") + log.debug( + f"Successfully gave {member} ({member.id}) permission " + f"to stream for {humanized_duration} (until {end_time})." + ) @commands.command(aliases=("pstream",)) @commands.has_any_role(*MODERATION_ROLES) @@ -202,17 +193,17 @@ class Stream(commands.Cog): @commands.command(aliases=('lstream',)) @commands.has_any_role(*MODERATION_ROLES) async def liststream(self, ctx: commands.Context) -> None: - """Lists all non-staff users who have permission to stream.""" - non_staff_members_with_stream = [ + """Lists all users who aren't staff, partners or members of the python community and have stream permissions.""" + non_staff_partners_community_members_with_stream = [ member for member in ctx.guild.get_role(Roles.video).members - if not any(role.id in STAFF_ROLES for role in member.roles) + if not any(role.id in STAFF_PARTNERS_COMMUNITY_ROLES for role in member.roles) ] # List of tuples (UtcPosixTimestamp, str) # So that the list can be sorted on the UtcPosixTimestamp before the message is passed to the paginator. streamer_info = [] - for member in non_staff_members_with_stream: + for member in non_staff_partners_community_members_with_stream: if revoke_time := await self.task_cache.get(member.id): # Member only has temporary streaming perms revoke_delta = Arrow.utcfromtimestamp(revoke_time).humanize() diff --git a/bot/exts/moderation/verification.py b/bot/exts/moderation/verification.py index bfe9b74b4..37338d19c 100644 --- a/bot/exts/moderation/verification.py +++ b/bot/exts/moderation/verification.py @@ -1,4 +1,3 @@ -import logging import typing as t import discord @@ -6,10 +5,9 @@ from discord.ext.commands import Cog, Context, command, has_any_role from bot import constants from bot.bot import Bot -from bot.decorators import in_whitelist -from bot.utils.checks import InWhitelistCheckFailure +from bot.log import get_logger -log = logging.getLogger(__name__) +log = get_logger(__name__) # Sent via DMs once user joins the guild ON_JOIN_MESSAGE = """ @@ -29,11 +27,11 @@ You can find a copy of our rules for reference at <https://pythondiscord.com/pag Additionally, if you'd like to receive notifications for the announcements \ we post in <#{constants.Channels.announcements}> -from time to time, you can send `!subscribe` to <#{constants.Channels.bot_commands}> at any time \ +from time to time, you can send `{constants.Bot.prefix}subscribe` to <#{constants.Channels.bot_commands}> at any time \ to assign yourself the **Announcements** role. We'll mention this role every time we make an announcement. -If you'd like to unsubscribe from the announcement notifications, simply send `!unsubscribe` to \ -<#{constants.Channels.bot_commands}>. +If you'd like to unsubscribe from the announcement notifications, simply send `{constants.Bot.prefix}subscribe` to \ +<#{constants.Channels.bot_commands}> and click the role again!. To introduce you to our community, we've made the following video: https://youtu.be/ZH26PuX3re0 @@ -61,11 +59,9 @@ async def safe_dm(coro: t.Coroutine) -> None: class Verification(Cog): """ - User verification and role management. + User verification. Statistics are collected in the 'verification.' namespace. - - Additionally, this cog offers the !subscribe and !unsubscribe commands, """ def __init__(self, bot: Bot) -> None: @@ -108,67 +104,8 @@ class Verification(Cog): log.exception("DM dispatch failed on unexpected error code") # endregion - # region: subscribe commands - - @command(name='subscribe') - @in_whitelist(channels=(constants.Channels.bot_commands,)) - async def subscribe_command(self, ctx: Context, *_) -> None: # We don't actually care about the args - """Subscribe to announcement notifications by assigning yourself the role.""" - has_role = False - - for role in ctx.author.roles: - if role.id == constants.Roles.announcements: - has_role = True - break - - if has_role: - await ctx.send(f"{ctx.author.mention} You're already subscribed!") - return - - log.debug(f"{ctx.author} called !subscribe. Assigning the 'Announcements' role.") - await ctx.author.add_roles(discord.Object(constants.Roles.announcements), reason="Subscribed to announcements") - - log.trace(f"Deleting the message posted by {ctx.author}.") - - await ctx.send( - f"{ctx.author.mention} Subscribed to <#{constants.Channels.announcements}> notifications.", - ) - - @command(name='unsubscribe') - @in_whitelist(channels=(constants.Channels.bot_commands,)) - async def unsubscribe_command(self, ctx: Context, *_) -> None: # We don't actually care about the args - """Unsubscribe from announcement notifications by removing the role from yourself.""" - has_role = False - - for role in ctx.author.roles: - if role.id == constants.Roles.announcements: - has_role = True - break - - if not has_role: - await ctx.send(f"{ctx.author.mention} You're already unsubscribed!") - return - - log.debug(f"{ctx.author} called !unsubscribe. Removing the 'Announcements' role.") - await ctx.author.remove_roles( - discord.Object(constants.Roles.announcements), reason="Unsubscribed from announcements" - ) - - log.trace(f"Deleting the message posted by {ctx.author}.") - - await ctx.send( - f"{ctx.author.mention} Unsubscribed from <#{constants.Channels.announcements}> notifications." - ) - - # endregion # region: miscellaneous - # This cannot be static (must have a __func__ attribute). - async def cog_command_error(self, ctx: Context, error: Exception) -> None: - """Check for & ignore any InWhitelistCheckFailure.""" - if isinstance(error, InWhitelistCheckFailure): - error.handled = True - @command(name='verify') @has_any_role(*constants.MODERATION_ROLES) async def perform_manual_verification(self, ctx: Context, user: discord.Member) -> None: diff --git a/bot/exts/moderation/voice_gate.py b/bot/exts/moderation/voice_gate.py index 8494a1e2e..fa66b00dd 100644 --- a/bot/exts/moderation/voice_gate.py +++ b/bot/exts/moderation/voice_gate.py @@ -1,22 +1,22 @@ import asyncio -import logging from contextlib import suppress -from datetime import datetime, timedelta +from datetime import timedelta +import arrow import discord from async_rediscache import RedisCache from discord import Colour, Member, VoiceState from discord.ext.commands import Cog, Context, command - from bot.api import ResponseCodeError from bot.bot import Bot from bot.constants import Channels, Event, MODERATION_ROLES, Roles, VoiceGate as GateConf from bot.decorators import has_no_roles, in_whitelist from bot.exts.moderation.modlog import ModLog +from bot.log import get_logger from bot.utils.checks import InWhitelistCheckFailure -log = logging.getLogger(__name__) +log = get_logger(__name__) # Flag written to the cog's RedisCache as a value when the Member's (key) notification # was already removed ~ this signals both that no further notifications should be sent, @@ -30,7 +30,7 @@ FAILED_MESSAGE = ( MESSAGE_FIELD_MAP = { "joined_at": f"have been on the server for less than {GateConf.minimum_days_member} days", - "voice_banned": "have an active voice ban infraction", + "voice_gate_blocked": "have an active voice infraction", "total_messages": f"have sent less than {GateConf.minimum_messages} messages", "activity_blocks": f"have been active for fewer than {GateConf.minimum_activity_blocks} ten-minute blocks", } @@ -166,11 +166,14 @@ class VoiceGate(Cog): return checks = { - "joined_at": ctx.author.joined_at > datetime.utcnow() - timedelta(days=GateConf.minimum_days_member), + "joined_at": ( + ctx.author.joined_at > arrow.utcnow() - timedelta(days=GateConf.minimum_days_member) + ), "total_messages": data["total_messages"] < GateConf.minimum_messages, - "voice_banned": data["voice_banned"], - "activity_blocks": data["activity_blocks"] < GateConf.minimum_activity_blocks + "voice_gate_blocked": data["voice_gate_blocked"], + "activity_blocks": data["activity_blocks"] < GateConf.minimum_activity_blocks, } + failed = any(checks.values()) failed_reasons = [MESSAGE_FIELD_MAP[key] for key, value in checks.items() if value is True] [self.bot.stats.incr(f"voice_gate.failed.{key}") for key, value in checks.items() if value is True] diff --git a/bot/exts/moderation/watchchannels/_watchchannel.py b/bot/exts/moderation/watchchannels/_watchchannel.py index 9f26c34f2..ee9b6ba45 100644 --- a/bot/exts/moderation/watchchannels/_watchchannel.py +++ b/bot/exts/moderation/watchchannels/_watchchannel.py @@ -1,5 +1,4 @@ import asyncio -import logging import re import textwrap from abc import abstractmethod @@ -17,11 +16,12 @@ from bot.constants import BigBrother as BigBrotherConfig, Guild as GuildConfig, from bot.exts.filters.token_remover import TokenRemover from bot.exts.filters.webhook_remover import WEBHOOK_URL_RE from bot.exts.moderation.modlog import ModLog +from bot.log import CustomLogger, get_logger from bot.pagination import LinePaginator -from bot.utils import CogABCMeta, messages -from bot.utils.time import get_time_delta +from bot.utils import CogABCMeta, messages, scheduling, time +from bot.utils.members import get_or_fetch_member -log = logging.getLogger(__name__) +log = get_logger(__name__) URL_RE = re.compile(r"(https?://[^\s]+)") @@ -46,7 +46,7 @@ class WatchChannel(metaclass=CogABCMeta): webhook_id: int, api_endpoint: str, api_default_params: dict, - logger: logging.Logger, + logger: CustomLogger, *, disable_header: bool = False ) -> None: @@ -69,7 +69,7 @@ class WatchChannel(metaclass=CogABCMeta): self.message_history = MessageHistory() self.disable_header = disable_header - self._start = self.bot.loop.create_task(self.start_watchchannel()) + self._start = scheduling.create_task(self.start_watchchannel(), event_loop=self.bot.loop) @property def modlog(self) -> ModLog: @@ -169,7 +169,7 @@ class WatchChannel(metaclass=CogABCMeta): """Queues up messages sent by watched users.""" if msg.author.id in self.watched_users: if not self.consuming_messages: - self._consume_task = self.bot.loop.create_task(self.consume_messages()) + self._consume_task = scheduling.create_task(self.consume_messages(), event_loop=self.bot.loop) self.log.trace(f"Received message: {msg.content} ({len(msg.attachments)} attachments)") self.message_queue[msg.author.id][msg.channel.id].append(msg) @@ -199,7 +199,10 @@ class WatchChannel(metaclass=CogABCMeta): if self.message_queue: self.log.trace("Channel queue not empty: Continuing consuming queues") - self._consume_task = self.bot.loop.create_task(self.consume_messages(delay_consumption=False)) + self._consume_task = scheduling.create_task( + self.consume_messages(delay_consumption=False), + event_loop=self.bot.loop, + ) else: self.log.trace("Done consuming messages.") @@ -246,7 +249,7 @@ class WatchChannel(metaclass=CogABCMeta): await self.webhook_send( cleaned_content, username=msg.author.display_name, - avatar_url=msg.author.avatar_url + avatar_url=msg.author.display_avatar.url ) if msg.attachments: @@ -260,7 +263,7 @@ class WatchChannel(metaclass=CogABCMeta): await self.webhook_send( embed=e, username=msg.author.display_name, - avatar_url=msg.author.avatar_url + avatar_url=msg.author.display_avatar.url ) except discord.HTTPException as exc: self.log.exception( @@ -278,11 +281,11 @@ class WatchChannel(metaclass=CogABCMeta): user_id = msg.author.id guild = self.bot.get_guild(GuildConfig.id) - actor = guild.get_member(self.watched_users[user_id]['actor']) + actor = await get_or_fetch_member(guild, self.watched_users[user_id]['actor']) actor = actor.display_name if actor else self.watched_users[user_id]['actor'] inserted_at = self.watched_users[user_id]['inserted_at'] - time_delta = get_time_delta(inserted_at) + time_delta = time.format_relative(inserted_at) reason = self.watched_users[user_id]['reason'] @@ -294,10 +297,9 @@ class WatchChannel(metaclass=CogABCMeta): message_jump = f"in [#{msg.channel.name}]({msg.jump_url})" footer = f"Added {time_delta} by {actor} | Reason: {reason}" - embed = Embed(description=f"{msg.author.mention} {message_jump}") - embed.set_footer(text=textwrap.shorten(footer, width=128, placeholder="...")) + embed = Embed(description=f"{msg.author.mention} {message_jump}\n\n{footer}") - await self.webhook_send(embed=embed, username=msg.author.display_name, avatar_url=msg.author.avatar_url) + await self.webhook_send(embed=embed, username=msg.author.display_name, avatar_url=msg.author.display_avatar.url) async def list_watched_users( self, ctx: Context, oldest_first: bool = False, update_cache: bool = True @@ -352,12 +354,12 @@ class WatchChannel(metaclass=CogABCMeta): list_data["info"] = {} for user_id, user_data in watched_iter: - member = ctx.guild.get_member(user_id) + member = await get_or_fetch_member(ctx.guild, user_id) line = f"• `{user_id}`" if member: line += f" ({member.name}#{member.discriminator})" inserted_at = user_data['inserted_at'] - line += f", added {get_time_delta(inserted_at)}" + line += f", added {time.format_relative(inserted_at)}" if not member: # Cross off users who left the server. line = f"~~{line}~~" list_data["info"][user_id] = line diff --git a/bot/exts/moderation/watchchannels/bigbrother.py b/bot/exts/moderation/watchchannels/bigbrother.py index c6ee844ef..ab37b1b80 100644 --- a/bot/exts/moderation/watchchannels/bigbrother.py +++ b/bot/exts/moderation/watchchannels/bigbrother.py @@ -1,4 +1,3 @@ -import logging import textwrap from collections import ChainMap @@ -6,11 +5,12 @@ from discord.ext.commands import Cog, Context, group, has_any_role from bot.bot import Bot from bot.constants import Channels, MODERATION_ROLES, Webhooks -from bot.converters import FetchedMember +from bot.converters import MemberOrUser from bot.exts.moderation.infraction._utils import post_infraction from bot.exts.moderation.watchchannels._watchchannel import WatchChannel +from bot.log import get_logger -log = logging.getLogger(__name__) +log = get_logger(__name__) class BigBrother(WatchChannel, Cog, name="Big Brother"): @@ -60,7 +60,7 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): @bigbrother_group.command(name='watch', aliases=('w',), root_aliases=('watch',)) @has_any_role(*MODERATION_ROLES) - async def watch_command(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: + async def watch_command(self, ctx: Context, user: MemberOrUser, *, reason: str) -> None: """ Relay messages sent by the given `user` to the `#big-brother` channel. @@ -71,11 +71,11 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): @bigbrother_group.command(name='unwatch', aliases=('uw',), root_aliases=('unwatch',)) @has_any_role(*MODERATION_ROLES) - async def unwatch_command(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: + async def unwatch_command(self, ctx: Context, user: MemberOrUser, *, reason: str) -> None: """Stop relaying messages by the given `user`.""" await self.apply_unwatch(ctx, user, reason) - async def apply_watch(self, ctx: Context, user: FetchedMember, reason: str) -> None: + async def apply_watch(self, ctx: Context, user: MemberOrUser, reason: str) -> None: """ Add `user` to watched users and apply a watch infraction with `reason`. @@ -87,14 +87,14 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): return if not await self.fetch_user_cache(): - await ctx.send(f":x: Updating the user cache failed, can't watch user {user}") + await ctx.send(f":x: Updating the user cache failed, can't watch user {user.mention}") return if user.id in self.watched_users: - await ctx.send(f":x: {user} is already being watched.") + await ctx.send(f":x: {user.mention} is already being watched.") return - # FetchedUser instances don't have a roles attribute + # discord.User instances don't have a roles attribute if hasattr(user, "roles") and any(role.id in MODERATION_ROLES for role in user.roles): await ctx.send(f":x: I'm sorry {ctx.author}, I'm afraid I can't do that. I must be kind to my masters.") return @@ -103,7 +103,7 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): if response is not None: self.watched_users[user.id] = response - msg = f":white_check_mark: Messages sent by {user} will now be relayed to Big Brother." + msg = f":white_check_mark: Messages sent by {user.mention} will now be relayed to Big Brother." history = await self.bot.api_client.get( self.api_endpoint, @@ -125,7 +125,7 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): await ctx.send(msg) - async def apply_unwatch(self, ctx: Context, user: FetchedMember, reason: str, send_message: bool = True) -> None: + async def apply_unwatch(self, ctx: Context, user: MemberOrUser, reason: str, send_message: bool = True) -> None: """ Remove `user` from watched users and mark their infraction as inactive with `reason`. @@ -156,7 +156,7 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): log.debug(f"Perma-banned user {user} was unwatched.") return log.trace("User is not banned. Sending message to channel") - message = f":white_check_mark: Messages sent by {user} will no longer be relayed." + message = f":white_check_mark: Messages sent by {user.mention} will no longer be relayed." else: log.trace("No active watches found for user.") diff --git a/bot/exts/recruitment/talentpool/_cog.py b/bot/exts/recruitment/talentpool/_cog.py index 03326cab2..0554bf37a 100644 --- a/bot/exts/recruitment/talentpool/_cog.py +++ b/bot/exts/recruitment/talentpool/_cog.py @@ -1,69 +1,152 @@ -import logging import textwrap -from collections import ChainMap +from collections import ChainMap, defaultdict from io import StringIO -from typing import Union +from typing import Optional, Union import discord +from async_rediscache import RedisCache from discord import Color, Embed, Member, PartialMessage, RawReactionActionEvent, User -from discord.ext.commands import Cog, Context, group, has_any_role +from discord.ext.commands import BadArgument, Cog, Context, group, has_any_role from bot.api import ResponseCodeError from bot.bot import Bot -from bot.constants import Channels, Emojis, Guild, MODERATION_ROLES, STAFF_ROLES, Webhooks -from bot.converters import FetchedMember -from bot.exts.moderation.watchchannels._watchchannel import WatchChannel +from bot.constants import Channels, Emojis, Guild, MODERATION_ROLES, Roles, STAFF_ROLES +from bot.converters import MemberOrUser, UnambiguousMemberOrUser from bot.exts.recruitment.talentpool._review import Reviewer +from bot.log import get_logger from bot.pagination import LinePaginator -from bot.utils import time +from bot.utils import scheduling, time +from bot.utils.members import get_or_fetch_member +AUTOREVIEW_ENABLED_KEY = "autoreview_enabled" REASON_MAX_CHARS = 1000 -log = logging.getLogger(__name__) +log = get_logger(__name__) -class TalentPool(WatchChannel, Cog, name="Talentpool"): - """Relays messages of helper candidates to a watch channel to observe them.""" +class TalentPool(Cog, name="Talentpool"): + """Used to nominate potential helper candidates.""" - def __init__(self, bot: Bot) -> None: - super().__init__( - bot, - destination=Channels.talent_pool, - webhook_id=Webhooks.talent_pool, - api_endpoint='bot/nominations', - api_default_params={'active': 'true', 'ordering': '-inserted_at'}, - logger=log, - disable_header=True, - ) + # RedisCache[str, bool] + # Can contain a single key, "autoreview_enabled", with the value a bool indicating if autoreview is enabled. + talentpool_settings = RedisCache() + def __init__(self, bot: Bot) -> None: + self.bot = bot self.reviewer = Reviewer(self.__class__.__name__, bot, self) - self.bot.loop.create_task(self.reviewer.reschedule_reviews()) + self.cache: Optional[defaultdict[dict]] = None + self.api_default_params = {'active': 'true', 'ordering': '-inserted_at'} + + self.initial_refresh_task = scheduling.create_task(self.refresh_cache(), event_loop=self.bot.loop) + scheduling.create_task(self.schedule_autoreviews(), event_loop=self.bot.loop) + + async def schedule_autoreviews(self) -> None: + """Reschedule reviews for active nominations if autoreview is enabled.""" + if await self.autoreview_enabled(): + # Wait for a populated cache first + await self.initial_refresh_task + await self.reviewer.reschedule_reviews() + else: + log.trace("Not scheduling reviews as autoreview is disabled.") + + async def autoreview_enabled(self) -> bool: + """Return whether automatic posting of nomination reviews is enabled.""" + return await self.talentpool_settings.get(AUTOREVIEW_ENABLED_KEY, True) + + async def refresh_cache(self) -> bool: + """Updates TalentPool users cache.""" + # Wait until logged in to ensure bot api client exists + await self.bot.wait_until_guild_available() + try: + data = await self.bot.api_client.get( + 'bot/nominations', + params=self.api_default_params + ) + except ResponseCodeError as err: + log.exception("Failed to fetch the currently nominated users from the API", exc_info=err) + return False + + self.cache = defaultdict(dict) + + for entry in data: + user_id = entry.pop('user') + self.cache[user_id] = entry + + return True @group(name='talentpool', aliases=('tp', 'talent', 'nomination', 'n'), invoke_without_command=True) - @has_any_role(*MODERATION_ROLES) + @has_any_role(*STAFF_ROLES) async def nomination_group(self, ctx: Context) -> None: """Highlights the activity of helper nominees by relaying their messages to the talent pool channel.""" await ctx.send_help(ctx.command) - @nomination_group.command(name='watched', aliases=('all', 'list'), root_aliases=("nominees",)) + @nomination_group.group(name="autoreview", aliases=("ar",), invoke_without_command=True) + @has_any_role(*MODERATION_ROLES) + async def nomination_autoreview_group(self, ctx: Context) -> None: + """Commands for enabling or disabling autoreview.""" + await ctx.send_help(ctx.command) + + @nomination_autoreview_group.command(name="enable", aliases=("on",)) + @has_any_role(Roles.admins) + async def autoreview_enable(self, ctx: Context) -> None: + """ + Enable automatic posting of reviews. + + This will post reviews up to one day overdue. Older nominations can be + manually reviewed with the `tp post_review <user_id>` command. + """ + if await self.autoreview_enabled(): + await ctx.send(":x: Autoreview is already enabled") + return + + await self.talentpool_settings.set(AUTOREVIEW_ENABLED_KEY, True) + await self.reviewer.reschedule_reviews() + await ctx.send(":white_check_mark: Autoreview enabled") + + @nomination_autoreview_group.command(name="disable", aliases=("off",)) + @has_any_role(Roles.admins) + async def autoreview_disable(self, ctx: Context) -> None: + """Disable automatic posting of reviews.""" + if not await self.autoreview_enabled(): + await ctx.send(":x: Autoreview is already disabled") + return + + await self.talentpool_settings.set(AUTOREVIEW_ENABLED_KEY, False) + self.reviewer.cancel_all() + await ctx.send(":white_check_mark: Autoreview disabled") + + @nomination_autoreview_group.command(name="status") @has_any_role(*MODERATION_ROLES) - async def watched_command( + async def autoreview_status(self, ctx: Context) -> None: + """Show whether automatic posting of reviews is enabled or disabled.""" + if await self.autoreview_enabled(): + await ctx.send("Autoreview is currently enabled") + else: + await ctx.send("Autoreview is currently disabled") + + @nomination_group.command( + name="nominees", + aliases=("nominated", "all", "list", "watched"), + root_aliases=("nominees",) + ) + @has_any_role(*MODERATION_ROLES) + async def list_command( self, ctx: Context, oldest_first: bool = False, update_cache: bool = True ) -> None: """ - Shows the users that are currently being monitored in the talent pool. + Shows the users that are currently in the talent pool. The optional kwarg `oldest_first` can be used to order the list by oldest nomination. The optional kwarg `update_cache` can be used to update the user cache using the API before listing the users. """ - await self.list_watched_users(ctx, oldest_first=oldest_first, update_cache=update_cache) + await self.list_nominated_users(ctx, oldest_first=oldest_first, update_cache=update_cache) - async def list_watched_users( + async def list_nominated_users( self, ctx: Context, oldest_first: bool = False, @@ -80,16 +163,27 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): The optional kwarg `update_cache` specifies whether the cache should be refreshed by polling the API. """ - # TODO Once the watch channel is removed, this can be done in a smarter way, without splitting and overriding - # the list_watched_users function. - watched_data = await self.prepare_watched_users_data(ctx, oldest_first, update_cache) + successful_update = False + if update_cache: + if not (successful_update := await self.refresh_cache()): + await ctx.send(":warning: Unable to update cache. Data may be inaccurate.") - if update_cache and not watched_data["updated"]: - await ctx.send(f":x: Failed to update {self.__class__.__name__} user cache, serving from cache") + nominations = self.cache.items() + if oldest_first: + nominations = reversed(nominations) lines = [] - for user_id, line in watched_data["info"].items(): - if self.watched_users[user_id]['reviewed']: + + for user_id, user_data in nominations: + member = await get_or_fetch_member(ctx.guild, user_id) + line = f"• `{user_id}`" + if member: + line += f" ({member.name}#{member.discriminator})" + inserted_at = user_data['inserted_at'] + line += f", added {time.format_relative(inserted_at)}" + if not member: # Cross off users who left the server. + line = f"~~{line}~~" + if user_data['reviewed']: line += " *(reviewed)*" elif user_id in self.reviewer: line += " *(scheduled)*" @@ -99,7 +193,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): lines = ("There's nothing here yet.",) embed = Embed( - title=watched_data["title"], + title=f"Talent Pool active nominations ({'updated' if update_cache and successful_update else 'cached'})", color=Color.blue() ) await LinePaginator.paginate(lines, ctx, embed, empty=False) @@ -108,26 +202,30 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): @has_any_role(*MODERATION_ROLES) async def oldest_command(self, ctx: Context, update_cache: bool = True) -> None: """ - Shows talent pool monitored users ordered by oldest nomination. + Shows talent pool users ordered by oldest nomination. The optional kwarg `update_cache` can be used to update the user cache using the API before listing the users. """ - await ctx.invoke(self.watched_command, oldest_first=True, update_cache=update_cache) + await ctx.invoke(self.list_command, oldest_first=True, update_cache=update_cache) - @nomination_group.command(name='forcewatch', aliases=('fw', 'forceadd', 'fa'), root_aliases=("forcenominate",)) + @nomination_group.command( + name="forcenominate", + aliases=("fw", "forceadd", "fa", "fn", "forcewatch"), + root_aliases=("forcenominate",) + ) @has_any_role(*MODERATION_ROLES) - async def force_watch_command(self, ctx: Context, user: FetchedMember, *, reason: str = '') -> None: + async def force_nominate_command(self, ctx: Context, user: MemberOrUser, *, reason: str = '') -> None: """ Adds the given `user` to the talent pool, from any channel. A `reason` for adding the user to the talent pool is optional. """ - await self._watch_user(ctx, user, reason) + await self._nominate_user(ctx, user, reason) - @nomination_group.command(name='watch', aliases=('w', 'add', 'a'), root_aliases=("nominate",)) + @nomination_group.command(name='nominate', aliases=("w", "add", "a", "watch"), root_aliases=("nominate",)) @has_any_role(*STAFF_ROLES) - async def watch_command(self, ctx: Context, user: FetchedMember, *, reason: str = '') -> None: + async def nominate_command(self, ctx: Context, user: MemberOrUser, *, reason: str = '') -> None: """ Adds the given `user` to the talent pool. @@ -138,35 +236,35 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): if any(role.id in MODERATION_ROLES for role in ctx.author.roles): await ctx.send( f":x: Nominations should be run in the <#{Channels.nominations}> channel. " - "Use `!tp forcewatch` to override this check." + "Use `!tp forcenominate` to override this check." ) else: await ctx.send(f":x: Nominations must be run in the <#{Channels.nominations}> channel") return - await self._watch_user(ctx, user, reason) + await self._nominate_user(ctx, user, reason) - async def _watch_user(self, ctx: Context, user: FetchedMember, reason: str) -> None: + async def _nominate_user(self, ctx: Context, user: MemberOrUser, reason: str) -> None: """Adds the given user to the talent pool.""" if user.bot: - await ctx.send(f":x: I'm sorry {ctx.author}, I'm afraid I can't do that. I only watch humans.") + await ctx.send(f":x: I'm sorry {ctx.author}, I'm afraid I can't do that. Only humans can be nominated.") return if isinstance(user, Member) and any(role.id in STAFF_ROLES for role in user.roles): await ctx.send(":x: Nominating staff members, eh? Here's a cookie :cookie:") return - if not await self.fetch_user_cache(): - await ctx.send(f":x: Failed to update the user cache; can't add {user}") + if not await self.refresh_cache(): + await ctx.send(f":x: Failed to update the cache; can't add {user}") return if len(reason) > REASON_MAX_CHARS: - await ctx.send(f":x: Maxiumum allowed characters for the reason is {REASON_MAX_CHARS}.") + await ctx.send(f":x: Maximum allowed characters for the reason is {REASON_MAX_CHARS}.") return # Manual request with `raise_for_status` as False because we want the actual response session = self.bot.api_client.session - url = self.bot.api_client._url_for(self.api_endpoint) + url = self.bot.api_client._url_for('bot/nominations') kwargs = { 'json': { 'actor': ctx.author.id, @@ -188,32 +286,21 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): else: resp.raise_for_status() - self.watched_users[user.id] = response_data + self.cache[user.id] = response_data - if user.id not in self.reviewer: + if await self.autoreview_enabled() and user.id not in self.reviewer: self.reviewer.schedule_review(user.id) - history = await self.bot.api_client.get( - self.api_endpoint, - params={ - "user__id": str(user.id), - "active": "false", - "ordering": "-inserted_at" - } - ) - - msg = f"✅ The nomination for {user} has been added to the talent pool" - if history: - msg += f"\n\n({len(history)} previous nominations in total)" + msg = f"✅ The nomination for {user.mention} has been added to the talent pool" await ctx.send(msg) @nomination_group.command(name='history', aliases=('info', 'search')) @has_any_role(*MODERATION_ROLES) - async def history_command(self, ctx: Context, user: FetchedMember) -> None: + async def history_command(self, ctx: Context, user: MemberOrUser) -> None: """Shows the specified user's nomination history.""" result = await self.bot.api_client.get( - self.api_endpoint, + 'bot/nominations', params={ 'user__id': str(user.id), 'ordering': "-active,-inserted_at" @@ -227,7 +314,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): title=f"Nominations for {user.display_name} `({user.id})`", color=Color.blue() ) - lines = [self._nomination_to_string(nomination) for nomination in result] + lines = [await self._nomination_to_string(nomination) for nomination in result] await LinePaginator.paginate( lines, ctx=ctx, @@ -237,42 +324,99 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): max_size=1000 ) - @nomination_group.command(name='unwatch', aliases=('end', ), root_aliases=("unnominate",)) + @nomination_group.command(name="end", aliases=("unwatch", "unnominate"), root_aliases=("unnominate",)) @has_any_role(*MODERATION_ROLES) - async def unwatch_command(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: + async def end_nomination_command(self, ctx: Context, user: MemberOrUser, *, reason: str) -> None: """ Ends the active nomination of the specified user with the given reason. Providing a `reason` is required. """ if len(reason) > REASON_MAX_CHARS: - await ctx.send(f":x: Maxiumum allowed characters for the end reason is {REASON_MAX_CHARS}.") + await ctx.send(f":x: Maximum allowed characters for the end reason is {REASON_MAX_CHARS}.") return - if await self.unwatch(user.id, reason): - await ctx.send(f":white_check_mark: Messages sent by {user} will no longer be relayed") + if await self.end_nomination(user.id, reason): + await ctx.send(f":white_check_mark: Successfully un-nominated {user}") else: await ctx.send(":x: The specified user does not have an active nomination") @nomination_group.group(name='edit', aliases=('e',), invoke_without_command=True) - @has_any_role(*MODERATION_ROLES) + @has_any_role(*STAFF_ROLES) async def nomination_edit_group(self, ctx: Context) -> None: """Commands to edit nominations.""" await ctx.send_help(ctx.command) @nomination_edit_group.command(name='reason') - @has_any_role(*MODERATION_ROLES) - async def edit_reason_command(self, ctx: Context, nomination_id: int, actor: FetchedMember, *, reason: str) -> None: - """Edits the reason of a specific nominator in a specific active nomination.""" + @has_any_role(*STAFF_ROLES) + async def edit_reason_command( + self, + ctx: Context, + nominee_or_nomination_id: Union[UnambiguousMemberOrUser, int], + nominator: Optional[UnambiguousMemberOrUser] = None, + *, + reason: str + ) -> None: + """ + Edit the nomination reason of a specific nominator for a given nomination. + + If nominee_or_nomination_id resolves to a member or user, edit the currently active nomination for that person. + Otherwise, if it's an int, look up that nomination ID to edit. + + If no nominator is specified, assume the invoker is editing their own nomination reason. + Otherwise, edit the reason from that specific nominator. + + Raise a permission error if a non-mod staff member invokes this command on a + specific nomination ID, or with an nominator other than themselves. + """ + # If not specified, assume the invoker is editing their own nomination reason. + nominator = nominator or ctx.author + + if not any(role.id in MODERATION_ROLES for role in ctx.author.roles): + if ctx.channel.id != Channels.nominations: + await ctx.send(f":x: Nomination edits must be run in the <#{Channels.nominations}> channel") + return + + if nominator != ctx.author or isinstance(nominee_or_nomination_id, int): + # Invoker has specified another nominator, or a specific nomination id + raise BadArgument( + "Only moderators can edit specific nomination IDs, " + "or the reason of a nominator other than themselves." + ) + + await self._edit_nomination_reason( + ctx, + target=nominee_or_nomination_id, + actor=nominator, + reason=reason + ) + + async def _edit_nomination_reason( + self, + ctx: Context, + *, + target: Union[int, Member, User], + actor: MemberOrUser, + reason: str, + ) -> None: + """Edit a nomination reason in the database after validating the input.""" if len(reason) > REASON_MAX_CHARS: - await ctx.send(f":x: Maxiumum allowed characters for the reason is {REASON_MAX_CHARS}.") + await ctx.send(f":x: Maximum allowed characters for the reason is {REASON_MAX_CHARS}.") return + if isinstance(target, int): + nomination_id = target + else: + if nomination := self.cache.get(target.id): + nomination_id = nomination["id"] + else: + await ctx.send("No active nomination found for that member.") + return try: - nomination = await self.bot.api_client.get(f"{self.api_endpoint}/{nomination_id}") + nomination = await self.bot.api_client.get(f"bot/nominations/{nomination_id}") except ResponseCodeError as e: if e.response.status == 404: - self.log.trace(f"Nomination API 404: Can't find a nomination with id {nomination_id}") + log.trace(f"Nomination API 404: Can't find a nomination with id {nomination_id}") await ctx.send(f":x: Can't find a nomination with id `{nomination_id}`") return else: @@ -283,16 +427,16 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): return if not any(entry["actor"] == actor.id for entry in nomination["entries"]): - await ctx.send(f":x: {actor} doesn't have an entry in this nomination.") + await ctx.send(f":x: {actor.mention} doesn't have an entry in this nomination.") return - self.log.trace(f"Changing reason for nomination with id {nomination_id} of actor {actor} to {repr(reason)}") + log.trace(f"Changing reason for nomination with id {nomination_id} of actor {actor} to {repr(reason)}") await self.bot.api_client.patch( - f"{self.api_endpoint}/{nomination_id}", + f"bot/nominations/{nomination_id}", json={"actor": actor.id, "reason": reason} ) - await self.fetch_user_cache() # Update cache + await self.refresh_cache() # Update cache await ctx.send(":white_check_mark: Successfully updated nomination reason.") @nomination_edit_group.command(name='end_reason') @@ -300,14 +444,14 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): async def edit_end_reason_command(self, ctx: Context, nomination_id: int, *, reason: str) -> None: """Edits the unnominate reason for the nomination with the given `id`.""" if len(reason) > REASON_MAX_CHARS: - await ctx.send(f":x: Maxiumum allowed characters for the end reason is {REASON_MAX_CHARS}.") + await ctx.send(f":x: Maximum allowed characters for the end reason is {REASON_MAX_CHARS}.") return try: - nomination = await self.bot.api_client.get(f"{self.api_endpoint}/{nomination_id}") + nomination = await self.bot.api_client.get(f"bot/nominations/{nomination_id}") except ResponseCodeError as e: if e.response.status == 404: - self.log.trace(f"Nomination API 404: Can't find a nomination with id {nomination_id}") + log.trace(f"Nomination API 404: Can't find a nomination with id {nomination_id}") await ctx.send(f":x: Can't find a nomination with id `{nomination_id}`") return else: @@ -317,13 +461,13 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): await ctx.send(":x: Can't edit the end reason of an active nomination.") return - self.log.trace(f"Changing end reason for nomination with id {nomination_id} to {repr(reason)}") + log.trace(f"Changing end reason for nomination with id {nomination_id} to {repr(reason)}") await self.bot.api_client.patch( - f"{self.api_endpoint}/{nomination_id}", + f"bot/nominations/{nomination_id}", json={"end_reason": reason} ) - await self.fetch_user_cache() # Update cache. + await self.refresh_cache() # Update cache. await ctx.send(":white_check_mark: Updated the end reason of the nomination!") @nomination_group.command(aliases=('mr',)) @@ -338,12 +482,9 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): @has_any_role(*MODERATION_ROLES) async def get_review(self, ctx: Context, user_id: int) -> None: """Get the user's review as a markdown file.""" - review = (await self.reviewer.make_review(user_id))[0] - if review: - file = discord.File(StringIO(review), f"{user_id}_review.md") - await ctx.send(file=file) - else: - await ctx.send(f"There doesn't appear to be an active nomination for {user_id}") + review, _, _ = await self.reviewer.make_review(user_id) + file = discord.File(StringIO(review), f"{user_id}_review.md") + await ctx.send(file=file) @nomination_group.command(aliases=('review',)) @has_any_role(*MODERATION_ROLES) @@ -356,9 +497,9 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): await ctx.message.add_reaction(Emojis.check_mark) @Cog.listener() - async def on_member_ban(self, guild: Guild, user: Union[User, Member]) -> None: + async def on_member_ban(self, guild: Guild, user: MemberOrUser) -> None: """Remove `user` from the talent pool after they are banned.""" - await self.unwatch(user.id, "User was banned.") + await self.end_nomination(user.id, "User was banned.") @Cog.listener() async def on_raw_reaction_add(self, payload: RawReactionActionEvent) -> None: @@ -371,6 +512,9 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): if payload.channel_id != Channels.nomination_voting: return + if payload.user_id == self.bot.user.id: + return + message: PartialMessage = self.bot.get_channel(payload.channel_id).get_partial_message(payload.message_id) emoji = str(payload.emoji) @@ -380,10 +524,10 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): log.info(f"Archiving nomination {message.id}") await self.reviewer.archive_vote(message, emoji == Emojis.incident_actioned) - async def unwatch(self, user_id: int, reason: str) -> bool: + async def end_nomination(self, user_id: int, reason: str) -> bool: """End the active nomination of a user with the given reason and return True on success.""" active_nomination = await self.bot.api_client.get( - self.api_endpoint, + 'bot/nominations', params=ChainMap( {"user__id": str(user_id)}, self.api_default_params, @@ -398,25 +542,26 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): nomination = active_nomination[0] await self.bot.api_client.patch( - f"{self.api_endpoint}/{nomination['id']}", + f"bot/nominations/{nomination['id']}", json={'end_reason': reason, 'active': False} ) - self._remove_user(user_id) - self.reviewer.cancel(user_id) + self.cache.pop(user_id) + if await self.autoreview_enabled(): + self.reviewer.cancel(user_id) return True - def _nomination_to_string(self, nomination_object: dict) -> str: + async def _nomination_to_string(self, nomination_object: dict) -> str: """Creates a string representation of a nomination.""" guild = self.bot.get_guild(Guild.id) entries = [] for site_entry in nomination_object["entries"]: actor_id = site_entry["actor"] - actor = guild.get_member(actor_id) + actor = await get_or_fetch_member(guild, actor_id) reason = site_entry["reason"] or "*None*" - created = time.format_infraction(site_entry["inserted_at"]) + created = time.discord_timestamp(site_entry["inserted_at"]) entries.append( f"Actor: {actor.mention if actor else actor_id}\nCreated: {created}\nReason: {reason}" ) @@ -425,7 +570,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): active = nomination_object["active"] - start_date = time.format_infraction(nomination_object["inserted_at"]) + start_date = time.discord_timestamp(nomination_object["inserted_at"]) if active: lines = textwrap.dedent( f""" @@ -439,7 +584,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): """ ) else: - end_date = time.format_infraction(nomination_object["ended_at"]) + end_date = time.discord_timestamp(nomination_object["ended_at"]) lines = textwrap.dedent( f""" =============== @@ -450,7 +595,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): {entries_string} End date: {end_date} - Unwatch reason: {nomination_object["end_reason"]} + Unnomination reason: {nomination_object["end_reason"]} =============== """ ) diff --git a/bot/exts/recruitment/talentpool/_review.py b/bot/exts/recruitment/talentpool/_review.py index 0cb786e4b..b4d177622 100644 --- a/bot/exts/recruitment/talentpool/_review.py +++ b/bot/exts/recruitment/talentpool/_review.py @@ -1,6 +1,5 @@ import asyncio import contextlib -import logging import random import re import textwrap @@ -9,33 +8,38 @@ from collections import Counter from datetime import datetime, timedelta from typing import List, Optional, Union +import arrow from dateutil.parser import isoparse -from dateutil.relativedelta import relativedelta -from discord import Embed, Emoji, Member, Message, NoMoreItems, PartialMessage, TextChannel +from discord import Embed, Emoji, Member, Message, NoMoreItems, NotFound, PartialMessage, TextChannel from discord.ext.commands import Context from bot.api import ResponseCodeError from bot.bot import Bot from bot.constants import Channels, Colours, Emojis, Guild, Roles +from bot.log import get_logger +from bot.utils import time +from bot.utils.members import get_or_fetch_member from bot.utils.messages import count_unique_users_reaction, pin_no_system_message from bot.utils.scheduling import Scheduler -from bot.utils.time import get_time_delta, humanize_delta, time_since if typing.TYPE_CHECKING: from bot.exts.recruitment.talentpool._cog import TalentPool -log = logging.getLogger(__name__) +log = get_logger(__name__) # Maximum amount of days before an automatic review is posted. MAX_DAYS_IN_POOL = 30 # Maximum amount of characters allowed in a message MAX_MESSAGE_SIZE = 2000 +# Maximum amount of characters allowed in an embed +MAX_EMBED_SIZE = 4000 -# Regex finding the user ID of a user mention -MENTION_RE = re.compile(r"<@!?(\d+?)>") -# Regex matching role pings -ROLE_MENTION_RE = re.compile(r"<@&\d+>") +# Regex for finding the first message of a nomination, and extracting the nominee. +NOMINATION_MESSAGE_REGEX = re.compile( + r"<@!?(\d+)> \(.+#\d{4}\) for Helper!\n\n", + re.MULTILINE +) class Reviewer: @@ -54,10 +58,8 @@ class Reviewer: """Reschedule all active nominations to be reviewed at the appropriate time.""" log.trace("Rescheduling reviews") await self.bot.wait_until_guild_available() - # TODO Once the watch channel is removed, this can be done in a smarter way, e.g create a sync function. - await self._pool.fetch_user_cache() - for user_id, user_data in self._pool.watched_users.items(): + for user_id, user_data in self._pool.cache.items(): if not user_data["reviewed"]: self.schedule_review(user_id) @@ -65,24 +67,24 @@ class Reviewer: """Schedules a single user for review.""" log.trace(f"Scheduling review of user with ID {user_id}") - user_data = self._pool.watched_users.get(user_id) - inserted_at = isoparse(user_data['inserted_at']).replace(tzinfo=None) + user_data = self._pool.cache.get(user_id) + inserted_at = isoparse(user_data['inserted_at']) review_at = inserted_at + timedelta(days=MAX_DAYS_IN_POOL) # If it's over a day overdue, it's probably an old nomination and shouldn't be automatically reviewed. - if datetime.utcnow() - review_at < timedelta(days=1): + if arrow.utcnow() - review_at < timedelta(days=1): self._review_scheduler.schedule_at(review_at, user_id, self.post_review(user_id, update_database=True)) async def post_review(self, user_id: int, update_database: bool) -> None: """Format the review of a user and post it to the nomination voting channel.""" - review, reviewed_emoji = await self.make_review(user_id) - if not review: + review, reviewed_emoji, nominee = await self.make_review(user_id) + if not nominee: return guild = self.bot.get_guild(Guild.id) channel = guild.get_channel(Channels.nomination_voting) - log.trace(f"Posting the review of {user_id}") + log.trace(f"Posting the review of {nominee} ({nominee.id})") messages = await self._bulk_send(channel, review) await pin_no_system_message(messages[0]) @@ -92,32 +94,37 @@ class Reviewer: for reaction in (reviewed_emoji, "\N{THUMBS UP SIGN}", "\N{THUMBS DOWN SIGN}"): await last_message.add_reaction(reaction) + thread = await last_message.create_thread( + name=f"Nomination - {nominee}", + ) + await thread.send(fr"<@&{Roles.mod_team}> <@&{Roles.admins}>") + if update_database: - nomination = self._pool.watched_users.get(user_id) - await self.bot.api_client.patch(f"{self._pool.api_endpoint}/{nomination['id']}", json={"reviewed": True}) + nomination = self._pool.cache.get(user_id) + await self.bot.api_client.patch(f"bot/nominations/{nomination['id']}", json={"reviewed": True}) - async def make_review(self, user_id: int) -> typing.Tuple[str, Optional[Emoji]]: - """Format a generic review of a user and return it with the reviewed emoji.""" + async def make_review(self, user_id: int) -> typing.Tuple[str, Optional[Emoji], Optional[Member]]: + """Format a generic review of a user and return it with the reviewed emoji and the user themselves.""" log.trace(f"Formatting the review of {user_id}") - # Since `watched_users` is a defaultdict, we should take care + # Since `cache` is a defaultdict, we should take care # not to accidentally insert the IDs of users that have no - # active nominated by using the `watched_users.get(user_id)` - # instead of `watched_users[user_id]`. - nomination = self._pool.watched_users.get(user_id) + # active nominated by using the `cache.get(user_id)` + # instead of `cache[user_id]`. + nomination = self._pool.cache.get(user_id) if not nomination: log.trace(f"There doesn't appear to be an active nomination for {user_id}") - return "", None + return f"There doesn't appear to be an active nomination for {user_id}", None, None guild = self.bot.get_guild(Guild.id) - member = guild.get_member(user_id) + nominee = await get_or_fetch_member(guild, user_id) - if not member: + if not nominee: return ( f"I tried to review the user with ID `{user_id}`, but they don't appear to be on the server :pensive:" - ), None + ), None, None - opening = f"<@&{Roles.mod_team}> <@&{Roles.admins}>\n{member.mention} ({member}) for Helper!" + opening = f"{nominee.mention} ({nominee}) for Helper!" current_nominations = "\n\n".join( f"**<@{entry['actor']}>:** {entry['reason'] or '*no reason given*'}" @@ -125,7 +132,7 @@ class Reviewer: ) current_nominations = f"**Nominated by:**\n{current_nominations}" - review_body = await self._construct_review_body(member) + review_body = await self._construct_review_body(nominee) reviewed_emoji = self._random_ducky(guild) vote_request = ( @@ -135,20 +142,20 @@ class Reviewer: ) review = "\n\n".join((opening, current_nominations, review_body, vote_request)) - return review, reviewed_emoji + return review, reviewed_emoji, nominee async def archive_vote(self, message: PartialMessage, passed: bool) -> None: """Archive this vote to #nomination-archive.""" message = await message.fetch() - # We consider the first message in the nomination to contain the two role pings + # We consider the first message in the nomination to contain the user ping, username#discrim, and fixed text messages = [message] - if not len(ROLE_MENTION_RE.findall(message.content)) >= 2: + if not NOMINATION_MESSAGE_REGEX.search(message.content): with contextlib.suppress(NoMoreItems): async for new_message in message.channel.history(before=message.created_at): messages.append(new_message) - if len(ROLE_MENTION_RE.findall(new_message.content)) >= 2: + if NOMINATION_MESSAGE_REGEX.search(new_message.content): break log.debug(f"Found {len(messages)} messages: {', '.join(str(m.id) for m in messages)}") @@ -160,7 +167,7 @@ class Reviewer: content = "".join(parts) # We assume that the first user mentioned is the user that we are voting on - user_id = int(MENTION_RE.search(content).group(1)) + user_id = int(NOMINATION_MESSAGE_REGEX.search(content).group(1)) # Get reaction counts reviewed = await count_unique_users_reaction( @@ -199,7 +206,7 @@ class Reviewer: channel = self.bot.get_channel(Channels.nomination_archive) for number, part in enumerate( - textwrap.wrap(embed_content, width=MAX_MESSAGE_SIZE, replace_whitespace=False, placeholder="") + textwrap.wrap(embed_content, width=MAX_EMBED_SIZE, replace_whitespace=False, placeholder="") ): await channel.send(embed=Embed( title=embed_title if number == 0 else None, @@ -207,8 +214,21 @@ class Reviewer: colour=colour )) + # Thread channel IDs are the same as the message ID of the parent message. + nomination_thread = message.guild.get_thread(message.id) + if not nomination_thread: + try: + nomination_thread = await message.guild.fetch_channel(message.id) + except NotFound: + log.warning(f"Could not find a thread linked to {message.channel.id}-{message.id}") + return + for message_ in messages: - await message_.delete() + with contextlib.suppress(NotFound): + await message_.delete() + + with contextlib.suppress(NotFound): + await nomination_thread.edit(archived=True) async def _construct_review_body(self, member: Member) -> str: """Formats the body of the nomination, with details of activity, infractions, and previous nominations.""" @@ -253,9 +273,9 @@ class Reviewer: last_channel = user_activity["top_channel_activity"][-1] channels += f", and {last_channel[1]} in {last_channel[0]}" - time_on_server = humanize_delta(relativedelta(datetime.utcnow(), member.joined_at), max_units=2) + joined_at_formatted = time.format_relative(member.joined_at) review = ( - f"{member.name} has been on the server for **{time_on_server}**" + f"{member.name} joined the server **{joined_at_formatted}**" f" and has **{messages} messages**{channels}." ) @@ -301,7 +321,7 @@ class Reviewer: infractions += ", with the last infraction issued " # Infractions were ordered by time since insertion descending. - infractions += get_time_delta(infraction_list[0]['inserted_at']) + infractions += time.format_relative(infraction_list[0]['inserted_at']) return f"They have {infractions}." @@ -329,7 +349,7 @@ class Reviewer: """ log.trace(f"Fetching the nomination history data for {member.id}'s review") history = await self.bot.api_client.get( - self._pool.api_endpoint, + "bot/nominations", params={ "user__id": str(member.id), "active": "false", @@ -345,7 +365,7 @@ class Reviewer: nomination_times = f"{num_entries} times" if num_entries > 1 else "once" rejection_times = f"{len(history)} times" if len(history) > 1 else "once" - end_time = time_since(isoparse(history[0]['ended_at']).replace(tzinfo=None), max_units=2) + end_time = time.format_relative(history[0]['ended_at']) review = ( f"They were nominated **{nomination_times}** before" @@ -357,10 +377,10 @@ class Reviewer: @staticmethod def _random_ducky(guild: Guild) -> Union[Emoji, str]: - """Picks a random ducky emoji. If no duckies found returns :eyes:.""" + """Picks a random ducky emoji. If no duckies found returns 👀.""" duckies = [emoji for emoji in guild.emojis if emoji.name.startswith("ducky")] if not duckies: - return ":eyes:" + return "\N{EYES}" return random.choice(duckies) @staticmethod @@ -387,18 +407,18 @@ class Reviewer: Returns True if the user was successfully marked as reviewed, False otherwise. """ log.trace(f"Updating user {user_id} as reviewed") - await self._pool.fetch_user_cache() - if user_id not in self._pool.watched_users: + await self._pool.refresh_cache() + if user_id not in self._pool.cache: log.trace(f"Can't find a nominated user with id {user_id}") await ctx.send(f":x: Can't find a currently nominated user with id `{user_id}`") return False - nomination = self._pool.watched_users.get(user_id) + nomination = self._pool.cache.get(user_id) if nomination["reviewed"]: await ctx.send(":x: This nomination was already reviewed, but here's a cookie :cookie:") return False - await self.bot.api_client.patch(f"{self._pool.api_endpoint}/{nomination['id']}", json={"reviewed": True}) + await self.bot.api_client.patch(f"bot/nominations/{nomination['id']}", json={"reviewed": True}) if user_id in self._review_scheduler: self._review_scheduler.cancel(user_id) diff --git a/bot/exts/utils/bot.py b/bot/exts/utils/bot.py index d84709616..8f0094bc9 100644 --- a/bot/exts/utils/bot.py +++ b/bot/exts/utils/bot.py @@ -1,4 +1,3 @@ -import logging from typing import Optional from discord import Embed, TextChannel @@ -6,8 +5,9 @@ from discord.ext.commands import Cog, Context, command, group, has_any_role from bot.bot import Bot from bot.constants import Guild, MODERATION_ROLES, URLs +from bot.log import get_logger -log = logging.getLogger(__name__) +log = get_logger(__name__) class BotCog(Cog, name="Bot"): diff --git a/bot/exts/utils/clean.py b/bot/exts/utils/clean.py deleted file mode 100644 index cb662e852..000000000 --- a/bot/exts/utils/clean.py +++ /dev/null @@ -1,276 +0,0 @@ -import logging -import random -import re -from typing import Iterable, Optional - -from discord import Colour, Embed, Message, TextChannel, User, errors -from discord.ext import commands -from discord.ext.commands import Cog, Context, group, has_any_role - -from bot.bot import Bot -from bot.constants import ( - Channels, CleanMessages, Colours, Event, Icons, MODERATION_ROLES, NEGATIVE_REPLIES -) -from bot.exts.moderation.modlog import ModLog - -log = logging.getLogger(__name__) - - -class Clean(Cog): - """ - A cog that allows messages to be deleted in bulk, while applying various filters. - - You can delete messages sent by a specific user, messages sent by bots, all messages, or messages that match a - specific regular expression. - - The deleted messages are saved and uploaded to the database via an API endpoint, and a URL is returned which can be - used to view the messages in the Discord dark theme style. - """ - - def __init__(self, bot: Bot): - self.bot = bot - self.cleaning = False - - @property - def mod_log(self) -> ModLog: - """Get currently loaded ModLog cog instance.""" - return self.bot.get_cog("ModLog") - - async def _clean_messages( - self, - amount: int, - ctx: Context, - channels: Iterable[TextChannel], - bots_only: bool = False, - user: User = None, - regex: Optional[str] = None, - until_message: Optional[Message] = None, - ) -> None: - """A helper function that does the actual message cleaning.""" - def predicate_bots_only(message: Message) -> bool: - """Return True if the message was sent by a bot.""" - return message.author.bot - - def predicate_specific_user(message: Message) -> bool: - """Return True if the message was sent by the user provided in the _clean_messages call.""" - return message.author == user - - def predicate_regex(message: Message) -> bool: - """Check if the regex provided in _clean_messages matches the message content or any embed attributes.""" - content = [message.content] - - # Add the content for all embed attributes - for embed in message.embeds: - content.append(embed.title) - content.append(embed.description) - content.append(embed.footer.text) - content.append(embed.author.name) - for field in embed.fields: - content.append(field.name) - content.append(field.value) - - # Get rid of empty attributes and turn it into a string - content = [attr for attr in content if attr] - content = "\n".join(content) - - # Now let's see if there's a regex match - if not content: - return False - else: - return bool(re.search(regex.lower(), content.lower())) - - # Is this an acceptable amount of messages to clean? - if amount > CleanMessages.message_limit: - embed = Embed( - color=Colour(Colours.soft_red), - title=random.choice(NEGATIVE_REPLIES), - description=f"You cannot clean more than {CleanMessages.message_limit} messages." - ) - await ctx.send(embed=embed) - return - - # Are we already performing a clean? - if self.cleaning: - embed = Embed( - color=Colour(Colours.soft_red), - title=random.choice(NEGATIVE_REPLIES), - description="Please wait for the currently ongoing clean operation to complete." - ) - await ctx.send(embed=embed) - return - - # Set up the correct predicate - if bots_only: - predicate = predicate_bots_only # Delete messages from bots - elif user: - predicate = predicate_specific_user # Delete messages from specific user - elif regex: - predicate = predicate_regex # Delete messages that match regex - else: - predicate = None # Delete all messages - - # Default to using the invoking context's channel - if not channels: - channels = [ctx.channel] - - # Delete the invocation first - self.mod_log.ignore(Event.message_delete, ctx.message.id) - try: - await ctx.message.delete() - except errors.NotFound: - # Invocation message has already been deleted - log.info("Tried to delete invocation message, but it was already deleted.") - - messages = [] - message_ids = [] - self.cleaning = True - - # Find the IDs of the messages to delete. IDs are needed in order to ignore mod log events. - for channel in channels: - async for message in channel.history(limit=amount): - - # If at any point the cancel command is invoked, we should stop. - if not self.cleaning: - return - - # If we are looking for specific message. - if until_message: - - # we could use ID's here however in case if the message we are looking for gets deleted, - # we won't have a way to figure that out thus checking for datetime should be more reliable - if message.created_at < until_message.created_at: - # means we have found the message until which we were supposed to be deleting. - break - - # Since we will be using `delete_messages` method of a TextChannel and we need message objects to - # use it as well as to send logs we will start appending messages here instead adding them from - # purge. - messages.append(message) - - # If the message passes predicate, let's save it. - if predicate is None or predicate(message): - message_ids.append(message.id) - - self.cleaning = False - - # Now let's delete the actual messages with purge. - self.mod_log.ignore(Event.message_delete, *message_ids) - for channel in channels: - if until_message: - for i in range(0, len(messages), 100): - # while purge automatically handles the amount of messages - # delete_messages only allows for up to 100 messages at once - # thus we need to paginate the amount to always be <= 100 - await channel.delete_messages(messages[i:i + 100]) - else: - messages += await channel.purge(limit=amount, check=predicate) - - # Reverse the list to restore chronological order - if messages: - messages = reversed(messages) - log_url = await self.mod_log.upload_log(messages, ctx.author.id) - else: - # Can't build an embed, nothing to clean! - embed = Embed( - color=Colour(Colours.soft_red), - description="No matching messages could be found." - ) - await ctx.send(embed=embed, delete_after=10) - return - - # Build the embed and send it - target_channels = ", ".join(channel.mention for channel in channels) - - message = ( - f"**{len(message_ids)}** messages deleted in {target_channels} by " - f"{ctx.author.mention}\n\n" - f"A log of the deleted messages can be found [here]({log_url})." - ) - - await self.mod_log.send_log_message( - icon_url=Icons.message_bulk_delete, - colour=Colour(Colours.soft_red), - title="Bulk message delete", - text=message, - channel_id=Channels.mod_log, - ) - - @group(invoke_without_command=True, name="clean", aliases=["clear", "purge"]) - @has_any_role(*MODERATION_ROLES) - async def clean_group(self, ctx: Context) -> None: - """Commands for cleaning messages in channels.""" - await ctx.send_help(ctx.command) - - @clean_group.command(name="user", aliases=["users"]) - @has_any_role(*MODERATION_ROLES) - async def clean_user( - self, - ctx: Context, - user: User, - amount: Optional[int] = 10, - channels: commands.Greedy[TextChannel] = None - ) -> None: - """Delete messages posted by the provided user, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, user=user, channels=channels) - - @clean_group.command(name="all", aliases=["everything"]) - @has_any_role(*MODERATION_ROLES) - async def clean_all( - self, - ctx: Context, - amount: Optional[int] = 10, - channels: commands.Greedy[TextChannel] = None - ) -> None: - """Delete all messages, regardless of poster, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, channels=channels) - - @clean_group.command(name="bots", aliases=["bot"]) - @has_any_role(*MODERATION_ROLES) - async def clean_bots( - self, - ctx: Context, - amount: Optional[int] = 10, - channels: commands.Greedy[TextChannel] = None - ) -> None: - """Delete all messages posted by a bot, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, bots_only=True, channels=channels) - - @clean_group.command(name="regex", aliases=["word", "expression"]) - @has_any_role(*MODERATION_ROLES) - async def clean_regex( - self, - ctx: Context, - regex: str, - amount: Optional[int] = 10, - channels: commands.Greedy[TextChannel] = None - ) -> None: - """Delete all messages that match a certain regex, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, regex=regex, channels=channels) - - @clean_group.command(name="message", aliases=["messages"]) - @has_any_role(*MODERATION_ROLES) - async def clean_message(self, ctx: Context, message: Message) -> None: - """Delete all messages until certain message, stop cleaning after hitting the `message`.""" - await self._clean_messages( - CleanMessages.message_limit, - ctx, - channels=[message.channel], - until_message=message - ) - - @clean_group.command(name="stop", aliases=["cancel", "abort"]) - @has_any_role(*MODERATION_ROLES) - async def clean_cancel(self, ctx: Context) -> None: - """If there is an ongoing cleaning process, attempt to immediately cancel it.""" - self.cleaning = False - - embed = Embed( - color=Colour.blurple(), - description="Clean interrupted." - ) - await ctx.send(embed=embed, delete_after=10) - - -def setup(bot: Bot) -> None: - """Load the Clean cog.""" - bot.add_cog(Clean(bot)) diff --git a/bot/exts/utils/extensions.py b/bot/exts/utils/extensions.py index 8a1ed98f4..fda1e49e2 100644 --- a/bot/exts/utils/extensions.py +++ b/bot/exts/utils/extensions.py @@ -1,5 +1,4 @@ import functools -import logging import typing as t from enum import Enum @@ -10,10 +9,12 @@ from discord.ext.commands import Context, group from bot import exts from bot.bot import Bot from bot.constants import Emojis, MODERATION_ROLES, Roles, URLs +from bot.converters import Extension +from bot.log import get_logger from bot.pagination import LinePaginator -from bot.utils.extensions import EXTENSIONS, unqualify +from bot.utils.extensions import EXTENSIONS -log = logging.getLogger(__name__) +log = get_logger(__name__) UNLOAD_BLACKLIST = {f"{exts.__name__}.utils.extensions", f"{exts.__name__}.moderation.modlog"} @@ -29,51 +30,13 @@ class Action(Enum): RELOAD = functools.partial(Bot.reload_extension) -class Extension(commands.Converter): - """ - Fully qualify the name of an extension and ensure it exists. - - The * and ** values bypass this when used with the reload command. - """ - - async def convert(self, ctx: Context, argument: str) -> str: - """Fully qualify the name of an extension and ensure it exists.""" - # Special values to reload all extensions - if argument == "*" or argument == "**": - return argument - - argument = argument.lower() - - if argument in EXTENSIONS: - return argument - elif (qualified_arg := f"{exts.__name__}.{argument}") in EXTENSIONS: - return qualified_arg - - matches = [] - for ext in EXTENSIONS: - if argument == unqualify(ext): - matches.append(ext) - - if len(matches) > 1: - matches.sort() - names = "\n".join(matches) - raise commands.BadArgument( - f":x: `{argument}` is an ambiguous extension name. " - f"Please use one of the following fully-qualified names.```\n{names}```" - ) - elif matches: - return matches[0] - else: - raise commands.BadArgument(f":x: Could not find the extension `{argument}`.") - - class Extensions(commands.Cog): """Extension management commands.""" def __init__(self, bot: Bot): self.bot = bot - @group(name="extensions", aliases=("ext", "exts", "c", "cogs"), invoke_without_command=True) + @group(name="extensions", aliases=("ext", "exts", "c", "cog", "cogs"), invoke_without_command=True) async def extensions_group(self, ctx: Context) -> None: """Load, unload, reload, and list loaded extensions.""" await ctx.send_help(ctx.command) @@ -150,7 +113,7 @@ class Extensions(commands.Cog): Grey indicates that the extension is unloaded. Green indicates that the extension is currently loaded. """ - embed = Embed(colour=Colour.blurple()) + embed = Embed(colour=Colour.og_blurple()) embed.set_author( name="Extensions List", url=URLs.github_bot_repo, diff --git a/bot/exts/utils/internal.py b/bot/exts/utils/internal.py index 6f2da3131..e7113c09c 100644 --- a/bot/exts/utils/internal.py +++ b/bot/exts/utils/internal.py @@ -1,23 +1,23 @@ import contextlib import inspect -import logging import pprint import re import textwrap import traceback from collections import Counter -from datetime import datetime from io import StringIO from typing import Any, Optional, Tuple +import arrow import discord -from discord.ext.commands import Cog, Context, group, has_any_role +from discord.ext.commands import Cog, Context, group, has_any_role, is_owner from bot.bot import Bot -from bot.constants import Roles +from bot.constants import DEBUG_MODE, Roles +from bot.log import get_logger from bot.utils import find_nth_occurrence, send_to_paste_service -log = logging.getLogger(__name__) +log = get_logger(__name__) class Internal(Cog): @@ -29,16 +29,18 @@ class Internal(Cog): self.ln = 0 self.stdout = StringIO() - self.socket_since = datetime.utcnow() + self.socket_since = arrow.utcnow() self.socket_event_total = 0 self.socket_events = Counter() + if DEBUG_MODE: + self.eval.add_check(is_owner().predicate) + @Cog.listener() - async def on_socket_response(self, msg: dict) -> None: + async def on_socket_event_type(self, event_type: str) -> None: """When a websocket event is received, increase our counters.""" - if event_type := msg.get("t"): - self.socket_event_total += 1 - self.socket_events[event_type] += 1 + self.socket_event_total += 1 + self.socket_events[event_type] += 1 def _format(self, inp: str, out: Any) -> Tuple[str, Optional[discord.Embed]]: """Format the eval output into a string & attempt to format it into an Embed.""" @@ -234,14 +236,14 @@ async def func(): # (None,) -> Any @has_any_role(Roles.admins, Roles.owners, Roles.core_developers) async def socketstats(self, ctx: Context) -> None: """Fetch information on the socket events received from Discord.""" - running_s = (datetime.utcnow() - self.socket_since).total_seconds() + running_s = (arrow.utcnow() - self.socket_since).total_seconds() per_s = self.socket_event_total / running_s stats_embed = discord.Embed( title="WebSocket statistics", description=f"Receiving {per_s:0.2f} events per second.", - color=discord.Color.blurple() + color=discord.Color.og_blurple() ) for event_type, count in self.socket_events.most_common(25): diff --git a/bot/exts/utils/jams.py b/bot/exts/utils/jams.py deleted file mode 100644 index 87ae847f6..000000000 --- a/bot/exts/utils/jams.py +++ /dev/null @@ -1,176 +0,0 @@ -import csv -import logging -import typing as t -from collections import defaultdict - -import discord -from discord.ext import commands - -from bot.bot import Bot -from bot.constants import Categories, Channels, Emojis, Roles - -log = logging.getLogger(__name__) - -MAX_CHANNELS = 50 -CATEGORY_NAME = "Code Jam" -TEAM_LEADERS_COLOUR = 0x11806a - - -class CodeJams(commands.Cog): - """Manages the code-jam related parts of our server.""" - - def __init__(self, bot: Bot): - self.bot = bot - - @commands.group() - @commands.has_any_role(Roles.admins) - async def codejam(self, ctx: commands.Context) -> None: - """A Group of commands for managing Code Jams.""" - if ctx.invoked_subcommand is None: - await ctx.send_help(ctx.command) - - @codejam.command() - async def create(self, ctx: commands.Context, csv_file: t.Optional[str]) -> None: - """ - Create code-jam teams from a CSV file or a link to one, specifying the team names, leaders and members. - - The CSV file must have 3 columns: 'Team Name', 'Team Member Discord ID', and 'Team Leader'. - - This will create the text channels for the teams, and give the team leaders their roles. - """ - async with ctx.typing(): - if csv_file: - async with self.bot.http_session.get(csv_file) as response: - if response.status != 200: - await ctx.send(f"Got a bad response from the URL: {response.status}") - return - - csv_file = await response.text() - - elif ctx.message.attachments: - csv_file = (await ctx.message.attachments[0].read()).decode("utf8") - else: - raise commands.BadArgument("You must include either a CSV file or a link to one.") - - teams = defaultdict(list) - reader = csv.DictReader(csv_file.splitlines()) - - for row in reader: - member = ctx.guild.get_member(int(row["Team Member Discord ID"])) - - if member is None: - log.trace(f"Got an invalid member ID: {row['Team Member Discord ID']}") - continue - - teams[row["Team Name"]].append((member, row["Team Leader"].upper() == "Y")) - - team_leaders = await ctx.guild.create_role(name="Code Jam Team Leaders", colour=TEAM_LEADERS_COLOUR) - - for team_name, members in teams.items(): - await self.create_team_channel(ctx.guild, team_name, members, team_leaders) - - await self.create_team_leader_channel(ctx.guild, team_leaders) - await ctx.send(f"{Emojis.check_mark} Created Code Jam with {len(teams)} teams.") - - async def get_category(self, guild: discord.Guild) -> discord.CategoryChannel: - """ - Return a code jam category. - - If all categories are full or none exist, create a new category. - """ - for category in guild.categories: - if category.name == CATEGORY_NAME and len(category.channels) < MAX_CHANNELS: - return category - - return await self.create_category(guild) - - async def create_category(self, guild: discord.Guild) -> discord.CategoryChannel: - """Create a new code jam category and return it.""" - log.info("Creating a new code jam category.") - - category_overwrites = { - guild.default_role: discord.PermissionOverwrite(read_messages=False), - guild.me: discord.PermissionOverwrite(read_messages=True) - } - - category = await guild.create_category_channel( - CATEGORY_NAME, - overwrites=category_overwrites, - reason="It's code jam time!" - ) - - await self.send_status_update( - guild, f"Created a new category with the ID {category.id} for this Code Jam's team channels." - ) - - return category - - @staticmethod - def get_overwrites( - members: list[tuple[discord.Member, bool]], - guild: discord.Guild, - ) -> dict[t.Union[discord.Member, discord.Role], discord.PermissionOverwrite]: - """Get code jam team channels permission overwrites.""" - team_channel_overwrites = { - guild.default_role: discord.PermissionOverwrite(read_messages=False), - guild.get_role(Roles.code_jam_event_team): discord.PermissionOverwrite(read_messages=True) - } - - for member, _ in members: - team_channel_overwrites[member] = discord.PermissionOverwrite( - read_messages=True - ) - - return team_channel_overwrites - - async def create_team_channel( - self, - guild: discord.Guild, - team_name: str, - members: list[tuple[discord.Member, bool]], - team_leaders: discord.Role - ) -> None: - """Create the team's text channel.""" - await self.add_team_leader_roles(members, team_leaders) - - # Get permission overwrites and category - team_channel_overwrites = self.get_overwrites(members, guild) - code_jam_category = await self.get_category(guild) - - # Create a text channel for the team - await code_jam_category.create_text_channel( - team_name, - overwrites=team_channel_overwrites, - ) - - async def create_team_leader_channel(self, guild: discord.Guild, team_leaders: discord.Role) -> None: - """Create the Team Leader Chat channel for the Code Jam team leaders.""" - category: discord.CategoryChannel = guild.get_channel(Categories.summer_code_jam) - - team_leaders_chat = await category.create_text_channel( - name="team-leaders-chat", - overwrites={ - guild.default_role: discord.PermissionOverwrite(read_messages=False), - team_leaders: discord.PermissionOverwrite(read_messages=True) - } - ) - - await self.send_status_update(guild, f"Created {team_leaders_chat.mention} in the {category} category.") - - async def send_status_update(self, guild: discord.Guild, message: str) -> None: - """Inform the events lead with a status update when the command is ran.""" - channel: discord.TextChannel = guild.get_channel(Channels.code_jam_planning) - - await channel.send(f"<@&{Roles.events_lead}>\n\n{message}") - - @staticmethod - async def add_team_leader_roles(members: list[tuple[discord.Member, bool]], team_leaders: discord.Role) -> None: - """Assign team leader role, the jammer role and their team role.""" - for member, is_leader in members: - if is_leader: - await member.add_roles(team_leaders) - - -def setup(bot: Bot) -> None: - """Load the CodeJams cog.""" - bot.add_cog(CodeJams(bot)) diff --git a/bot/exts/utils/ping.py b/bot/exts/utils/ping.py index c6d7bd900..9fb5b7b8f 100644 --- a/bot/exts/utils/ping.py +++ b/bot/exts/utils/ping.py @@ -1,11 +1,10 @@ -from datetime import datetime - +import arrow from aiohttp import client_exceptions from discord import Embed from discord.ext import commands from bot.bot import Bot -from bot.constants import Channels, STAFF_ROLES, URLs +from bot.constants import Channels, STAFF_PARTNERS_COMMUNITY_ROLES, URLs from bot.decorators import in_whitelist DESCRIPTIONS = ( @@ -23,7 +22,7 @@ class Latency(commands.Cog): self.bot = bot @commands.command() - @in_whitelist(channels=(Channels.bot_commands,), roles=STAFF_ROLES) + @in_whitelist(channels=(Channels.bot_commands,), roles=STAFF_PARTNERS_COMMUNITY_ROLES) async def ping(self, ctx: commands.Context) -> None: """ Gets different measures of latency within the bot. @@ -32,7 +31,7 @@ class Latency(commands.Cog): """ # datetime.datetime objects do not have the "milliseconds" attribute. # It must be converted to seconds before converting to milliseconds. - bot_ping = (datetime.utcnow() - ctx.message.created_at).total_seconds() * 1000 + bot_ping = (arrow.utcnow() - ctx.message.created_at).total_seconds() * 1000 if bot_ping <= 0: bot_ping = "Your clock is out of sync, could not calculate ping." else: diff --git a/bot/exts/utils/reminders.py b/bot/exts/utils/reminders.py index 6c21920a1..ad82d49c9 100644 --- a/bot/exts/utils/reminders.py +++ b/bot/exts/utils/reminders.py @@ -1,33 +1,33 @@ -import asyncio -import logging import random import textwrap import typing as t -from datetime import datetime, timedelta +from datetime import datetime, timezone from operator import itemgetter import discord from dateutil.parser import isoparse -from dateutil.relativedelta import relativedelta from discord.ext.commands import Cog, Context, Greedy, group from bot.bot import Bot -from bot.constants import Guild, Icons, MODERATION_ROLES, POSITIVE_REPLIES, Roles, STAFF_ROLES -from bot.converters import Duration +from bot.constants import Guild, Icons, MODERATION_ROLES, POSITIVE_REPLIES, Roles, STAFF_PARTNERS_COMMUNITY_ROLES +from bot.converters import Duration, UnambiguousUser +from bot.log import get_logger from bot.pagination import LinePaginator +from bot.utils import scheduling, time from bot.utils.checks import has_any_role_check, has_no_roles_check from bot.utils.lock import lock_arg +from bot.utils.members import get_or_fetch_member from bot.utils.messages import send_denial from bot.utils.scheduling import Scheduler -from bot.utils.time import humanize_delta -log = logging.getLogger(__name__) +log = get_logger(__name__) LOCK_NAMESPACE = "reminder" WHITELISTED_CHANNELS = Guild.reminder_whitelist MAXIMUM_REMINDERS = 5 Mentionable = t.Union[discord.Member, discord.Role] +ReminderMention = t.Union[UnambiguousUser, discord.Role] class Reminders(Cog): @@ -37,7 +37,7 @@ class Reminders(Cog): self.bot = bot self.scheduler = Scheduler(self.__class__.__name__) - self.bot.loop.create_task(self.reschedule_reminders()) + scheduling.create_task(self.reschedule_reminders(), event_loop=self.bot.loop) def cog_unload(self) -> None: """Cancel scheduled tasks.""" @@ -51,43 +51,40 @@ class Reminders(Cog): params={'active': 'true'} ) - now = datetime.utcnow() + now = datetime.now(timezone.utc) for reminder in response: is_valid, *_ = self.ensure_valid_reminder(reminder) if not is_valid: continue - remind_at = isoparse(reminder['expiration']).replace(tzinfo=None) + remind_at = isoparse(reminder['expiration']) # If the reminder is already overdue ... if remind_at < now: - late = relativedelta(now, remind_at) - await self.send_reminder(reminder, late) + await self.send_reminder(reminder, remind_at) else: self.schedule_reminder(reminder) - def ensure_valid_reminder(self, reminder: dict) -> t.Tuple[bool, discord.User, discord.TextChannel]: - """Ensure reminder author and channel can be fetched otherwise delete the reminder.""" - user = self.bot.get_user(reminder['author']) + def ensure_valid_reminder(self, reminder: dict) -> t.Tuple[bool, discord.TextChannel]: + """Ensure reminder channel can be fetched otherwise delete the reminder.""" channel = self.bot.get_channel(reminder['channel_id']) is_valid = True - if not user or not channel: + if not channel: is_valid = False log.info( f"Reminder {reminder['id']} invalid: " - f"User {reminder['author']}={user}, Channel {reminder['channel_id']}={channel}." + f"Channel {reminder['channel_id']}={channel}." ) - asyncio.create_task(self.bot.api_client.delete(f"bot/reminders/{reminder['id']}")) + scheduling.create_task(self.bot.api_client.delete(f"bot/reminders/{reminder['id']}")) - return is_valid, user, channel + return is_valid, channel @staticmethod async def _send_confirmation( ctx: Context, on_success: str, - reminder_id: t.Union[str, int], - delivery_dt: t.Optional[datetime], + reminder_id: t.Union[str, int] ) -> None: """Send an embed confirming the reminder change was made successfully.""" embed = discord.Embed( @@ -98,11 +95,6 @@ class Reminders(Cog): footer_str = f"ID: {reminder_id}" - if delivery_dt: - # Reminder deletion will have a `None` `delivery_dt` - footer_str += ', Due' - embed.timestamp = delivery_dt - embed.set_footer(text=footer_str) await ctx.send(embed=embed) @@ -118,10 +110,10 @@ class Reminders(Cog): If mentions aren't allowed, also return the type of mention(s) disallowed. """ - if await has_no_roles_check(ctx, *STAFF_ROLES): + if await has_no_roles_check(ctx, *STAFF_PARTNERS_COMMUNITY_ROLES): return False, "members/roles" elif await has_no_roles_check(ctx, *MODERATION_ROLES): - return all(isinstance(mention, discord.Member) for mention in mentions), "roles" + return all(isinstance(mention, (discord.User, discord.Member)) for mention in mentions), "roles" else: return True, "" @@ -140,16 +132,17 @@ class Reminders(Cog): await send_denial(ctx, f"You can't mention other {disallowed_mentions} in your reminder!") return False - def get_mentionables(self, mention_ids: t.List[int]) -> t.Iterator[Mentionable]: + async def get_mentionables(self, mention_ids: t.List[int]) -> t.Iterator[Mentionable]: """Converts Role and Member ids to their corresponding objects if possible.""" guild = self.bot.get_guild(Guild.id) for mention_id in mention_ids: - if (mentionable := (guild.get_member(mention_id) or guild.get_role(mention_id))): + member = await get_or_fetch_member(guild, mention_id) + if mentionable := (member or guild.get_role(mention_id)): yield mentionable def schedule_reminder(self, reminder: dict) -> None: """A coroutine which sends the reminder once the time is reached, and cancels the running task.""" - reminder_datetime = isoparse(reminder['expiration']).replace(tzinfo=None) + reminder_datetime = isoparse(reminder['expiration']) self.scheduler.schedule_at(reminder_datetime, reminder["id"], self.send_reminder(reminder)) async def _edit_reminder(self, reminder_id: int, payload: dict) -> dict: @@ -174,59 +167,90 @@ class Reminders(Cog): self.schedule_reminder(reminder) @lock_arg(LOCK_NAMESPACE, "reminder", itemgetter("id"), raise_error=True) - async def send_reminder(self, reminder: dict, late: relativedelta = None) -> None: + async def send_reminder(self, reminder: dict, expected_time: t.Optional[time.Timestamp] = None) -> None: """Send the reminder.""" - is_valid, user, channel = self.ensure_valid_reminder(reminder) + is_valid, channel = self.ensure_valid_reminder(reminder) if not is_valid: # No need to cancel the task too; it'll simply be done once this coroutine returns. return - embed = discord.Embed() - embed.colour = discord.Colour.blurple() - embed.set_author( - icon_url=Icons.remind_blurple, - name="It has arrived!" - ) - - embed.description = f"Here's your reminder: `{reminder['content']}`." - - if reminder.get("jump_url"): # keep backward compatibility - embed.description += f"\n[Jump back to when you created the reminder]({reminder['jump_url']})" - - if late: + if expected_time: embed.colour = discord.Colour.red() embed.set_author( icon_url=Icons.remind_red, - name=f"Sorry it arrived {humanize_delta(late, max_units=2)} late!" + name="Sorry, your reminder should have arrived earlier!" + ) + else: + embed.colour = discord.Colour.og_blurple() + embed.set_author( + icon_url=Icons.remind_blurple, + name="It has arrived!" ) - additional_mentions = ' '.join( - mentionable.mention for mentionable in self.get_mentionables(reminder["mentions"]) - ) + # Let's not use a codeblock to keep emojis and mentions working. Embeds are safe anyway. + embed.description = f"Here's your reminder: {reminder['content']}" - await channel.send(content=f"{user.mention} {additional_mentions}", embed=embed) + # Here the jump URL is in the format of base_url/guild_id/channel_id/message_id + additional_mentions = ' '.join([ + mentionable.mention async for mentionable in self.get_mentionables(reminder["mentions"]) + ]) + + jump_url = reminder.get("jump_url") + embed.description += f"\n[Jump back to when you created the reminder]({jump_url})" + partial_message = channel.get_partial_message(int(jump_url.split("/")[-1])) + try: + await partial_message.reply(content=f"{additional_mentions}", embed=embed) + except discord.HTTPException as e: + log.info( + f"There was an error when trying to reply to a reminder invocation message, {e}, " + "fall back to using jump_url" + ) + await channel.send(content=f"<@{reminder['author']}> {additional_mentions}", embed=embed) log.debug(f"Deleting reminder #{reminder['id']} (the user has been reminded).") await self.bot.api_client.delete(f"bot/reminders/{reminder['id']}") @group(name="remind", aliases=("reminder", "reminders", "remindme"), invoke_without_command=True) async def remind_group( - self, ctx: Context, mentions: Greedy[Mentionable], expiration: Duration, *, content: str + self, ctx: Context, mentions: Greedy[ReminderMention], expiration: Duration, *, content: t.Optional[str] = None ) -> None: - """Commands for managing your reminders.""" + """ + Commands for managing your reminders. + + The `expiration` duration of `!remind new` supports the following symbols for each unit of time: + - years: `Y`, `y`, `year`, `years` + - months: `m`, `month`, `months` + - weeks: `w`, `W`, `week`, `weeks` + - days: `d`, `D`, `day`, `days` + - hours: `H`, `h`, `hour`, `hours` + - minutes: `M`, `minute`, `minutes` + - seconds: `S`, `s`, `second`, `seconds` + + For example, to set a reminder that expires in 3 days and 1 minute, you can do `!remind new 3d1M Do something`. + """ await self.new_reminder(ctx, mentions=mentions, expiration=expiration, content=content) @remind_group.command(name="new", aliases=("add", "create")) async def new_reminder( - self, ctx: Context, mentions: Greedy[Mentionable], expiration: Duration, *, content: str + self, ctx: Context, mentions: Greedy[ReminderMention], expiration: Duration, *, content: t.Optional[str] = None ) -> None: """ Set yourself a simple reminder. - Expiration is parsed per: http://strftime.org/ + The `expiration` duration supports the following symbols for each unit of time: + - years: `Y`, `y`, `year`, `years` + - months: `m`, `month`, `months` + - weeks: `w`, `W`, `week`, `weeks` + - days: `d`, `D`, `day`, `days` + - hours: `H`, `h`, `hour`, `hours` + - minutes: `M`, `minute`, `minutes` + - seconds: `S`, `s`, `second`, `seconds` + + For example, to set a reminder that expires in 3 days and 1 minute, you can do `!remind new 3d1M Do something`. """ - # If the user is not staff, we need to verify whether or not to make a reminder at all. - if await has_no_roles_check(ctx, *STAFF_ROLES): + # If the user is not staff, partner or part of the python community, + # we need to verify whether or not to make a reminder at all. + if await has_no_roles_check(ctx, *STAFF_PARTNERS_COMMUNITY_ROLES): # If they don't have permission to set a reminder in this channel if ctx.channel.id not in WHITELISTED_CHANNELS: @@ -257,6 +281,20 @@ class Reminders(Cog): mention_ids = [mention.id for mention in mentions] + # If `content` isn't provided then we try to get message content of a replied message + if not content: + if reference := ctx.message.reference: + if isinstance((resolved_message := reference.resolved), discord.Message): + content = resolved_message.content + # If we weren't able to get the content of a replied message + if content is None: + await send_denial(ctx, "Your reminder must have a content and/or reply to a message.") + return + + # If the replied message has no content (e.g. only attachments/embeds) + if content == "": + content = "See referenced message." + # Now we can attempt to actually set the reminder. reminder = await self.bot.api_client.post( 'bot/reminders', @@ -270,9 +308,8 @@ class Reminders(Cog): } ) - now = datetime.utcnow() - timedelta(seconds=1) - humanized_delta = humanize_delta(relativedelta(expiration, now)) - mention_string = f"Your reminder will arrive in {humanized_delta}" + formatted_time = time.discord_timestamp(expiration, time.TimestampFormats.DAY_TIME) + mention_string = f"Your reminder will arrive on {formatted_time}" if mentions: mention_string += f" and will mention {len(mentions)} other(s)" @@ -282,8 +319,7 @@ class Reminders(Cog): await self._send_confirmation( ctx, on_success=mention_string, - reminder_id=reminder["id"], - delivery_dt=expiration, + reminder_id=reminder["id"] ) self.schedule_reminder(reminder) @@ -297,8 +333,6 @@ class Reminders(Cog): params={'author__id': str(ctx.author.id)} ) - now = datetime.utcnow() - # Make a list of tuples so it can be sorted by time. reminders = sorted( ( @@ -312,24 +346,23 @@ class Reminders(Cog): for content, remind_at, id_, mentions in reminders: # Parse and humanize the time, make it pretty :D - remind_datetime = isoparse(remind_at).replace(tzinfo=None) - time = humanize_delta(relativedelta(remind_datetime, now)) + expiry = time.format_relative(remind_at) - mentions = ", ".join( + mentions = ", ".join([ # Both Role and User objects have the `name` attribute - mention.name for mention in self.get_mentionables(mentions) - ) + mention.name async for mention in self.get_mentionables(mentions) + ]) mention_string = f"\n**Mentions:** {mentions}" if mentions else "" text = textwrap.dedent(f""" - **Reminder #{id_}:** *expires in {time}* (ID: {id_}){mention_string} + **Reminder #{id_}:** *expires {expiry}* (ID: {id_}){mention_string} {content} """).strip() lines.append(text) embed = discord.Embed() - embed.colour = discord.Colour.blurple() + embed.colour = discord.Colour.og_blurple() embed.title = f"Reminders for {ctx.author}" # Remind the user that they have no reminders :^) @@ -339,7 +372,7 @@ class Reminders(Cog): return # Construct the embed and paginate it. - embed.colour = discord.Colour.blurple() + embed.colour = discord.Colour.og_blurple() await LinePaginator.paginate( lines, @@ -350,7 +383,20 @@ class Reminders(Cog): @remind_group.group(name="edit", aliases=("change", "modify"), invoke_without_command=True) async def edit_reminder_group(self, ctx: Context) -> None: - """Commands for modifying your current reminders.""" + """ + Commands for modifying your current reminders. + + The `expiration` duration supports the following symbols for each unit of time: + - years: `Y`, `y`, `year`, `years` + - months: `m`, `month`, `months` + - weeks: `w`, `W`, `week`, `weeks` + - days: `d`, `D`, `day`, `days` + - hours: `H`, `h`, `hour`, `hours` + - minutes: `M`, `minute`, `minutes` + - seconds: `S`, `s`, `second`, `seconds` + + For example, to edit a reminder to expire in 3 days and 1 minute, you can do `!remind edit duration 1234 3d1M`. + """ await ctx.send_help(ctx.command) @edit_reminder_group.command(name="duration", aliases=("time",)) @@ -358,7 +404,16 @@ class Reminders(Cog): """ Edit one of your reminder's expiration. - Expiration is parsed per: http://strftime.org/ + The `expiration` duration supports the following symbols for each unit of time: + - years: `Y`, `y`, `year`, `years` + - months: `m`, `month`, `months` + - weeks: `w`, `W`, `week`, `weeks` + - days: `d`, `D`, `day`, `days` + - hours: `H`, `h`, `hour`, `hours` + - minutes: `M`, `minute`, `minutes` + - seconds: `S`, `s`, `second`, `seconds` + + For example, to edit a reminder to expire in 3 days and 1 minute, you can do `!remind edit duration 1234 3d1M`. """ await self.edit_reminder(ctx, id_, {'expiration': expiration.isoformat()}) @@ -368,7 +423,7 @@ class Reminders(Cog): await self.edit_reminder(ctx, id_, {"content": content}) @edit_reminder_group.command(name="mentions", aliases=("pings",)) - async def edit_reminder_mentions(self, ctx: Context, id_: int, mentions: Greedy[Mentionable]) -> None: + async def edit_reminder_mentions(self, ctx: Context, id_: int, mentions: Greedy[ReminderMention]) -> None: """Edit one of your reminder's mentions.""" # Remove duplicate mentions mentions = set(mentions) @@ -388,15 +443,11 @@ class Reminders(Cog): return reminder = await self._edit_reminder(id_, payload) - # Parse the reminder expiration back into a datetime - expiration = isoparse(reminder["expiration"]).replace(tzinfo=None) - # Send a confirmation message to the channel await self._send_confirmation( ctx, on_success="That reminder has been edited successfully!", reminder_id=id_, - delivery_dt=expiration, ) await self._reschedule_reminder(reminder) @@ -413,8 +464,7 @@ class Reminders(Cog): await self._send_confirmation( ctx, on_success="That reminder has been deleted successfully!", - reminder_id=id_, - delivery_dt=None, + reminder_id=id_ ) async def _can_modify(self, ctx: Context, reminder_id: t.Union[str, int]) -> bool: diff --git a/bot/exts/utils/snekbox.py b/bot/exts/utils/snekbox.py index b1f1ba6a8..cc3a2e1d7 100644 --- a/bot/exts/utils/snekbox.py +++ b/bot/exts/utils/snekbox.py @@ -1,40 +1,26 @@ import asyncio import contextlib import datetime -import logging import re import textwrap from functools import partial from signal import Signals from typing import Optional, Tuple -from discord import HTTPException, Message, NotFound, Reaction, User +from botcore.regex import FORMATTED_CODE_REGEX, RAW_CODE_REGEX +from discord import AllowedMentions, HTTPException, Message, NotFound, Reaction, User from discord.ext.commands import Cog, Context, command, guild_only from bot.bot import Bot from bot.constants import Categories, Channels, Roles, URLs from bot.decorators import redirect_output -from bot.utils import send_to_paste_service +from bot.log import get_logger +from bot.utils import scheduling, send_to_paste_service from bot.utils.messages import wait_for_deletion -log = logging.getLogger(__name__) +log = get_logger(__name__) ESCAPE_REGEX = re.compile("[`\u202E\u200B]{3,}") -FORMATTED_CODE_REGEX = re.compile( - r"(?P<delim>(?P<block>```)|``?)" # code delimiter: 1-3 backticks; (?P=block) only matches if it's a block - r"(?(block)(?:(?P<lang>[a-z]+)\n)?)" # if we're in a block, match optional language (only letters plus newline) - r"(?:[ \t]*\n)*" # any blank (empty or tabs/spaces only) lines before the code - r"(?P<code>.*?)" # extract all code inside the markup - r"\s*" # any more whitespace before the end of the code markup - r"(?P=delim)", # match the exact same delimiter from the start again - re.DOTALL | re.IGNORECASE # "." also matches newlines, case insensitive -) -RAW_CODE_REGEX = re.compile( - r"^(?:[ \t]*\n)*" # any blank (empty or tabs/spaces only) lines before the code - r"(?P<code>.*?)" # extract all the rest as code - r"\s*$", # any trailing whitespace until the end of the string - re.DOTALL # "." also matches newlines -) MAX_PASTE_LEN = 10000 @@ -218,8 +204,9 @@ class Snekbox(Cog): if filter_triggered: response = await ctx.send("Attempt to circumvent filter detected. Moderator team has been alerted.") else: - response = await ctx.send(msg) - self.bot.loop.create_task(wait_for_deletion(response, (ctx.author.id,))) + allowed_mentions = AllowedMentions(everyone=False, roles=False, users=[ctx.author]) + response = await ctx.send(msg, allowed_mentions=allowed_mentions) + scheduling.create_task(wait_for_deletion(response, (ctx.author.id,)), event_loop=self.bot.loop) log.info(f"{ctx.author}'s job had a return code of {results['returncode']}") return response diff --git a/bot/exts/utils/thread_bumper.py b/bot/exts/utils/thread_bumper.py new file mode 100644 index 000000000..35057f1fe --- /dev/null +++ b/bot/exts/utils/thread_bumper.py @@ -0,0 +1,147 @@ +import typing as t + +import discord +from async_rediscache import RedisCache +from discord.ext import commands + +from bot import constants +from bot.bot import Bot +from bot.log import get_logger +from bot.pagination import LinePaginator +from bot.utils import channel, scheduling + +log = get_logger(__name__) + + +class ThreadBumper(commands.Cog): + """Cog that allow users to add the current thread to a list that get reopened on archive.""" + + # RedisCache[discord.Thread.id, "sentinel"] + threads_to_bump = RedisCache() + + def __init__(self, bot: Bot): + self.bot = bot + self.init_task = scheduling.create_task(self.ensure_bumped_threads_are_active(), event_loop=self.bot.loop) + + async def unarchive_threads_not_manually_archived(self, threads: list[discord.Thread]) -> None: + """ + Iterate through and unarchive any threads that weren't manually archived recently. + + This is done by extracting the manually archived threads from the audit log. + + Only the last 200 thread_update logs are checked, + as this is assumed to be more than enough to cover bot downtime. + """ + guild = self.bot.get_guild(constants.Guild.id) + + recent_manually_archived_thread_ids = [] + async for thread_update in guild.audit_logs(limit=200, action=discord.AuditLogAction.thread_update): + if getattr(thread_update.after, "archived", False): + recent_manually_archived_thread_ids.append(thread_update.target.id) + + for thread in threads: + if thread.id in recent_manually_archived_thread_ids: + log.info( + "#%s (%d) was manually archived. Leaving archived, and removing from bumped threads.", + thread.name, + thread.id + ) + await self.threads_to_bump.delete(thread.id) + else: + await thread.edit(archived=False) + + async def ensure_bumped_threads_are_active(self) -> None: + """Ensure bumped threads are active, since threads could have been archived while the bot was down.""" + await self.bot.wait_until_guild_available() + + threads_to_maybe_bump = [] + for thread_id, _ in await self.threads_to_bump.items(): + try: + thread = await channel.get_or_fetch_channel(thread_id) + except discord.NotFound: + log.info("Thread %d has been deleted, removing from bumped threads.", thread_id) + await self.threads_to_bump.delete(thread_id) + continue + + if thread.archived: + threads_to_maybe_bump.append(thread) + + await self.unarchive_threads_not_manually_archived(threads_to_maybe_bump) + + @commands.group(name="bump") + async def thread_bump_group(self, ctx: commands.Context) -> None: + """A group of commands to manage the bumping of threads.""" + if not ctx.invoked_subcommand: + await ctx.send_help(ctx.command) + + @thread_bump_group.command(name="add", aliases=("a",)) + async def add_thread_to_bump_list(self, ctx: commands.Context, thread: t.Optional[discord.Thread]) -> None: + """Add a thread to the bump list.""" + await self.init_task + + if not thread: + if isinstance(ctx.channel, discord.Thread): + thread = ctx.channel + else: + raise commands.BadArgument("You must provide a thread, or run this command within a thread.") + + if await self.threads_to_bump.contains(thread.id): + raise commands.BadArgument("This thread is already in the bump list.") + + await self.threads_to_bump.set(thread.id, "sentinel") + await ctx.send(f":ok_hand:{thread.mention} has been added to the bump list.") + + @thread_bump_group.command(name="remove", aliases=("r", "rem", "d", "del", "delete")) + async def remove_thread_from_bump_list(self, ctx: commands.Context, thread: t.Optional[discord.Thread]) -> None: + """Remove a thread from the bump list.""" + await self.init_task + + if not thread: + if isinstance(ctx.channel, discord.Thread): + thread = ctx.channel + else: + raise commands.BadArgument("You must provide a thread, or run this command within a thread.") + + if not await self.threads_to_bump.contains(thread.id): + raise commands.BadArgument("This thread is not in the bump list.") + + await self.threads_to_bump.delete(thread.id) + await ctx.send(f":ok_hand: {thread.mention} has been removed from the bump list.") + + @thread_bump_group.command(name="list", aliases=("get",)) + async def list_all_threads_in_bump_list(self, ctx: commands.Context) -> None: + """List all the threads in the bump list.""" + await self.init_task + + lines = [f"<#{k}>" for k, _ in await self.threads_to_bump.items()] + embed = discord.Embed( + title="Threads in the bump list", + colour=constants.Colours.blue + ) + await LinePaginator.paginate(lines, ctx, embed) + + @commands.Cog.listener() + async def on_thread_update(self, _: discord.Thread, after: discord.Thread) -> None: + """ + Listen for thread updates and check if the thread has been archived. + + If the thread has been archived, and is in the bump list, un-archive it. + """ + await self.init_task + + if not after.archived: + return + + if await self.threads_to_bump.contains(after.id): + await self.unarchive_threads_not_manually_archived([after]) + + async def cog_check(self, ctx: commands.Context) -> bool: + """Only allow staff & partner roles to invoke the commands in this cog.""" + return await commands.has_any_role( + *constants.STAFF_PARTNERS_COMMUNITY_ROLES + ).predicate(ctx) + + +def setup(bot: Bot) -> None: + """Load the ThreadBumper cog.""" + bot.add_cog(ThreadBumper(bot)) diff --git a/bot/exts/utils/utils.py b/bot/exts/utils/utils.py index 3b8564aee..2a074788e 100644 --- a/bot/exts/utils/utils.py +++ b/bot/exts/utils/utils.py @@ -1,5 +1,4 @@ import difflib -import logging import re import unicodedata from typing import Tuple, Union @@ -9,15 +8,14 @@ from discord.ext.commands import BadArgument, Cog, Context, clean_content, comma from discord.utils import snowflake_time from bot.bot import Bot -from bot.constants import Channels, MODERATION_ROLES, Roles, STAFF_ROLES +from bot.constants import Channels, MODERATION_ROLES, Roles, STAFF_PARTNERS_COMMUNITY_ROLES from bot.converters import Snowflake from bot.decorators import in_whitelist +from bot.log import get_logger from bot.pagination import LinePaginator -from bot.utils import messages -from bot.utils.checks import has_no_roles_check -from bot.utils.time import time_since +from bot.utils import messages, time -log = logging.getLogger(__name__) +log = get_logger(__name__) ZEN_OF_PYTHON = """\ Beautiful is better than ugly. @@ -50,20 +48,22 @@ class Utils(Cog): self.bot = bot @command() - @in_whitelist(channels=(Channels.bot_commands,), roles=STAFF_ROLES) + @in_whitelist(channels=(Channels.bot_commands, Channels.discord_bots), roles=STAFF_PARTNERS_COMMUNITY_ROLES) async def charinfo(self, ctx: Context, *, characters: str) -> None: """Shows you information on up to 50 unicode characters.""" match = re.match(r"<(a?):(\w+):(\d+)>", characters) if match: - return await messages.send_denial( + await messages.send_denial( ctx, "**Non-Character Detected**\n" "Only unicode characters can be processed, but a custom Discord emoji " "was found. Please remove it and try again." ) + return if len(characters) > 50: - return await messages.send_denial(ctx, f"Too many characters ({len(characters)}/50)") + await messages.send_denial(ctx, f"Too many characters ({len(characters)}/50)") + return def get_info(char: str) -> Tuple[str, str]: digit = f"{ord(char):x}" @@ -95,7 +95,7 @@ class Utils(Cog): If a string is provided, the line which matches best will be produced. """ embed = Embed( - colour=Colour.blurple(), + colour=Colour.og_blurple(), title="The Zen of Python", description=ZEN_OF_PYTHON ) @@ -157,12 +157,9 @@ class Utils(Cog): await ctx.send(embed=embed) @command(aliases=("snf", "snfl", "sf")) - @in_whitelist(channels=(Channels.bot_commands,), roles=STAFF_ROLES) + @in_whitelist(channels=(Channels.bot_commands,), roles=STAFF_PARTNERS_COMMUNITY_ROLES) async def snowflake(self, ctx: Context, *snowflakes: Snowflake) -> None: """Get Discord snowflake creation time.""" - if len(snowflakes) > 1 and await has_no_roles_check(ctx, *STAFF_ROLES): - raise BadArgument("Cannot process more than one snowflake in one invocation.") - if not snowflakes: raise BadArgument("At least one snowflake must be provided.") @@ -175,7 +172,7 @@ class Utils(Cog): lines = [] for snowflake in snowflakes: created_at = snowflake_time(snowflake) - lines.append(f"**{snowflake}**\nCreated at {created_at} ({time_since(created_at, max_units=3)}).") + lines.append(f"**{snowflake}**\nCreated at {created_at} ({time.format_relative(created_at)}).") await LinePaginator.paginate( lines, diff --git a/bot/log.py b/bot/log.py index 4e20c005e..100cd06f6 100644 --- a/bot/log.py +++ b/bot/log.py @@ -3,6 +3,7 @@ import os import sys from logging import Logger, handlers from pathlib import Path +from typing import Optional, TYPE_CHECKING, cast import coloredlogs import sentry_sdk @@ -14,22 +15,50 @@ from bot import constants TRACE_LEVEL = 5 +if TYPE_CHECKING: + LoggerClass = Logger +else: + LoggerClass = logging.getLoggerClass() + + +class CustomLogger(LoggerClass): + """Custom implementation of the `Logger` class with an added `trace` method.""" + + def trace(self, msg: str, *args, **kwargs) -> None: + """ + Log 'msg % args' with severity 'TRACE'. + + To pass exception information, use the keyword argument exc_info with + a true value, e.g. + + logger.trace("Houston, we have an %s", "interesting problem", exc_info=1) + """ + if self.isEnabledFor(TRACE_LEVEL): + self.log(TRACE_LEVEL, msg, *args, **kwargs) + + +def get_logger(name: Optional[str] = None) -> CustomLogger: + """Utility to make mypy recognise that logger is of type `CustomLogger`.""" + return cast(CustomLogger, logging.getLogger(name)) + + def setup() -> None: """Set up loggers.""" logging.TRACE = TRACE_LEVEL logging.addLevelName(TRACE_LEVEL, "TRACE") - Logger.trace = _monkeypatch_trace + logging.setLoggerClass(CustomLogger) + + root_log = get_logger() format_string = "%(asctime)s | %(name)s | %(levelname)s | %(message)s" log_format = logging.Formatter(format_string) - log_file = Path("logs", "bot.log") - log_file.parent.mkdir(exist_ok=True) - file_handler = handlers.RotatingFileHandler(log_file, maxBytes=5242880, backupCount=7, encoding="utf8") - file_handler.setFormatter(log_format) - - root_log = logging.getLogger() - root_log.addHandler(file_handler) + if constants.FILE_LOGS: + log_file = Path("logs", "bot.log") + log_file.parent.mkdir(exist_ok=True) + file_handler = handlers.RotatingFileHandler(log_file, maxBytes=5242880, backupCount=7, encoding="utf8") + file_handler.setFormatter(log_format) + root_log.addHandler(file_handler) if "COLOREDLOGS_LEVEL_STYLES" not in os.environ: coloredlogs.DEFAULT_LEVEL_STYLES = { @@ -42,16 +71,16 @@ def setup() -> None: if "COLOREDLOGS_LOG_FORMAT" not in os.environ: coloredlogs.DEFAULT_LOG_FORMAT = format_string - coloredlogs.install(level=logging.TRACE, logger=root_log, stream=sys.stdout) + coloredlogs.install(level=TRACE_LEVEL, logger=root_log, stream=sys.stdout) root_log.setLevel(logging.DEBUG if constants.DEBUG_MODE else logging.INFO) - logging.getLogger("discord").setLevel(logging.WARNING) - logging.getLogger("websockets").setLevel(logging.WARNING) - logging.getLogger("chardet").setLevel(logging.WARNING) - logging.getLogger("async_rediscache").setLevel(logging.WARNING) + get_logger("discord").setLevel(logging.WARNING) + get_logger("websockets").setLevel(logging.WARNING) + get_logger("chardet").setLevel(logging.WARNING) + get_logger("async_rediscache").setLevel(logging.WARNING) # Set back to the default of INFO even if asyncio's debug mode is enabled. - logging.getLogger("asyncio").setLevel(logging.INFO) + get_logger("asyncio").setLevel(logging.INFO) _set_trace_loggers() @@ -73,19 +102,6 @@ def setup_sentry() -> None: ) -def _monkeypatch_trace(self: logging.Logger, msg: str, *args, **kwargs) -> None: - """ - Log 'msg % args' with severity 'TRACE'. - - To pass exception information, use the keyword argument exc_info with - a true value, e.g. - - logger.trace("Houston, we have an %s", "interesting problem", exc_info=1) - """ - if self.isEnabledFor(TRACE_LEVEL): - self._log(TRACE_LEVEL, msg, args, **kwargs) - - def _set_trace_loggers() -> None: """ Set loggers to the trace level according to the value from the BOT_TRACE_LOGGERS env var. @@ -101,13 +117,13 @@ def _set_trace_loggers() -> None: level_filter = constants.Bot.trace_loggers if level_filter: if level_filter.startswith("*"): - logging.getLogger().setLevel(logging.TRACE) + get_logger().setLevel(TRACE_LEVEL) elif level_filter.startswith("!"): - logging.getLogger().setLevel(logging.TRACE) + get_logger().setLevel(TRACE_LEVEL) for logger_name in level_filter.strip("!,").split(","): - logging.getLogger(logger_name).setLevel(logging.DEBUG) + get_logger(logger_name).setLevel(logging.DEBUG) else: for logger_name in level_filter.strip(",").split(","): - logging.getLogger(logger_name).setLevel(logging.TRACE) + get_logger(logger_name).setLevel(TRACE_LEVEL) diff --git a/bot/monkey_patches.py b/bot/monkey_patches.py new file mode 100644 index 000000000..4840fa454 --- /dev/null +++ b/bot/monkey_patches.py @@ -0,0 +1,76 @@ +import re +from datetime import timedelta + +import arrow +from discord import Forbidden, http +from discord.ext import commands + +from bot.log import get_logger + +log = get_logger(__name__) +MESSAGE_ID_RE = re.compile(r'(?P<message_id>[0-9]{15,20})$') + + +class Command(commands.Command): + """ + A `discord.ext.commands.Command` subclass which supports root aliases. + + A `root_aliases` keyword argument is added, which is a sequence of alias names that will act as + top-level commands rather than being aliases of the command's group. It's stored as an attribute + also named `root_aliases`. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.root_aliases = kwargs.get("root_aliases", []) + + if not isinstance(self.root_aliases, (list, tuple)): + raise TypeError("Root aliases of a command must be a list or a tuple of strings.") + + +def patch_typing() -> None: + """ + Sometimes discord turns off typing events by throwing 403's. + + Handle those issues by patching the trigger_typing method so it ignores 403's in general. + """ + log.debug("Patching send_typing, which should fix things breaking when discord disables typing events. Stay safe!") + + original = http.HTTPClient.send_typing + last_403 = None + + async def honeybadger_type(self, channel_id: int) -> None: # noqa: ANN001 + nonlocal last_403 + if last_403 and (arrow.utcnow() - last_403) < timedelta(minutes=5): + log.warning("Not sending typing event, we got a 403 less than 5 minutes ago.") + return + try: + await original(self, channel_id) + except Forbidden: + last_403 = arrow.utcnow() + log.warning("Got a 403 from typing event!") + pass + + http.HTTPClient.send_typing = honeybadger_type + + +class FixedPartialMessageConverter(commands.PartialMessageConverter): + """ + Make the Message converter infer channelID from the given context if only a messageID is given. + + Discord.py's Message converter is supposed to infer channelID based + on ctx.channel if only a messageID is given. A refactor commit, linked below, + a few weeks before d.py's archival broke this defined behaviour of the converter. + Currently, if only a messageID is given to the converter, it will only find that message + if it's in the bot's cache. + + https://github.com/Rapptz/discord.py/commit/1a4e73d59932cdbe7bf2c281f25e32529fc7ae1f + """ + + @staticmethod + def _get_id_matches(ctx: commands.Context, argument: str) -> tuple[int, int, int]: + """Inserts ctx.channel.id before calling super method if argument is just a messageID.""" + match = MESSAGE_ID_RE.match(argument) + if match: + argument = f"{ctx.channel.id}-{match.group('message_id')}" + return commands.PartialMessageConverter._get_id_matches(ctx, argument) diff --git a/bot/pagination.py b/bot/pagination.py index fbab74021..8f4353eb1 100644 --- a/bot/pagination.py +++ b/bot/pagination.py @@ -1,5 +1,4 @@ import asyncio -import logging import typing as t from contextlib import suppress from functools import partial @@ -9,6 +8,7 @@ from discord.abc import User from discord.ext.commands import Context, Paginator from bot import constants +from bot.log import get_logger from bot.utils import messages FIRST_EMOJI = "\u23EE" # [:track_previous:] @@ -19,7 +19,7 @@ DELETE_EMOJI = constants.Emojis.trashcan # [:trashcan:] PAGINATION_EMOJI = (FIRST_EMOJI, LEFT_EMOJI, RIGHT_EMOJI, LAST_EMOJI, DELETE_EMOJI) -log = logging.getLogger(__name__) +log = get_logger(__name__) class EmptyPaginatorEmbedError(Exception): @@ -49,8 +49,8 @@ class LinePaginator(Paginator): self, prefix: str = '```', suffix: str = '```', - max_size: int = 2000, - scale_to_size: int = 2000, + max_size: int = 4000, + scale_to_size: int = 4000, max_lines: t.Optional[int] = None, linesep: str = "\n" ) -> None: @@ -59,10 +59,10 @@ class LinePaginator(Paginator): It overrides in order to allow us to configure the maximum number of lines per page. """ - # Embeds that exceed 2048 characters will result in an HTTPException - # (Discord API limit), so we've set a limit of 2000 - if max_size > 2000: - raise ValueError(f"max_size must be <= 2,000 characters. ({max_size} > 2000)") + # Embeds that exceed 4096 characters will result in an HTTPException + # (Discord API limit), so we've set a limit of 4000 + if max_size > 4000: + raise ValueError(f"max_size must be <= 4,000 characters. ({max_size} > 4000)") super().__init__( prefix, @@ -74,8 +74,8 @@ class LinePaginator(Paginator): if scale_to_size < max_size: raise ValueError(f"scale_to_size must be >= max_size. ({scale_to_size} < {max_size})") - if scale_to_size > 2000: - raise ValueError(f"scale_to_size must be <= 2,000 characters. ({scale_to_size} > 2000)") + if scale_to_size > 4000: + raise ValueError(f"scale_to_size must be <= 4,000 characters. ({scale_to_size} > 4000)") self.scale_to_size = scale_to_size - len(suffix) self.max_lines = max_lines @@ -197,7 +197,7 @@ class LinePaginator(Paginator): suffix: str = "", max_lines: t.Optional[int] = None, max_size: int = 500, - scale_to_size: int = 2000, + scale_to_size: int = 4000, empty: bool = True, restrict_to_user: User = None, timeout: int = 300, diff --git a/bot/resources/tags/async-await.md b/bot/resources/tags/async-await.md index ff71ace07..01ab28fe3 100644 --- a/bot/resources/tags/async-await.md +++ b/bot/resources/tags/async-await.md @@ -2,27 +2,26 @@ Python provides the ability to run multiple tasks and coroutines simultaneously with the use of the `asyncio` library, which is included in the Python standard library. -This works by running these coroutines in an event loop, where the context of which coroutine is being run is switches periodically to allow all of them to run, giving the appearance of running at the same time. This is different to using threads or processes in that all code is run in the main process and thread, although it is possible to run coroutines in threads. +This works by running these coroutines in an event loop, where the context of the running coroutine switches periodically to allow all other coroutines to run, thus giving the appearance of running at the same time. This is different to using threads or processes in that all code runs in the main process and thread, although it is possible to run coroutines in other threads. To call an async function we can either `await` it, or run it in an event loop which we get from `asyncio`. -To create a coroutine that can be used with asyncio we need to define a function using the async keyword: +To create a coroutine that can be used with asyncio we need to define a function using the `async` keyword: ```py async def main(): await something_awaitable() ``` -Which means we can call `await something_awaitable()` directly from within the function. If this were a non-async function this would have raised an exception like: `SyntaxError: 'await' outside async function` +Which means we can call `await something_awaitable()` directly from within the function. If this were a non-async function, it would raise the exception `SyntaxError: 'await' outside async function` -To run the top level async function from outside of the event loop we can get an event loop from `asyncio`, and then use that loop to run the function: +To run the top level async function from outside the event loop we need to use [`asyncio.run()`](https://docs.python.org/3/library/asyncio-task.html#asyncio.run), like this: ```py -from asyncio import get_event_loop +import asyncio async def main(): await something_awaitable() -loop = get_event_loop() -loop.run_until_complete(main()) +asyncio.run(main()) ``` -Note that in the `run_until_complete()` where we appear to be calling `main()`, this does not execute the code in `main`, rather it returns a `coroutine` object which is then handled and run by the event loop via `run_until_complete()`. +Note that in the `asyncio.run()`, where we appear to be calling `main()`, this does not execute the code in `main`. Rather, it creates and returns a new `coroutine` object (i.e `main() is not main()`) which is then handled and run by the event loop via `asyncio.run()`. To learn more about asyncio and its use, see the [asyncio documentation](https://docs.python.org/3/library/asyncio.html). diff --git a/bot/resources/tags/blocking.md b/bot/resources/tags/blocking.md index 31d91294c..5554d7eba 100644 --- a/bot/resources/tags/blocking.md +++ b/bot/resources/tags/blocking.md @@ -1,9 +1,7 @@ **Why do we need asynchronous programming?** - Imagine that you're coding a Discord bot and every time somebody uses a command, you need to get some information from a database. But there's a catch: the database servers are acting up today and take a whole 10 seconds to respond. If you do **not** use asynchronous methods, your whole bot will stop running until it gets a response from the database. How do you fix this? Asynchronous programming. **What is asynchronous programming?** - An asynchronous program utilises the `async` and `await` keywords. An asynchronous program pauses what it's doing and does something else whilst it waits for some third-party service to complete whatever it's supposed to do. Any code within an `async` context manager or function marked with the `await` keyword indicates to Python, that whilst this operation is being completed, it can do something else. For example: ```py @@ -14,13 +12,10 @@ import discord async def ping(ctx): await ctx.send("Pong!") ``` - **What does the term "blocking" mean?** - A blocking operation is wherever you do something without `await`ing it. This tells Python that this step must be completed before it can do anything else. Common examples of blocking operations, as simple as they may seem, include: outputting text, adding two numbers and appending an item onto a list. Most common Python libraries have an asynchronous version available to use in asynchronous contexts. **`async` libraries** - The standard async library - `asyncio` Asynchronous web requests - `aiohttp` Talking to PostgreSQL asynchronously - `asyncpg` diff --git a/bot/resources/tags/botvar.md b/bot/resources/tags/botvar.md new file mode 100644 index 000000000..3db6ae7ac --- /dev/null +++ b/bot/resources/tags/botvar.md @@ -0,0 +1,23 @@ +Python allows you to set custom attributes to most objects, like your bot! By storing things as attributes of the bot object, you can access them anywhere you access your bot. In the discord.py library, these custom attributes are commonly known as "bot variables" and can be a lifesaver if your bot is divided into many different files. An example on how to use custom attributes on your bot is shown below: + +```py +bot = commands.Bot(command_prefix="!") +# Set an attribute on our bot +bot.test = "I am accessible everywhere!" + +async def get(ctx: commands.Context): + """A command to get the current value of `test`.""" + # Send what the test attribute is currently set to + await ctx.send(ctx.bot.test) + +async def setval(ctx: commands.Context, *, new_text: str): + """A command to set a new value of `test`.""" + # Here we change the attribute to what was specified in new_text + bot.test = new_text +``` + +This all applies to cogs as well! You can set attributes to `self` as you wish. + +*Be sure **not** to overwrite attributes discord.py uses, like `cogs` or `users`. Name your attributes carefully!* diff --git a/bot/resources/tags/contribute.md b/bot/resources/tags/contribute.md new file mode 100644 index 000000000..50c5cd11f --- /dev/null +++ b/bot/resources/tags/contribute.md @@ -0,0 +1,12 @@ +**Contribute to Python Discord's Open Source Projects** +Looking to contribute to Open Source Projects for the first time? Want to add a feature or fix a bug on the bots on this server? We have on-going projects that people can contribute to, even if you've never contributed to open source before! + +**Projects to Contribute to** +• [Sir Lancebot](https://github.com/python-discord/sir-lancebot) - our fun, beginner-friendly bot +• [Python](https://github.com/python-discord/bot) - our utility & moderation bot +• [Site](https://github.com/python-discord/site) - resources, guides, and more + +**Where to start** +1. Read our [contribution guide](https://pythondiscord.com/pages/guides/pydis-guides/contributing/) +2. Chat with us in <#635950537262759947> if you're ready to jump in or have any questions +3. Open an issue or ask to be assigned to an issue to work on diff --git a/bot/resources/tags/docstring.md b/bot/resources/tags/docstring.md new file mode 100644 index 000000000..20043131e --- /dev/null +++ b/bot/resources/tags/docstring.md @@ -0,0 +1,18 @@ +A [`docstring`](https://docs.python.org/3/glossary.html#term-docstring) is a string - always using triple quotes - that's placed at the top of files, classes and functions. A docstring should contain a clear explanation of what it's describing. You can also include descriptions of the subject's parameter(s) and what it returns, as shown below: +```py +def greet(name: str, age: int) -> str: + """ + Return a string that greets the given person, using their name and age. + + :param name: The name of the person to greet. + :param age: The age of the person to greet. + + :return: The greeting. + """ + return f"Hello {name}, you are {age} years old!" +``` +You can get the docstring by using the [`inspect.getdoc`](https://docs.python.org/3/library/inspect.html#inspect.getdoc) function, from the built-in [`inspect`](https://docs.python.org/3/library/inspect.html) module, or by accessing the `.__doc__` attribute. `inspect.getdoc` is often preferred, as it clears indents from the docstring. + +For the last example, you can print it by doing this: `print(inspect.getdoc(greet))`. + +For more details about what a docstring is and its usage, check out this guide by [Real Python](https://realpython.com/documenting-python-code/#docstrings-background), or the [official docstring specification](https://www.python.org/dev/peps/pep-0257/#what-is-a-docstring). diff --git a/bot/resources/tags/faq.md b/bot/resources/tags/faq.md new file mode 100644 index 000000000..e1c57b3a0 --- /dev/null +++ b/bot/resources/tags/faq.md @@ -0,0 +1,6 @@ +--- +embed: + title: "Frequently asked questions" +--- + +As the largest Python community on Discord, we get hundreds of questions every day. Many of these questions have been asked before. We've compiled a list of the most frequently asked questions along with their answers, which can be found on our [FAQ page](https://www.pythondiscord.com/pages/frequently-asked-questions/). diff --git a/bot/resources/tags/modmail.md b/bot/resources/tags/modmail.md index 412468174..8ac19c8a7 100644 --- a/bot/resources/tags/modmail.md +++ b/bot/resources/tags/modmail.md @@ -6,4 +6,4 @@ It supports attachments, codeblocks, and reactions. As communication happens ove **To use it, simply send a direct message to the bot.** -Should there be an urgent and immediate need for a moderator or admin to look at a channel, feel free to ping the <@&831776746206265384> or <@&267628507062992896> role instead. +Should there be an urgent and immediate need for a moderator to look at a channel, feel free to ping the <@&831776746206265384> role instead. diff --git a/bot/resources/tags/off-topic-names.md b/bot/resources/tags/off-topic-names.md new file mode 100644 index 000000000..5d0614aaa --- /dev/null +++ b/bot/resources/tags/off-topic-names.md @@ -0,0 +1,10 @@ +**Off-topic channels** + +There are three off-topic channels: +• <#291284109232308226> +• <#463035241142026251> +• <#463035268514185226> + +The channel names change every night at midnight UTC and are often fun meta references to jokes or conversations that happened on the server. + +See our [off-topic etiquette](https://pythondiscord.com/pages/resources/guides/off-topic-etiquette/) page for more guidance on how the channels should be used. diff --git a/bot/resources/tags/off-topic.md b/bot/resources/tags/off-topic.md deleted file mode 100644 index 6a864a1d5..000000000 --- a/bot/resources/tags/off-topic.md +++ /dev/null @@ -1,10 +0,0 @@ -**Off-topic channels** - -There are three off-topic channels: -• <#291284109232308226> -• <#463035241142026251> -• <#463035268514185226> - -Their names change randomly every 24 hours, but you can always find them under the `OFF-TOPIC/GENERAL` category in the channel list. - -Please read our [off-topic etiquette](https://pythondiscord.com/pages/resources/guides/off-topic-etiquette/) before participating in conversations. diff --git a/bot/resources/tags/ot.md b/bot/resources/tags/ot.md new file mode 100644 index 000000000..636e59110 --- /dev/null +++ b/bot/resources/tags/ot.md @@ -0,0 +1,3 @@ +**Off-topic channel:** <#463035268514185226> + +Please read our [off-topic etiquette](https://pythondiscord.com/pages/resources/guides/off-topic-etiquette/) before participating in conversations. diff --git a/bot/resources/tags/paste.md b/bot/resources/tags/paste.md index 2ed51def7..8c3c2985d 100644 --- a/bot/resources/tags/paste.md +++ b/bot/resources/tags/paste.md @@ -1,6 +1,6 @@ **Pasting large amounts of code** If your code is too long to fit in a codeblock in discord, you can paste your code here: -https://paste.pydis.com/ +https://paste.pythondiscord.com/ After pasting your code, **save** it by clicking the floppy disk icon in the top right, or by typing `ctrl + S`. After doing that, the URL should **change**. Copy the URL and post it here so others can see it. diff --git a/bot/resources/tags/resources.md b/bot/resources/tags/resources.md new file mode 100644 index 000000000..201e0eb1e --- /dev/null +++ b/bot/resources/tags/resources.md @@ -0,0 +1,6 @@ +--- +embed: + title: "Resources" +--- + +The [Resources page](https://www.pythondiscord.com/resources/) on our website contains a list of hand-selected learning resources that we regularly recommend to both beginners and experts. diff --git a/bot/resources/tags/site.md b/bot/resources/tags/site.md new file mode 100644 index 000000000..376f84742 --- /dev/null +++ b/bot/resources/tags/site.md @@ -0,0 +1,6 @@ +--- +embed: + title: "Python Discord Website" +--- + +[Our official website](https://www.pythondiscord.com/) is an open-source community project created with Python and Django. It contains information about the server itself, lets you sign up for upcoming events, has its own wiki, contains a list of valuable learning resources, and much more. diff --git a/bot/resources/tags/sql-fstring.md b/bot/resources/tags/sql-fstring.md new file mode 100644 index 000000000..94dd870fd --- /dev/null +++ b/bot/resources/tags/sql-fstring.md @@ -0,0 +1,16 @@ +**SQL & f-strings** +Don't use f-strings (`f""`) or other forms of "string interpolation" (`%`, `+`, `.format`) to inject data into a SQL query. It is an endless source of bugs and syntax errors. Additionally, in user-facing applications, it presents a major security risk via SQL injection. + +Your database library should support "query parameters". A query parameter is a placeholder that you put in the SQL query. When the query is executed, you provide data to the database library, and the library inserts the data into the query for you, **safely**. + +For example, the sqlite3 package supports using `?` as a placeholder: +```py +query = "SELECT * FROM stocks WHERE symbol = ?;" +params = ("RHAT",) +db.execute(query, params) +``` +Note: Different database libraries support different placeholder styles, e.g. `%s` and `$1`. Consult your library's documentation for details. + +**See Also** +• [Extended Example with SQLite](https://docs.python.org/3/library/sqlite3.html) (search for "Instead, use the DB-API's parameter substitution") +• [PEP-249](https://www.python.org/dev/peps/pep-0249) - A specification of how database libraries in Python should work diff --git a/bot/resources/tags/string-formatting.md b/bot/resources/tags/string-formatting.md new file mode 100644 index 000000000..707d19c90 --- /dev/null +++ b/bot/resources/tags/string-formatting.md @@ -0,0 +1,24 @@ +**String Formatting Mini-Language** +The String Formatting Language in Python is a powerful way to tailor the display of strings and other data structures. This string formatting mini language works for f-strings and `.format()`. + +Take a look at some of these examples! +```py +>>> my_num = 2134234523 +>>> print(f"{my_num:,}") +2,134,234,523 + +>>> my_smaller_num = -30.0532234 +>>> print(f"{my_smaller_num:=09.2f}") +-00030.05 + +>>> my_str = "Center me!" +>>> print(f"{my_str:-^20}") +-----Center me!----- + +>>> repr_str = "Spam \t Ham" +>>> print(f"{repr_str!r}") +'Spam \t Ham' +``` +**Full Specification & Resources** +[String Formatting Mini Language Specification](https://docs.python.org/3/library/string.html#format-specification-mini-language) +[pyformat.info](https://pyformat.info/) diff --git a/bot/resources/tags/strip-gotcha.md b/bot/resources/tags/strip-gotcha.md new file mode 100644 index 000000000..9ad495cd2 --- /dev/null +++ b/bot/resources/tags/strip-gotcha.md @@ -0,0 +1,17 @@ +When working with `strip`, `lstrip`, or `rstrip`, you might think that this would be the case: +```py +>>> "Monty Python".rstrip(" Python") +"Monty" +``` +While this seems intuitive, it would actually result in: +```py +"M" +``` +as Python interprets the argument to these functions as a set of characters rather than a substring. + +If you want to remove a prefix/suffix from a string, `str.removeprefix` and `str.removesuffix` are recommended and were added in 3.9. +```py +>>> "Monty Python".removesuffix(" Python") +"Monty" +``` +See the documentation of [str.removeprefix](https://docs.python.org/3.10/library/stdtypes.html#str.removeprefix) and [str.removesuffix](https://docs.python.org/3.10/library/stdtypes.html#str.removesuffix) for more information. diff --git a/bot/resources/tags/tools.md b/bot/resources/tags/tools.md new file mode 100644 index 000000000..3cae75552 --- /dev/null +++ b/bot/resources/tags/tools.md @@ -0,0 +1,6 @@ +--- +embed: + title: "Tools" +--- + +The [Tools page](https://www.pythondiscord.com/resources/tools/) on our website contains a couple of the most popular tools for programming in Python. diff --git a/bot/resources/tags/traceback.md b/bot/resources/tags/traceback.md index e770fa86d..e21fa6c6e 100644 --- a/bot/resources/tags/traceback.md +++ b/bot/resources/tags/traceback.md @@ -1,18 +1,15 @@ -Please provide a full traceback to your exception in order for us to identify your issue. +Please provide the full traceback for your exception in order to help us identify your issue. +While the last line of the error message tells us what kind of error you got, +the full traceback will tell us which line, and other critical information to solve your problem. +Please avoid screenshots so we can copy and paste parts of the message. A full traceback could look like: ```py Traceback (most recent call last): - File "tiny", line 3, in - do_something() - File "tiny", line 2, in do_something - a = 6 / 0 -ZeroDivisionError: integer division or modulo by zero + File "my_file.py", line 5, in <module> + add_three("6") + File "my_file.py", line 2, in add_three + a = num + 3 +TypeError: can only concatenate str (not "int") to str ``` -The best way to read your traceback is bottom to top. - -• Identify the exception raised (e.g. ZeroDivisionError) -• Make note of the line number, and navigate there in your program. -• Try to understand why the error occurred. - -To read more about exceptions and errors, please refer to the [PyDis Wiki](https://pythondiscord.com/pages/asking-good-questions/#examining-tracebacks) or the [official Python tutorial.](https://docs.python.org/3.7/tutorial/errors.html) +If the traceback is long, use [our pastebin](https://paste.pythondiscord.com/). diff --git a/bot/resources/tags/venv.md b/bot/resources/tags/venv.md new file mode 100644 index 000000000..a4fc62151 --- /dev/null +++ b/bot/resources/tags/venv.md @@ -0,0 +1,20 @@ +**Virtual Environments** + +Virtual environments are isolated Python environments, which make it easier to keep your system clean and manage dependencies. By default, when activated, only libraries and scripts installed in the virtual environment are accessible, preventing cross-project dependency conflicts, and allowing easy isolation of requirements. + +To create a new virtual environment, you can use the standard library `venv` module: `python3 -m venv .venv` (replace `python3` with `python` or `py` on Windows) + +Then, to activate the new virtual environment: + +**Windows** (PowerShell): `.venv\Scripts\Activate.ps1` +or (Command Prompt): `.venv\Scripts\activate.bat` +**MacOS / Linux** (Bash): `source .venv/bin/activate` + +Packages can then be installed to the virtual environment using `pip`, as normal. + +For more information, take a read of the [documentation](https://docs.python.org/3/library/venv.html). If you run code through your editor, check its documentation on how to make it use your virtual environment. For example, see the [VSCode](https://code.visualstudio.com/docs/python/environments#_select-and-activate-an-environment) or [PyCharm](https://www.jetbrains.com/help/pycharm/creating-virtual-environment.html) docs. + +Tools such as [poetry](https://python-poetry.org/docs/basic-usage/) and [pipenv](https://pipenv.pypa.io/en/latest/) can manage the creation of virtual environments as well as project dependencies, making packaging and installing your project easier. + +**Note:** When using Windows PowerShell, you may need to change the [execution policy](https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_execution_policies) first. This is only required once: +`Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser` diff --git a/bot/resources/tags/windows-path.md b/bot/resources/tags/windows-path.md index da8edf685..b2b0da029 100644 --- a/bot/resources/tags/windows-path.md +++ b/bot/resources/tags/windows-path.md @@ -1,30 +1,17 @@ **PATH on Windows** -If you have installed Python but you forgot to check the *Add Python to PATH* option during the installation you may still be able to access your installation with ease. +If you have installed Python but forgot to check the `Add Python to PATH` option during the installation, you may still be able to access your installation with ease. -If you did not uncheck the option to install the Python launcher then you will find a `py` command on your system. If you want to be able to open your Python installation by running `python` then your best option is to re-install Python. +If you did not uncheck the option to install the `py launcher`, then you'll instead have a `py` command which can be used in the same way. If you want to be able to access your Python installation via the `python` command, then your best option is to re-install Python (remembering to tick the `Add Python to PATH` checkbox). -Otherwise, you can access your install using the `py` command in Command Prompt. Where you may type something with the `python` command like: -``` -C:\Users\Username> python3 my_application_file.py -``` - -You can achieve the same result using the `py` command like this: -``` -C:\Users\Username> py -3 my_application_file.py -``` - -You can pass any options to the Python interpreter after you specify a version, for example, to install a Python module using `pip` you can run: -``` -C:\Users\Username> py -3 -m pip install numpy -``` +You can pass any options to the Python interpreter, e.g. to install the `[numpy](https://pypi.org/project/numpy/)` module from PyPI you can run `py -3 -m pip install numpy` or `python -m pip install numpy`. -You can also access different versions of Python using the version flag, like so: +You can also access different versions of Python using the version flag of the `py` command, like so: ``` C:\Users\Username> py -3.7 ... Python 3.7 starts ... C:\Users\Username> py -3.6 -... Python 3.6 stars ... +... Python 3.6 starts ... C:\Users\Username> py -2 ... Python 2 (any version installed) starts ... ``` diff --git a/bot/resources/tags/xy-problem.md b/bot/resources/tags/xy-problem.md index b77bd27e8..8c508f18c 100644 --- a/bot/resources/tags/xy-problem.md +++ b/bot/resources/tags/xy-problem.md @@ -1,7 +1,7 @@ **xy-problem** -Asking about your attempted solution rather than your actual problem. +The XY problem can be summarised as asking about your attempted solution, rather than your actual problem. Often programmers will get distracted with a potential solution they've come up with, and will try asking for help getting it to work. However, it's possible this solution either wouldn't work as they expect, or there's a much better solution instead. -For more information and examples: http://xyproblem.info/ +For more information and examples, see http://xyproblem.info/ diff --git a/bot/resources/tags/ytdl.md b/bot/resources/tags/ytdl.md index f96b7f853..68a0a0cdb 100644 --- a/bot/resources/tags/ytdl.md +++ b/bot/resources/tags/ytdl.md @@ -1,4 +1,4 @@ -Per [Python Discord's Rule 5](https://pythondiscord.com/pages/rules), we are unable to assist with questions related to youtube-dl, pytube, or other YouTube video downloaders as their usage violates YouTube's Terms of Service. +Per [Python Discord's Rule 5](https://pythondiscord.com/pages/rules), we are unable to assist with questions related to youtube-dl, pytube, or other YouTube video downloaders, as their usage violates YouTube's Terms of Service. For reference, this usage is covered by the following clauses in [YouTube's TOS](https://www.youtube.com/static?gl=GB&template=terms), as of 2021-03-17: ``` diff --git a/bot/resources/tags/zip.md b/bot/resources/tags/zip.md index 6b05f0282..6f3157f71 100644 --- a/bot/resources/tags/zip.md +++ b/bot/resources/tags/zip.md @@ -3,7 +3,7 @@ The zip function allows you to iterate through multiple iterables simultaneously ```py letters = 'abc' numbers = [1, 2, 3] -# zip(letters, numbers) --> [('a', 1), ('b', 2), ('c', 3)] +# list(zip(letters, numbers)) --> [('a', 1), ('b', 2), ('c', 3)] for letter, number in zip(letters, numbers): print(letter, number) ``` diff --git a/bot/rules/discord_emojis.py b/bot/rules/discord_emojis.py index 41faf7ee8..d979ac5e7 100644 --- a/bot/rules/discord_emojis.py +++ b/bot/rules/discord_emojis.py @@ -4,7 +4,6 @@ from typing import Dict, Iterable, List, Optional, Tuple from discord import Member, Message from emoji import demojize - DISCORD_EMOJI_RE = re.compile(r"<:\w+:\d+>|:\w+:") CODE_BLOCK_RE = re.compile(r"```.*?```", flags=re.DOTALL) diff --git a/bot/rules/links.py b/bot/rules/links.py index ec75a19c5..c46b783c5 100644 --- a/bot/rules/links.py +++ b/bot/rules/links.py @@ -3,7 +3,6 @@ from typing import Dict, Iterable, List, Optional, Tuple from discord import Member, Message - LINK_RE = re.compile(r"(https?://[^\s]+)") diff --git a/bot/rules/mentions.py b/bot/rules/mentions.py index 79725a4b1..6f5addad1 100644 --- a/bot/rules/mentions.py +++ b/bot/rules/mentions.py @@ -13,7 +13,11 @@ async def apply( if msg.author == last_message.author ) - total_recent_mentions = sum(len(msg.mentions) for msg in relevant_messages) + total_recent_mentions = sum( + not user.bot + for msg in relevant_messages + for user in msg.mentions + ) if total_recent_mentions > config['max']: return ( diff --git a/bot/utils/cache.py b/bot/utils/caching.py index 68ce15607..68ce15607 100644 --- a/bot/utils/cache.py +++ b/bot/utils/caching.py diff --git a/bot/utils/channel.py b/bot/utils/channel.py index 72603c521..954a10e56 100644 --- a/bot/utils/channel.py +++ b/bot/utils/channel.py @@ -1,12 +1,13 @@ -import logging +from typing import Union import discord import bot from bot import constants from bot.constants import Categories +from bot.log import get_logger -log = logging.getLogger(__name__) +log = get_logger(__name__) def is_help_channel(channel: discord.TextChannel) -> bool: @@ -17,8 +18,11 @@ def is_help_channel(channel: discord.TextChannel) -> bool: return any(is_in_category(channel, category) for category in categories) -def is_mod_channel(channel: discord.TextChannel) -> bool: - """True if `channel` is considered a mod channel.""" +def is_mod_channel(channel: Union[discord.TextChannel, discord.Thread]) -> bool: + """True if channel, or channel.parent for threads, is considered a mod channel.""" + if isinstance(channel, discord.Thread): + channel = channel.parent + if channel.id in constants.MODERATION_CHANNELS: log.trace(f"Channel #{channel} is a configured mod channel") return True @@ -53,7 +57,7 @@ def is_in_category(channel: discord.TextChannel, category_id: int) -> bool: return getattr(channel, "category_id", None) == category_id -async def try_get_channel(channel_id: int) -> discord.abc.GuildChannel: +async def get_or_fetch_channel(channel_id: int) -> discord.abc.GuildChannel: """Attempt to get or fetch a channel and return it.""" log.trace(f"Getting the channel {channel_id}.") diff --git a/bot/utils/checks.py b/bot/utils/checks.py index 3d0c8a50c..188285684 100644 --- a/bot/utils/checks.py +++ b/bot/utils/checks.py @@ -1,23 +1,14 @@ -import datetime -import logging from typing import Callable, Container, Iterable, Optional, Union from discord.ext.commands import ( - BucketType, - CheckFailure, - Cog, - Command, - CommandOnCooldown, - Context, - Cooldown, - CooldownMapping, - NoPrivateMessage, - has_any_role, + BucketType, CheckFailure, Cog, Command, CommandOnCooldown, Context, Cooldown, CooldownMapping, NoPrivateMessage, + has_any_role ) from bot import constants +from bot.log import get_logger -log = logging.getLogger(__name__) +log = get_logger(__name__) class ContextCheckFailure(CheckFailure): @@ -134,7 +125,7 @@ def cooldown_with_role_bypass(rate: int, per: float, type: BucketType = BucketTy bypass = set(bypass_roles) # this handles the actual cooldown logic - buckets = CooldownMapping(Cooldown(rate, per, type)) + buckets = CooldownMapping(Cooldown(rate, per), type) # will be called after the command has been parse but before it has been invoked, ensures that # the cooldown won't be updated if the user screws up their input to the command @@ -145,11 +136,11 @@ def cooldown_with_role_bypass(rate: int, per: float, type: BucketType = BucketTy return # cooldown logic, taken from discord.py internals - current = ctx.message.created_at.replace(tzinfo=datetime.timezone.utc).timestamp() + current = ctx.message.created_at.timestamp() bucket = buckets.get_bucket(ctx.message) retry_after = bucket.update_rate_limit(current) if retry_after: - raise CommandOnCooldown(bucket, retry_after) + raise CommandOnCooldown(bucket, retry_after, type) def wrapper(command: Command) -> Command: # NOTE: this could be changed if a subclass of Command were to be used. I didn't see the need for it diff --git a/bot/utils/function.py b/bot/utils/function.py index 9bc44e753..55115d7d3 100644 --- a/bot/utils/function.py +++ b/bot/utils/function.py @@ -2,11 +2,12 @@ import functools import inspect -import logging import types import typing as t -log = logging.getLogger(__name__) +from bot.log import get_logger + +log = get_logger(__name__) Argument = t.Union[int, str] BoundArgs = t.OrderedDict[str, t.Any] diff --git a/bot/utils/lock.py b/bot/utils/lock.py index ec6f92cd4..c039a4f25 100644 --- a/bot/utils/lock.py +++ b/bot/utils/lock.py @@ -1,6 +1,5 @@ import asyncio import inspect -import logging import types from collections import defaultdict from functools import partial @@ -8,10 +7,11 @@ from typing import Any, Awaitable, Callable, Hashable, Union from weakref import WeakValueDictionary from bot.errors import LockedResourceError +from bot.log import get_logger from bot.utils import function from bot.utils.function import command_wraps -log = logging.getLogger(__name__) +log = get_logger(__name__) __lock_dicts = defaultdict(WeakValueDictionary) _IdCallableReturn = Union[Hashable, Awaitable[Hashable]] diff --git a/bot/utils/members.py b/bot/utils/members.py new file mode 100644 index 000000000..693286045 --- /dev/null +++ b/bot/utils/members.py @@ -0,0 +1,48 @@ +import typing as t + +import discord + +from bot.log import get_logger + +log = get_logger(__name__) + + +async def get_or_fetch_member(guild: discord.Guild, member_id: int) -> t.Optional[discord.Member]: + """ + Attempt to get a member from cache; on failure fetch from the API. + + Return `None` to indicate the member could not be found. + """ + if member := guild.get_member(member_id): + log.trace("%s retrieved from cache.", member) + else: + try: + member = await guild.fetch_member(member_id) + except discord.errors.NotFound: + log.trace("Failed to fetch %d from API.", member_id) + return None + log.trace("%s fetched from API.", member) + return member + + +async def handle_role_change( + member: discord.Member, + coro: t.Callable[..., t.Coroutine], + role: discord.Role +) -> None: + """ + Change `member`'s cooldown role via awaiting `coro` and handle errors. + + `coro` is intended to be `discord.Member.add_roles` or `discord.Member.remove_roles`. + """ + try: + await coro(role) + except discord.NotFound: + log.debug(f"Failed to change role for {member} ({member.id}): member not found") + except discord.Forbidden: + log.debug( + f"Forbidden to change role for {member} ({member.id}); " + f"possibly due to role hierarchy" + ) + except discord.HTTPException as e: + log.error(f"Failed to change role for {member} ({member.id}): {e.status} {e.code}") diff --git a/bot/utils/message_cache.py b/bot/utils/message_cache.py new file mode 100644 index 000000000..f68d280c9 --- /dev/null +++ b/bot/utils/message_cache.py @@ -0,0 +1,197 @@ +import typing as t +from math import ceil + +from discord import Message + + +class MessageCache: + """ + A data structure for caching messages. + + The cache is implemented as a circular buffer to allow constant time append, prepend, pop from either side, + and lookup by index. The cache therefore does not support removal at an arbitrary index (although it can be + implemented to work in linear time relative to the maximum size). + + The object additionally holds a mapping from Discord message ID's to the index in which the corresponding message + is stored, to allow for constant time lookup by message ID. + + The cache has a size limit operating the same as with a collections.deque, and most of its method names mirror those + of a deque. + + The implementation is transparent to the user: to the user the first element is always at index 0, and there are + only as many elements as were inserted (meaning, without any pre-allocated placeholder values). + """ + + def __init__(self, maxlen: int, *, newest_first: bool = False): + if maxlen <= 0: + raise ValueError("maxlen must be positive") + self.maxlen = maxlen + self.newest_first = newest_first + + self._start = 0 + self._end = 0 + + self._messages: list[t.Optional[Message]] = [None] * self.maxlen + self._message_id_mapping = {} + + def append(self, message: Message) -> None: + """Add the received message to the cache, depending on the order of messages defined by `newest_first`.""" + if self.newest_first: + self._appendleft(message) + else: + self._appendright(message) + + def _appendright(self, message: Message) -> None: + """Add the received message to the end of the cache.""" + if self._is_full(): + del self._message_id_mapping[self._messages[self._start].id] + self._start = (self._start + 1) % self.maxlen + + self._messages[self._end] = message + self._message_id_mapping[message.id] = self._end + self._end = (self._end + 1) % self.maxlen + + def _appendleft(self, message: Message) -> None: + """Add the received message to the beginning of the cache.""" + if self._is_full(): + self._end = (self._end - 1) % self.maxlen + del self._message_id_mapping[self._messages[self._end].id] + + self._start = (self._start - 1) % self.maxlen + self._messages[self._start] = message + self._message_id_mapping[message.id] = self._start + + def pop(self) -> Message: + """Remove the last message in the cache and return it.""" + if self._is_empty(): + raise IndexError("pop from an empty cache") + + self._end = (self._end - 1) % self.maxlen + message = self._messages[self._end] + del self._message_id_mapping[message.id] + self._messages[self._end] = None + + return message + + def popleft(self) -> Message: + """Return the first message in the cache and return it.""" + if self._is_empty(): + raise IndexError("pop from an empty cache") + + message = self._messages[self._start] + del self._message_id_mapping[message.id] + self._messages[self._start] = None + self._start = (self._start + 1) % self.maxlen + + return message + + def clear(self) -> None: + """Remove all messages from the cache.""" + self._messages = [None] * self.maxlen + self._message_id_mapping = {} + + self._start = 0 + self._end = 0 + + def get_message(self, message_id: int) -> t.Optional[Message]: + """Return the message that has the given message ID, if it is cached.""" + index = self._message_id_mapping.get(message_id, None) + return self._messages[index] if index is not None else None + + def update(self, message: Message) -> bool: + """ + Update a cached message with new contents. + + Return True if the given message had a matching ID in the cache. + """ + index = self._message_id_mapping.get(message.id, None) + if index is None: + return False + self._messages[index] = message + return True + + def __contains__(self, message_id: int) -> bool: + """Return True if the cache contains a message with the given ID .""" + return message_id in self._message_id_mapping + + def __getitem__(self, item: t.Union[int, slice]) -> t.Union[Message, list[Message]]: + """ + Return the message(s) in the index or slice provided. + + This method makes the circular buffer implementation transparent to the user. + Providing 0 will return the message at the position perceived by the user to be the beginning of the cache, + meaning at `self._start`. + """ + # Keep in mind that for the modulo operator used throughout this function, Python modulo behaves similarly when + # the left operand is negative. E.g -1 % 5 == 4, because the closest number from the bottom that wholly divides + # by 5 is -5. + if isinstance(item, int): + if item >= len(self) or item < -len(self): + raise IndexError("cache index out of range") + return self._messages[(item + self._start) % self.maxlen] + + elif isinstance(item, slice): + length = len(self) + start, stop, step = item.indices(length) + + # This needs to be checked explicitly now, because otherwise self._start >= self._end is a valid state. + if (start >= stop and step >= 0) or (start <= stop and step <= 0): + return [] + + start = (start + self._start) % self.maxlen + stop = (stop + self._start) % self.maxlen + + # Having empty cells is an implementation detail. To the user the cache contains as many elements as they + # inserted, therefore any empty cells should be ignored. There can only be Nones at the tail. + if step > 0: + if ( + (self._start < self._end and not self._start < stop <= self._end) + or (self._start > self._end and self._end < stop <= self._start) + ): + stop = self._end + else: + lower_boundary = (self._start - 1) % self.maxlen + if ( + (self._start < self._end and not self._start - 1 <= stop < self._end) + or (self._start > self._end and self._end < stop < lower_boundary) + ): + stop = lower_boundary + + if (start < stop and step > 0) or (start > stop and step < 0): + return self._messages[start:stop:step] + # step != 1 may require a start offset in the second slicing. + if step > 0: + offset = ceil((self.maxlen - start) / step) * step + start - self.maxlen + return self._messages[start::step] + self._messages[offset:stop:step] + else: + offset = ceil((start + 1) / -step) * -step - start - 1 + return self._messages[start::step] + self._messages[self.maxlen - 1 - offset:stop:step] + + else: + raise TypeError(f"cache indices must be integers or slices, not {type(item)}") + + def __iter__(self) -> t.Iterator[Message]: + if self._is_empty(): + return + + if self._start < self._end: + yield from self._messages[self._start:self._end] + else: + yield from self._messages[self._start:] + yield from self._messages[:self._end] + + def __len__(self): + """Get the number of non-empty cells in the cache.""" + if self._is_empty(): + return 0 + if self._end > self._start: + return self._end - self._start + return self.maxlen - self._start + self._end + + def _is_empty(self) -> bool: + """Return True if the cache has no messages.""" + return self._messages[self._start] is None + + def _is_full(self) -> bool: + """Return True if every cell in the cache already contains a message.""" + return self._messages[self._end] is not None diff --git a/bot/utils/messages.py b/bot/utils/messages.py index d4a921161..e55c07062 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -1,6 +1,4 @@ import asyncio -import contextlib -import logging import random import re from functools import partial @@ -8,15 +6,14 @@ from io import BytesIO from typing import Callable, List, Optional, Sequence, Union import discord -from discord import Message, MessageType, Reaction, User -from discord.errors import HTTPException from discord.ext.commands import Context import bot from bot.constants import Emojis, MODERATION_ROLES, NEGATIVE_REPLIES +from bot.log import get_logger from bot.utils import scheduling -log = logging.getLogger(__name__) +log = get_logger(__name__) def reaction_check( @@ -54,7 +51,7 @@ def reaction_check( log.trace(f"Removing reaction {reaction} by {user} on {reaction.message.id}: disallowed user.") scheduling.create_task( reaction.message.remove_reaction(reaction.emoji, user), - suppressed_exceptions=(HTTPException,), + suppressed_exceptions=(discord.HTTPException,), name=f"remove_reaction-{reaction}-{reaction.message.id}-{user}" ) return False @@ -69,7 +66,9 @@ async def wait_for_deletion( allow_mods: bool = True ) -> None: """ - Wait for up to `timeout` seconds for a reaction by any of the specified `user_ids` to delete the message. + Wait for any of `user_ids` to react with one of the `deletion_emojis` within `timeout` seconds to delete `message`. + + If `timeout` expires then reactions are cleared to indicate the option to delete has expired. An `attach_emojis` bool may be specified to determine whether to attach the given `deletion_emojis` to the message in the given `context`. @@ -95,9 +94,15 @@ async def wait_for_deletion( allow_mods=allow_mods, ) - with contextlib.suppress(asyncio.TimeoutError): - await bot.instance.wait_for('reaction_add', check=check, timeout=timeout) - await message.delete() + try: + try: + await bot.instance.wait_for('reaction_add', check=check, timeout=timeout) + except asyncio.TimeoutError: + await message.clear_reactions() + else: + await message.delete() + except discord.NotFound: + log.trace(f"wait_for_deletion: message {message.id} deleted prematurely.") async def send_attachments( @@ -116,7 +121,7 @@ async def send_attachments( """ webhook_send_kwargs = { 'username': message.author.display_name, - 'avatar_url': message.author.avatar_url, + 'avatar_url': message.author.display_avatar.url, } webhook_send_kwargs.update(kwargs) webhook_send_kwargs['username'] = sub_clyde(webhook_send_kwargs['username']) @@ -146,7 +151,7 @@ async def send_attachments( large.append(attachment) else: log.info(f"{failure_msg} because it's too large.") - except HTTPException as e: + except discord.HTTPException as e: if link_large and e.status == 413: large.append(attachment) else: @@ -167,8 +172,8 @@ async def send_attachments( async def count_unique_users_reaction( message: discord.Message, - reaction_predicate: Callable[[Reaction], bool] = lambda _: True, - user_predicate: Callable[[User], bool] = lambda _: True, + reaction_predicate: Callable[[discord.Reaction], bool] = lambda _: True, + user_predicate: Callable[[discord.User], bool] = lambda _: True, count_bots: bool = True ) -> int: """ @@ -188,7 +193,7 @@ async def count_unique_users_reaction( return len(unique_users) -async def pin_no_system_message(message: Message) -> bool: +async def pin_no_system_message(message: discord.Message) -> bool: """Pin the given message, wait a couple of seconds and try to delete the system message.""" await message.pin() @@ -196,7 +201,7 @@ async def pin_no_system_message(message: Message) -> bool: await asyncio.sleep(2) # Search for the system message in the last 10 messages async for historical_message in message.channel.history(limit=10): - if historical_message.type == MessageType.pins_add: + if historical_message.type == discord.MessageType.pins_add: await historical_message.delete() return True diff --git a/bot/utils/regex.py b/bot/utils/regex.py deleted file mode 100644 index a8efe1446..000000000 --- a/bot/utils/regex.py +++ /dev/null @@ -1,13 +0,0 @@ -import re - -INVITE_RE = re.compile( - r"(?:discord(?:[\.,]|dot)gg|" # Could be discord.gg/ - r"discord(?:[\.,]|dot)com(?:\/|slash)invite|" # or discord.com/invite/ - r"discordapp(?:[\.,]|dot)com(?:\/|slash)invite|" # or discordapp.com/invite/ - r"discord(?:[\.,]|dot)me|" # or discord.me - r"discord(?:[\.,]|dot)li|" # or discord.li - r"discord(?:[\.,]|dot)io" # or discord.io. - r")(?:[\/]|slash)" # / or 'slash' - r"([a-zA-Z0-9\-]+)", # the invite code itself - flags=re.IGNORECASE -) diff --git a/bot/utils/scheduling.py b/bot/utils/scheduling.py index bb83b5c0d..23acacf74 100644 --- a/bot/utils/scheduling.py +++ b/bot/utils/scheduling.py @@ -1,11 +1,14 @@ import asyncio import contextlib import inspect -import logging import typing as t from datetime import datetime from functools import partial +from arrow import Arrow + +from bot.log import get_logger + class Scheduler: """ @@ -27,7 +30,7 @@ class Scheduler: def __init__(self, name: str): self.name = name - self._log = logging.getLogger(f"{__name__}.{name}") + self._log = get_logger(f"{__name__}.{name}") self._scheduled_tasks: t.Dict[t.Hashable, asyncio.Task] = {} def __contains__(self, task_id: t.Hashable) -> bool: @@ -57,7 +60,7 @@ class Scheduler: self._scheduled_tasks[task_id] = task self._log.debug(f"Scheduled task #{task_id} {id(task)}.") - def schedule_at(self, time: datetime, task_id: t.Hashable, coroutine: t.Coroutine) -> None: + def schedule_at(self, time: t.Union[datetime, Arrow], task_id: t.Hashable, coroutine: t.Coroutine) -> None: """ Schedule `coroutine` to be executed at the given `time`. @@ -187,5 +190,5 @@ def _log_task_exception(task: asyncio.Task, *, suppressed_exceptions: t.Tuple[t. exception = task.exception() # Log the exception if one exists. if exception and not isinstance(exception, suppressed_exceptions): - log = logging.getLogger(__name__) + log = get_logger(__name__) log.error(f"Error in task {task.get_name()} {id(task)}!", exc_info=exception) diff --git a/bot/utils/services.py b/bot/utils/services.py index db9c93d0f..439c8d500 100644 --- a/bot/utils/services.py +++ b/bot/utils/services.py @@ -1,12 +1,12 @@ -import logging from typing import Optional from aiohttp import ClientConnectorError import bot from bot.constants import URLs +from bot.log import get_logger -log = logging.getLogger(__name__) +log = get_logger(__name__) FAILED_REQUEST_ATTEMPTS = 3 diff --git a/bot/utils/time.py b/bot/utils/time.py index d55a0e532..a0379c3ef 100644 --- a/bot/utils/time.py +++ b/bot/utils/time.py @@ -1,13 +1,12 @@ import datetime import re -from typing import Optional +from enum import Enum +from time import struct_time +from typing import Literal, Optional, Union, overload -import dateutil.parser +import arrow from dateutil.relativedelta import relativedelta -RFC1123_FORMAT = "%a, %d %b %Y %H:%M:%S GMT" -INFRACTION_FORMAT = "%Y-%m-%d %H:%M" - _DURATION_REGEX = re.compile( r"((?P<years>\d+?) ?(years|year|Y|y) ?)?" r"((?P<months>\d+?) ?(months|month|m) ?)?" @@ -18,10 +17,40 @@ _DURATION_REGEX = re.compile( r"((?P<seconds>\d+?) ?(seconds|second|S|s))?" ) +# All supported types for the single-argument overload of arrow.get(). tzinfo is excluded because +# it's too implicit of a way for the caller to specify that they want the current time. +Timestamp = Union[ + arrow.Arrow, + datetime.datetime, + datetime.date, + struct_time, + int, # POSIX timestamp + float, # POSIX timestamp + str, # ISO 8601-formatted string + tuple[int, int, int], # ISO calendar tuple +] +_Precision = Literal["years", "months", "days", "hours", "minutes", "seconds"] + + +class TimestampFormats(Enum): + """ + Represents the different formats possible for Discord timestamps. + + Examples are given in epoch time. + """ + + DATE_TIME = "f" # January 1, 1970 1:00 AM + DAY_TIME = "F" # Thursday, January 1, 1970 1:00 AM + DATE_SHORT = "d" # 01/01/1970 + DATE = "D" # January 1, 1970 + TIME = "t" # 1:00 AM + TIME_SECONDS = "T" # 1:00:00 AM + RELATIVE = "R" # 52 years ago + def _stringify_time_unit(value: int, unit: str) -> str: """ - Returns a string to represent a value and time unit, ensuring that it uses the right plural form of the unit. + Return a string to represent a value and time unit, ensuring the unit's correct plural form is used. >>> _stringify_time_unit(1, "seconds") "1 second" @@ -40,15 +69,140 @@ def _stringify_time_unit(value: int, unit: str) -> str: return f"{value} {unit}" -def humanize_delta(delta: relativedelta, precision: str = "seconds", max_units: int = 6) -> str: +def discord_timestamp(timestamp: Timestamp, format: TimestampFormats = TimestampFormats.DATE_TIME) -> str: """ - Returns a human-readable version of the relativedelta. + Format a timestamp as a Discord-flavored Markdown timestamp. - precision specifies the smallest unit of time to include (e.g. "seconds", "minutes"). - max_units specifies the maximum number of units of time to include (e.g. 1 may include days but not hours). + `timestamp` can be any type supported by the single-arg `arrow.get()`, except for a `tzinfo`. """ + timestamp = int(arrow.get(timestamp).timestamp()) + return f"<t:{timestamp}:{format.value}>" + + +# region humanize_delta overloads +@overload +def humanize_delta( + arg1: Union[relativedelta, Timestamp], + /, + *, + precision: _Precision = "seconds", + max_units: int = 6, + absolute: bool = True, +) -> str: + ... + + +@overload +def humanize_delta( + end: Timestamp, + start: Timestamp, + /, + *, + precision: _Precision = "seconds", + max_units: int = 6, + absolute: bool = True, +) -> str: + ... + + +@overload +def humanize_delta( + *, + years: int = 0, + months: int = 0, + weeks: float = 0, + days: float = 0, + hours: float = 0, + minutes: float = 0, + seconds: float = 0, + precision: _Precision = "seconds", + max_units: int = 6, + absolute: bool = True, +) -> str: + ... +# endregion + + +def humanize_delta( + *args, + precision: _Precision = "seconds", + max_units: int = 6, + absolute: bool = True, + **kwargs, +) -> str: + """ + Return a human-readable version of a time duration. + + `precision` is the smallest unit of time to include (e.g. "seconds", "minutes"). + + `max_units` is the maximum number of units of time to include. + Count units from largest to smallest (e.g. count days before months). + + Use the absolute value of the duration if `absolute` is True. + + Usage: + + Keyword arguments specifying values for time units, to construct a `relativedelta` and humanize + the duration represented by it: + + >>> humanize_delta(days=2, hours=16, seconds=23) + '2 days, 16 hours and 23 seconds' + + **One** `relativedelta` object, to humanize the duration represented by it: + + >>> humanize_delta(relativedelta(years=12, months=6)) + '12 years and 6 months' + + Note that `leapdays` and absolute info (singular names) will be ignored during humanization. + + **One** timestamp of a type supported by the single-arg `arrow.get()`, except for `tzinfo`, + to humanize the duration between it and the current time: + + >>> humanize_delta('2021-08-06T12:43:01Z', absolute=True) # now = 2021-08-06T12:33:33Z + '9 minutes and 28 seconds' + + >>> humanize_delta('2021-08-06T12:43:01Z', absolute=False) # now = 2021-08-06T12:33:33Z + '-9 minutes and -28 seconds' + + **Two** timestamps, each of a type supported by the single-arg `arrow.get()`, except for + `tzinfo`, to humanize the duration between them: + + >>> humanize_delta(datetime.datetime(2020, 1, 1), '2021-01-01T12:00:00Z', absolute=False) + '1 year and 12 hours' + + >>> humanize_delta('2021-01-01T12:00:00Z', datetime.datetime(2020, 1, 1), absolute=False) + '-1 years and -12 hours' + + Note that order of the arguments can result in a different output even if `absolute` is True: + + >>> x = datetime.datetime(3000, 11, 1) + >>> y = datetime.datetime(3000, 9, 2) + >>> humanize_delta(y, x, absolute=True), humanize_delta(x, y, absolute=True) + ('1 month and 30 days', '1 month and 29 days') + + This is due to the nature of `relativedelta`; it does not represent a fixed period of time. + Instead, it's relative to the `datetime` to which it's added to get the other `datetime`. + In the example, the difference arises because all months don't have the same number of days. + """ + if args and kwargs: + raise ValueError("Unsupported combination of positional and keyword arguments.") + + if len(args) == 0: + delta = relativedelta(**kwargs) + elif len(args) == 1 and isinstance(args[0], relativedelta): + delta = args[0] + elif len(args) <= 2: + end = arrow.get(args[0]) + start = arrow.get(args[1]) if len(args) == 2 else arrow.utcnow() + + delta = relativedelta(end.datetime, start.datetime) + if absolute: + delta = abs(delta) + else: + raise ValueError(f"Received {len(args)} positional arguments, but expected 1 or 2.") + if max_units <= 0: - raise ValueError("max_units must be positive") + raise ValueError("max_units must be positive.") units = ( ("years", delta.years), @@ -59,7 +213,7 @@ def humanize_delta(delta: relativedelta, precision: str = "seconds", max_units: ("seconds", delta.seconds), ) - # Add the time units that are >0, but stop at accuracy or max_units. + # Add the time units that are >0, but stop at precision or max_units. time_strings = [] unit_count = 0 for unit, value in units: @@ -70,7 +224,7 @@ def humanize_delta(delta: relativedelta, precision: str = "seconds", max_units: if unit == precision or unit_count >= max_units: break - # Add the 'and' between the last two units, if necessary + # Add the 'and' between the last two units, if necessary. if len(time_strings) > 1: time_strings[-1] = f"{time_strings[-2]} and {time_strings[-1]}" del time_strings[-2] @@ -84,19 +238,12 @@ def humanize_delta(delta: relativedelta, precision: str = "seconds", max_units: return humanized -def get_time_delta(time_string: str) -> str: - """Returns the time in human-readable time delta format.""" - date_time = dateutil.parser.isoparse(time_string).replace(tzinfo=None) - time_delta = time_since(date_time, precision="minutes", max_units=1) - - return time_delta - - def parse_duration_string(duration: str) -> Optional[relativedelta]: """ - Converts a `duration` string to a relativedelta object. + Convert a `duration` string to a relativedelta object. + + The following symbols are supported for each unit of time: - The function supports the following symbols for each unit of time: - years: `Y`, `y`, `year`, `years` - months: `m`, `month`, `months` - weeks: `w`, `W`, `week`, `weeks` @@ -104,8 +251,9 @@ def parse_duration_string(duration: str) -> Optional[relativedelta]: - hours: `H`, `h`, `hour`, `hours` - minutes: `M`, `minute`, `minutes` - seconds: `S`, `s`, `second`, `seconds` + The units need to be provided in descending order of magnitude. - If the string does represent a durationdelta object, it will return None. + Return None if the `duration` string cannot be parsed according to the symbols above. """ match = _DURATION_REGEX.fullmatch(duration) if not match: @@ -118,93 +266,63 @@ def parse_duration_string(duration: str) -> Optional[relativedelta]: def relativedelta_to_timedelta(delta: relativedelta) -> datetime.timedelta: - """Converts a relativedelta object to a timedelta object.""" - utcnow = datetime.datetime.utcnow() + """Convert a relativedelta object to a timedelta object.""" + utcnow = arrow.utcnow() return utcnow + delta - utcnow -def time_since(past_datetime: datetime.datetime, precision: str = "seconds", max_units: int = 6) -> str: +def format_relative(timestamp: Timestamp) -> str: """ - Takes a datetime and returns a human-readable string that describes how long ago that datetime was. - - precision specifies the smallest unit of time to include (e.g. "seconds", "minutes"). - max_units specifies the maximum number of units of time to include (e.g. 1 may include days but not hours). - """ - now = datetime.datetime.utcnow() - delta = abs(relativedelta(now, past_datetime)) - - humanized = humanize_delta(delta, precision, max_units) + Format `timestamp` as a relative Discord timestamp. - return f"{humanized} ago" + A relative timestamp describes how much time has elapsed since `timestamp` or how much time + remains until `timestamp` is reached. - -def parse_rfc1123(stamp: str) -> datetime.datetime: - """Parse RFC1123 time string into datetime.""" - return datetime.datetime.strptime(stamp, RFC1123_FORMAT).replace(tzinfo=datetime.timezone.utc) - - -def format_infraction(timestamp: str) -> str: - """Format an infraction timestamp to a more readable ISO 8601 format.""" - return dateutil.parser.isoparse(timestamp).strftime(INFRACTION_FORMAT) + `timestamp` can be any type supported by the single-arg `arrow.get()`, except for a `tzinfo`. + """ + return discord_timestamp(timestamp, TimestampFormats.RELATIVE) -def format_infraction_with_duration( - date_to: Optional[str], - date_from: Optional[datetime.datetime] = None, +def format_with_duration( + timestamp: Optional[Timestamp], + other_timestamp: Optional[Timestamp] = None, max_units: int = 2, - absolute: bool = True ) -> Optional[str]: """ - Return `date_to` formatted as a readable ISO-8601 with the humanized duration since `date_from`. + Return `timestamp` formatted as a discord timestamp with the timestamp duration since `other_timestamp`. - `date_from` must be an ISO-8601 formatted timestamp. The duration is calculated as from - `date_from` until `date_to` with a precision of seconds. If `date_from` is unspecified, the - current time is used. + `timestamp` and `other_timestamp` can be any type supported by the single-arg `arrow.get()`, + except for a `tzinfo`. Use the current time if `other_timestamp` is None or unspecified. - `max_units` specifies the maximum number of units of time to include in the duration. For - example, a value of 1 may include days but not hours. + `max_units` is forwarded to `time.humanize_delta`. See its documentation for more information. - If `absolute` is True, the absolute value of the duration delta is used. This prevents negative - values in the case that `date_to` is in the past relative to `date_from`. + Return None if `timestamp` is None. """ - if not date_to: + if timestamp is None: return None - date_to_formatted = format_infraction(date_to) - - date_from = date_from or datetime.datetime.utcnow() - date_to = dateutil.parser.isoparse(date_to).replace(tzinfo=None, microsecond=0) - - delta = relativedelta(date_to, date_from) - if absolute: - delta = abs(delta) + if other_timestamp is None: + other_timestamp = arrow.utcnow() - duration = humanize_delta(delta, max_units=max_units) - duration_formatted = f" ({duration})" if duration else "" + formatted_timestamp = discord_timestamp(timestamp) + duration = humanize_delta(timestamp, other_timestamp, max_units=max_units) - return f"{date_to_formatted}{duration_formatted}" + return f"{formatted_timestamp} ({duration})" -def until_expiration( - expiry: Optional[str], - now: Optional[datetime.datetime] = None, - max_units: int = 2 -) -> Optional[str]: +def until_expiration(expiry: Optional[Timestamp]) -> str: """ - Get the remaining time until infraction's expiration, in a human-readable version of the relativedelta. + Get the remaining time until an infraction's expiration as a Discord timestamp. - Returns a human-readable version of the remaining duration between datetime.utcnow() and an expiry. - Unlike `humanize_delta`, this function will force the `precision` to be `seconds` by not passing it. - `max_units` specifies the maximum number of units of time to include (e.g. 1 may include days but not hours). - By default, max_units is 2. - """ - if not expiry: - return None + `expiry` can be any type supported by the single-arg `arrow.get()`, except for a `tzinfo`. - now = now or datetime.datetime.utcnow() - since = dateutil.parser.isoparse(expiry).replace(tzinfo=None, microsecond=0) + Return "Permanent" if `expiry` is None. Return "Expired" if `expiry` is in the past. + """ + if expiry is None: + return "Permanent" - if since < now: - return None + expiry = arrow.get(expiry) + if expiry < arrow.utcnow(): + return "Expired" - return humanize_delta(relativedelta(since, now), max_units=max_units) + return format_relative(expiry) diff --git a/bot/utils/webhooks.py b/bot/utils/webhooks.py index 66f82ec66..9c916b63a 100644 --- a/bot/utils/webhooks.py +++ b/bot/utils/webhooks.py @@ -1,12 +1,12 @@ -import logging from typing import Optional import discord from discord import Embed +from bot.log import get_logger from bot.utils.messages import sub_clyde -log = logging.getLogger(__name__) +log = get_logger(__name__) async def send_webhook( diff --git a/config-default.yml b/config-default.yml index 811640034..583733fda 100644 --- a/config-default.yml +++ b/config-default.yml @@ -1,3 +1,7 @@ +debug: !ENV ["BOT_DEBUG", "true"] +file_logs: !ENV ["FILE_LOGS", "false"] + + bot: prefix: "!" sentry_dsn: !ENV "BOT_SENTRY_DSN" @@ -141,6 +145,8 @@ guild: logs: &LOGS 468520609152892958 moderators: &MODS_CATEGORY 749736277464842262 modmail: &MODMAIL 714494672835444826 + appeals: &APPEALS 890331800025563216 + appeals2: &APPEALS2 895417395261341766 voice: 356013253765234688 summer_code_jam: 861692638540857384 @@ -154,9 +160,10 @@ guild: reddit: &REDDIT_CHANNEL 458224812528238616 # Development - dev_contrib: &DEV_CONTRIB 635950537262759947 - dev_core: &DEV_CORE 411200599653351425 - dev_log: &DEV_LOG 622895325144940554 + dev_contrib: &DEV_CONTRIB 635950537262759947 + dev_core: &DEV_CORE 411200599653351425 + dev_voting: &DEV_CORE_VOTING 839162966519447552 + dev_log: &DEV_LOG 622895325144940554 # Discussion meta: 429409067623251969 @@ -167,7 +174,7 @@ guild: how_to_get_help: 704250143020417084 # Topical - discord_py: 343944376055103488 + discord_bots: 343944376055103488 # Logs attachment_log: &ATTACH_LOG 649243850006855680 @@ -201,6 +208,7 @@ guild: incidents_archive: 720668923636351037 mod_alerts: 473092532147060736 mods: &MODS 305126844661760000 + mod_meta: 775412552795947058 nominations: 822920136150745168 nomination_voting: 822853512709931008 organisation: &ORGANISATION 551789653284356126 @@ -229,22 +237,22 @@ guild: # Watch big_brother_logs: &BB_LOGS 468507907357409333 - talent_pool: &TALENT_POOL 534321732593647616 moderation_categories: - *MODS_CATEGORY - *MODMAIL - *LOGS + - *APPEALS + - *APPEALS2 moderation_channels: - *ADMINS - *ADMIN_SPAM - *MODS - # Modlog cog ignores events which occur in these channels + # Modlog cog explicitly ignores events which occur in these channels. + # This is on top of implicitly ignoring events in channels that the mod team cannot view. modlog_blacklist: - - *ADMINS - - *ADMINS_VOICE - *ATTACH_LOG - *MESSAGE_LOG - *MOD_LOG @@ -256,11 +264,16 @@ guild: - *BLACK_FORMATTER roles: + # Self-assignable roles, see the Subscribe cog + advent_of_code: 518565788744024082 announcements: 463658397560995840 + lovefest: 542431903886606399 + pyweek_announcements: 897568414044938310 + contributors: 295488872404484098 help_cooldown: 699189276025421825 muted: &MUTED_ROLE 277914926603829249 - partners: 323426753857191936 + partners: &PY_PARTNER_ROLE 323426753857191936 python_community: &PY_COMMUNITY_ROLE 458226413825294336 sprinters: &SPRINTERS 758422482289426471 voice_verified: 764802720779337729 @@ -300,9 +313,9 @@ guild: big_brother: 569133704568373283 dev_log: 680501655111729222 duck_pond: 637821475327311927 + incidents: 816650601844572212 incidents_archive: 720671599790915702 python_news: &PYNEWS_WEBHOOK 704381182279942324 - talent_pool: 569145364800602132 filter: @@ -333,7 +346,6 @@ filter: - *MESSAGE_LOG - *MOD_LOG - *STAFF_LOUNGE - - *TALENT_POOL role_whitelist: - *ADMINS_ROLE @@ -342,6 +354,7 @@ filter: - *OWNERS_ROLE - *PY_COMMUNITY_ROLE - *SPRINTERS + - *PY_PARTNER_ROLE keys: @@ -354,17 +367,17 @@ urls: connect_max_retries: 3 connect_cooldown: 5 site: &DOMAIN "pythondiscord.com" - site_api: &API "pydis-api.default.svc.cluster.local" + site_api: &API "site.default.svc.cluster.local/api" site_api_schema: "http://" site_paste: &PASTE !JOIN ["paste.", *DOMAIN] site_schema: &SCHEMA "https://" - site_staff: &STAFF !JOIN ["staff.", *DOMAIN] + site_staff: &STAFF !JOIN [*SCHEMA, *DOMAIN, "/staff"] paste_service: !JOIN [*SCHEMA, *PASTE, "/{key}"] - site_logs_view: !JOIN [*SCHEMA, *STAFF, "/bot/logs"] + site_logs_view: !JOIN [*STAFF, "/bot/logs"] # Snekbox - snekbox_eval_api: "http://snekbox.default.svc.cluster.local/eval" + snekbox_eval_api: !ENV ["SNEKBOX_EVAL_API", "http://snekbox.default.svc.cluster.local/eval"] # Discord API URLs discord_api: &DISCORD_API "https://discordapp.com/api/v7/" @@ -376,6 +389,8 @@ urls: anti_spam: + cache_size: 100 + # Clean messages that violate a rule. clean_offending: true ping_everyone: true @@ -431,14 +446,13 @@ anti_spam: max: 3 - metabase: - username: !ENV "METABASE_USERNAME" - password: !ENV "METABASE_PASSWORD" - url: "http://metabase.default.svc.cluster.local/api" + username: !ENV "METABASE_USERNAME" + password: !ENV "METABASE_PASSWORD" + base_url: "http://metabase.default.svc.cluster.local" + public_url: "https://metabase.pythondiscord.com" # 14 days, see https://www.metabase.com/docs/latest/operations-guide/environment-variables.html#max_session_age - max_session_age: 20160 - + max_session_age: 20160 big_brother: diff --git a/docker-compose.yml b/docker-compose.yml index 0f0355dac..ce78f65aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,11 @@ services: POSTGRES_DB: pysite POSTGRES_PASSWORD: pysite POSTGRES_USER: pysite + healthcheck: + test: ["CMD-SHELL", "pg_isready -U pysite"] + interval: 2s + timeout: 1s + retries: 5 redis: << : *logging @@ -31,6 +36,22 @@ services: ports: - "127.0.0.1:6379:6379" + metricity: + << : *logging + << : *restart_policy + restart: on-failure # USE_METRICITY=false will stop the container, so this ensures it only restarts on error + depends_on: + postgres: + condition: service_healthy + image: ghcr.io/python-discord/metricity:latest + env_file: + - .env + environment: + DATABASE_URI: postgres://pysite:pysite@postgres/metricity + USE_METRICITY: ${USE_METRICITY-false} + volumes: + - .:/tmp/bot:ro + snekbox: << : *logging << : *restart_policy @@ -56,7 +77,7 @@ services: - "127.0.0.1:8000:8000" tty: true depends_on: - - postgres + - metricity environment: DATABASE_URL: postgres://pysite:pysite@postgres:5432/pysite METRICITY_DB_URL: postgres://pysite:pysite@postgres:5432/metricity @@ -70,7 +91,6 @@ services: context: . dockerfile: Dockerfile volumes: - - ./logs:/bot/logs - .:/bot:ro tty: true depends_on: diff --git a/poetry.lock b/poetry.lock index dac277ed8..6d3bd44bb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,10 @@ [[package]] name = "aio-pika" -version = "6.8.0" +version = "6.8.1" description = "Wrapper for the aiormq for asyncio and humans." category = "main" optional = false -python-versions = ">3.5.*, <4" +python-versions = ">=3.5, <4" [package.dependencies] aiormq = ">=3.2.3,<4" @@ -71,14 +71,6 @@ yarl = "*" develop = ["aiomisc (>=11.0,<12.0)", "async-generator", "coverage (!=4.3)", "coveralls", "pylava", "pytest", "pytest-cov", "tox (>=2.4)"] [[package]] -name = "appdirs" -version = "1.4.4" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = "*" - -[[package]] name = "arrow" version = "1.0.3" description = "Better dates & times for Python" @@ -122,36 +114,51 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "21.2.0" +version = "21.4.0" description = "Classes Without Boilerplate" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] [[package]] name = "beautifulsoup4" -version = "4.9.3" +version = "4.10.0" description = "Screen-scraping library" category = "main" optional = false -python-versions = "*" +python-versions = ">3.0.0" [package.dependencies] -soupsieve = {version = ">1.2", markers = "python_version >= \"3.0\""} +soupsieve = ">1.2" [package.extras] html5lib = ["html5lib"] lxml = ["lxml"] [[package]] +name = "bot-core" +version = "1.2.0" +description = "Bot-Core provides the core functionality and utilities for the bots of the Python Discord community." +category = "main" +optional = false +python-versions = "3.9.*" + +[package.dependencies] +"discord.py" = {url = "https://github.com/Rapptz/discord.py/archive/45d498c1b76deaf3b394d17ccf56112fa691d160.zip"} + +[package.source] +type = "url" +url = "https://github.com/python-discord/bot-core/archive/511bcba1b0196cd498c707a525ea56921bd971db.zip" + +[[package]] name = "certifi" -version = "2021.5.30" +version = "2021.10.8" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -159,7 +166,7 @@ python-versions = "*" [[package]] name = "cffi" -version = "1.14.5" +version = "1.15.0" description = "Foreign Function Interface for Python calling C code." category = "main" optional = false @@ -170,7 +177,7 @@ pycparser = "*" [[package]] name = "cfgv" -version = "3.3.0" +version = "3.3.1" description = "Validate configuration and produce human readable error messages." category = "dev" optional = false @@ -185,6 +192,17 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] +name = "charset-normalizer" +version = "2.0.10" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.5.0" + +[package.extras] +unicode_backport = ["unicodedata2"] + +[[package]] name = "colorama" version = "0.4.4" description = "Cross-platform colored terminal text." @@ -218,67 +236,62 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" toml = ["toml"] [[package]] -name = "coveralls" -version = "2.2.0" -description = "Show coverage stats online via coveralls.io" -category = "dev" +name = "deepdiff" +version = "4.3.2" +description = "Deep Difference and Search of any Python object/data." +category = "main" optional = false -python-versions = ">= 3.5" +python-versions = ">=3.5" [package.dependencies] -coverage = ">=4.1,<6.0" -docopt = ">=0.6.1" -requests = ">=1.0.0" +ordered-set = ">=3.1.1" [package.extras] -yaml = ["PyYAML (>=3.10)"] +murmur = ["mmh3"] [[package]] -name = "deepdiff" -version = "4.3.2" -description = "Deep Difference and Search of any Python object/data." +name = "deprecated" +version = "1.2.13" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] -ordered-set = ">=3.1.1" +wrapt = ">=1.10,<2" [package.extras] -murmur = ["mmh3"] +dev = ["tox", "bump2version (<1)", "sphinx (<2)", "importlib-metadata (<3)", "importlib-resources (<4)", "configparser (<5)", "sphinxcontrib-websupport (<2)", "zipp (<2)", "PyTest (<5)", "PyTest-Cov (<2.6)", "pytest", "pytest-cov"] [[package]] name = "discord.py" -version = "1.7.3" +version = "2.0.0a0" description = "A Python wrapper for the Discord API" category = "main" optional = false -python-versions = ">=3.5.3" +python-versions = ">=3.8.0" [package.dependencies] aiohttp = ">=3.6.0,<3.8.0" [package.extras] -docs = ["sphinx (==3.0.3)", "sphinxcontrib-trio (==1.1.2)", "sphinxcontrib-websupport"] +docs = ["sphinx (==4.0.2)", "sphinxcontrib-trio (==1.1.2)", "sphinxcontrib-websupport"] +speed = ["orjson (>=3.5.4)"] voice = ["PyNaCl (>=1.3.0,<1.5)"] +[package.source] +type = "url" +url = "https://github.com/Rapptz/discord.py/archive/45d498c1b76deaf3b394d17ccf56112fa691d160.zip" + [[package]] name = "distlib" -version = "0.3.2" +version = "0.3.4" description = "Distribution utilities" category = "dev" optional = false python-versions = "*" [[package]] -name = "docopt" -version = "0.6.2" -description = "Pythonic argument parser, that will make you smile" -category = "dev" -optional = false -python-versions = "*" - -[[package]] name = "emoji" version = "0.6.0" description = "Emoji for Python" @@ -302,19 +315,20 @@ testing = ["pre-commit"] [[package]] name = "fakeredis" -version = "1.5.2" +version = "1.7.0" description = "Fake implementation of redis API for testing purposes." category = "main" optional = false python-versions = ">=3.5" [package.dependencies] -redis = "<3.6.0" +packaging = "*" +redis = "<4.1.0" six = ">=1.12" sortedcontainers = "*" [package.extras] -aioredis = ["aioredis (<2)"] +aioredis = ["aioredis"] lua = ["lupa"] [[package]] @@ -330,11 +344,15 @@ sgmllib3k = "*" [[package]] name = "filelock" -version = "3.0.12" +version = "3.4.2" description = "A platform independent file lock." -category = "dev" +category = "main" optional = false -python-versions = "*" +python-versions = ">=3.7" + +[package.extras] +docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] +testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"] [[package]] name = "flake8" @@ -351,14 +369,14 @@ pyflakes = ">=2.3.0,<2.4.0" [[package]] name = "flake8-annotations" -version = "2.6.2" +version = "2.7.0" description = "Flake8 Type Annotation Checks" category = "dev" optional = false -python-versions = ">=3.6.1,<4.0.0" +python-versions = ">=3.6.2,<4.0.0" [package.dependencies] -flake8 = ">=3.7,<4.0" +flake8 = ">=3.7,<5.0" [[package]] name = "flake8-bugbear" @@ -388,15 +406,20 @@ flake8 = ">=3" pydocstyle = ">=2.1" [[package]] -name = "flake8-import-order" -version = "0.18.1" -description = "Flake8 and pylama plugin that checks the ordering of import statements." +name = "flake8-isort" +version = "4.1.1" +description = "flake8 plugin that integrates isort ." category = "dev" optional = false python-versions = "*" [package.dependencies] -pycodestyle = "*" +flake8 = ">=3.2.1,<5" +isort = ">=4.3.5,<6" +testfixtures = ">=6.8.0,<7" + +[package.extras] +test = ["pytest-cov"] [[package]] name = "flake8-polyfill" @@ -422,14 +445,14 @@ flake8 = "*" [[package]] name = "flake8-tidy-imports" -version = "4.3.0" +version = "4.5.0" description = "A flake8 plugin that helps you write tidier imports." category = "dev" optional = false python-versions = ">=3.6" [package.dependencies] -flake8 = ">=3.0,<3.2.0 || >3.2.0,<4" +flake8 = ">=3.8.0,<5" [[package]] name = "flake8-todo" @@ -452,33 +475,33 @@ python-versions = ">=3.6" [[package]] name = "humanfriendly" -version = "9.2" +version = "10.0" description = "Human friendly output for text interfaces using Python" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] -pyreadline = {version = "*", markers = "sys_platform == \"win32\""} +pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_version >= \"3.8\""} [[package]] name = "identify" -version = "2.2.10" +version = "2.4.2" description = "File identification library for Python" category = "dev" optional = false python-versions = ">=3.6.1" [package.extras] -license = ["editdistance-s"] +license = ["ukkonen"] [[package]] name = "idna" -version = "2.10" +version = "3.3" description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.5" [[package]] name = "iniconfig" @@ -489,8 +512,22 @@ optional = false python-versions = "*" [[package]] +name = "isort" +version = "5.10.1" +description = "A Python utility / library to sort Python imports." +category = "dev" +optional = false +python-versions = ">=3.6.1,<4.0" + +[package.extras] +pipfile_deprecated_finder = ["pipreqs", "requirementslib"] +requirements_deprecated_finder = ["pipreqs", "pip-api"] +colors = ["colorama (>=0.4.3,<0.5.0)"] +plugins = ["setuptools"] + +[[package]] name = "lxml" -version = "4.6.3" +version = "4.7.1" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." category = "main" optional = false @@ -524,7 +561,7 @@ python-versions = "*" [[package]] name = "more-itertools" -version = "8.8.0" +version = "8.12.0" description = "More routines for operating on iterables, beyond itertools" category = "main" optional = false @@ -540,7 +577,7 @@ python-versions = ">=3.5" [[package]] name = "multidict" -version = "5.1.0" +version = "5.2.0" description = "multidict implementation" category = "main" optional = false @@ -564,14 +601,14 @@ python-versions = ">=3.5" [[package]] name = "packaging" -version = "21.0" +version = "21.3" description = "Core utilities for Python packages" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" [package.dependencies] -pyparsing = ">=2.0.2" +pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" [[package]] name = "pamqp" @@ -586,7 +623,7 @@ codegen = ["lxml"] [[package]] name = "pep8-naming" -version = "0.12.0" +version = "0.12.1" description = "Check PEP-8 naming conventions, plugin for flake8" category = "dev" optional = false @@ -597,19 +634,46 @@ flake8 = ">=3.9.1" flake8-polyfill = ">=1.0.2,<2" [[package]] +name = "pip-licenses" +version = "3.5.3" +description = "Dump the software license list of Python packages installed with pip." +category = "dev" +optional = false +python-versions = "~=3.6" + +[package.dependencies] +PTable = "*" + +[package.extras] +test = ["docutils", "pytest-cov", "pytest-pycodestyle", "pytest-runner"] + +[[package]] +name = "platformdirs" +version = "2.4.1" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"] +test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] + +[[package]] name = "pluggy" -version = "0.13.1" +version = "1.0.0" description = "plugin and hook calling mechanisms for python" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.6" [package.extras] dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "pre-commit" -version = "2.13.0" +version = "2.16.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." category = "dev" optional = false @@ -625,7 +689,7 @@ virtualenv = ">=20.0.8" [[package]] name = "psutil" -version = "5.8.0" +version = "5.9.0" description = "Cross-platform lib for process and system monitoring in Python." category = "dev" optional = false @@ -635,16 +699,24 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" test = ["ipaddress", "mock", "unittest2", "enum34", "pywin32", "wmi"] [[package]] +name = "ptable" +version = "0.9.2" +description = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format" +category = "dev" +optional = false +python-versions = "*" + +[[package]] name = "py" -version = "1.10.0" +version = "1.11.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "pycares" -version = "4.0.0" +version = "4.1.2" description = "Python interface for c-ares" category = "main" optional = false @@ -666,7 +738,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pycparser" -version = "2.20" +version = "2.21" description = "C parser in Python" category = "main" optional = false @@ -696,23 +768,26 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pyparsing" -version = "2.4.7" +version = "3.0.6" description = "Python parsing module" -category = "dev" +category = "main" optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +python-versions = ">=3.6" + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] [[package]] -name = "pyreadline" -version = "2.1" -description = "A python implmementation of GNU readline." +name = "pyreadline3" +version = "3.3" +description = "A python implementation of GNU readline." category = "main" optional = false python-versions = "*" [[package]] name = "pytest" -version = "6.2.4" +version = "6.2.5" description = "pytest: simple powerful testing with Python" category = "dev" optional = false @@ -724,7 +799,7 @@ attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<1.0.0a1" +pluggy = ">=0.12,<2.0" py = ">=1.8.2" toml = "*" @@ -749,11 +824,11 @@ testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtuale [[package]] name = "pytest-forked" -version = "1.3.0" +version = "1.4.0" description = "run tests in isolated forked subprocesses" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.6" [package.dependencies] py = "*" @@ -779,7 +854,7 @@ testing = ["filelock"] [[package]] name = "python-dateutil" -version = "2.8.1" +version = "2.8.2" description = "Extensions to the standard Python datetime module" category = "main" optional = false @@ -824,22 +899,28 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [[package]] name = "rapidfuzz" -version = "1.4.1" +version = "1.9.1" description = "rapid fuzzy string matching" category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=2.7" + +[package.extras] +full = ["numpy"] [[package]] name = "redis" -version = "3.5.3" -description = "Python client for Redis key-value store" +version = "4.0.2" +description = "Python client for Redis database and key-value store" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.6" + +[package.dependencies] +deprecated = "*" [package.extras] -hiredis = ["hiredis (>=0.1.3)"] +hiredis = ["hiredis (>=1.0.0)"] [[package]] name = "regex" @@ -851,25 +932,37 @@ python-versions = "*" [[package]] name = "requests" -version = "2.25.1" +version = "2.27.1" description = "Python HTTP for Humans." -category = "dev" +category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.dependencies] certifi = ">=2017.4.17" -chardet = ">=3.0.2,<5" -idna = ">=2.5,<3" +charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} +idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} urllib3 = ">=1.21.1,<1.27" [package.extras] -security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] + +[[package]] +name = "requests-file" +version = "1.5.1" +description = "File transport adapter for Requests" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +requests = ">=1.0.0" +six = "*" [[package]] name = "sentry-sdk" -version = "0.20.3" +version = "1.5.1" description = "Python client for Sentry (https://sentry.io)" category = "main" optional = false @@ -888,6 +981,7 @@ chalice = ["chalice (>=1.16.0)"] django = ["django (>=1.8)"] falcon = ["falcon (>=1.4)"] flask = ["flask (>=0.11)", "blinker (>=1.1)"] +httpx = ["httpx (>=0.16.0)"] pure_eval = ["pure-eval", "executing", "asttokens"] pyspark = ["pyspark (>=2.4.4)"] rq = ["rq (>=0.6)"] @@ -913,7 +1007,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "snowballstemmer" -version = "2.1.0" +version = "2.2.0" description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." category = "dev" optional = false @@ -929,7 +1023,7 @@ python-versions = "*" [[package]] name = "soupsieve" -version = "2.2.1" +version = "2.3.1" description = "A modern CSS selector implementation for Beautiful Soup." category = "main" optional = false @@ -957,6 +1051,33 @@ psutil = ">=5.7.2,<6.0.0" toml = ">=0.10.0,<0.11.0" [[package]] +name = "testfixtures" +version = "6.18.3" +description = "A collection of helpers and mock objects for unit tests and doc tests." +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +build = ["setuptools-git", "wheel", "twine"] +docs = ["sphinx", "zope.component", "sybil", "twisted", "mock", "django (<2)", "django"] +test = ["pytest (>=3.6)", "pytest-cov", "pytest-django", "zope.component", "sybil", "twisted", "mock", "django (<2)", "django"] + +[[package]] +name = "tldextract" +version = "3.1.2" +description = "Accurately separate the TLD from the registered domain and subdomains of a URL, using the Public Suffix List. By default, this includes the public ICANN TLDs and their exceptions. You can optionally support the Public Suffix List's private domains as well." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +filelock = ">=3.0.8" +idna = "*" +requests = ">=2.1.0" +requests-file = ">=1.4" + +[[package]] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" @@ -966,15 +1087,15 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "typing-extensions" -version = "3.10.0.0" -description = "Backported and Experimental Type Hints for Python 3.5+" +version = "4.0.1" +description = "Backported and Experimental Type Hints for Python 3.6+" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.6" [[package]] name = "urllib3" -version = "1.26.6" +version = "1.26.8" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false @@ -987,25 +1108,33 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.4.7" +version = "20.13.0" description = "Virtual Python Environment builder" category = "dev" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [package.dependencies] -appdirs = ">=1.4.3,<2" distlib = ">=0.3.1,<1" -filelock = ">=3.0.0,<4" +filelock = ">=3.2,<4" +platformdirs = ">=2,<3" six = ">=1.9.0,<2" [package.extras] -docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=19.9.0rc1)"] -testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)", "xonsh (>=0.9.16)"] +docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=21.3)"] +testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"] + +[[package]] +name = "wrapt" +version = "1.13.3" +description = "Module for decorators, wrappers and monkey patching." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [[package]] name = "yarl" -version = "1.6.3" +version = "1.7.2" description = "Yet another URL library" category = "main" optional = false @@ -1018,12 +1147,12 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "3.9.*" -content-hash = "85160036e3b07c9d5d24a32302462591e82cc3bf3d5490b87550d9c26bc5648d" +content-hash = "0248fc7488c79af0cdb3a6db9528f4c3129db50b3a8d1dd3ba57dbc31b381c31" [metadata.files] aio-pika = [ - {file = "aio-pika-6.8.0.tar.gz", hash = "sha256:1d4305a5f78af3857310b4fe48348cdcf6c097e0e275ea88c2cd08570531a369"}, - {file = "aio_pika-6.8.0-py3-none-any.whl", hash = "sha256:e69afef8695f47c5d107bbdba21bdb845d5c249acb3be53ef5c2d497b02657c0"}, + {file = "aio-pika-6.8.1.tar.gz", hash = "sha256:c2b2b46949a34252ff0e64c3bc208eef1893e5791b51aeefabf1676788d56b66"}, + {file = "aio_pika-6.8.1-py3-none-any.whl", hash = "sha256:059ab8ecc03d73997f64ed28df7269105984232174d0e6406389c4e8ed30941c"}, ] aiodns = [ {file = "aiodns-2.0.0-py2.py3-none-any.whl", hash = "sha256:aaa5ac584f40fe778013df0aa6544bf157799bd3f608364b451840ed2c8688de"}, @@ -1076,10 +1205,6 @@ aiormq = [ {file = "aiormq-3.3.1-py3-none-any.whl", hash = "sha256:e584dac13a242589aaf42470fd3006cb0dc5aed6506cbd20357c7ec8bbe4a89e"}, {file = "aiormq-3.3.1.tar.gz", hash = "sha256:8218dd9f7198d6e7935855468326bbacf0089f926c70baa8dd92944cb2496573"}, ] -appdirs = [ - {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, - {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, -] arrow = [ {file = "arrow-1.0.3-py3-none-any.whl", hash = "sha256:3515630f11a15c61dcb4cdd245883270dd334c83f3e639824e65a4b79cc48543"}, {file = "arrow-1.0.3.tar.gz", hash = "sha256:399c9c8ae732270e1aa58ead835a79a40d7be8aa109c579898eb41029b5a231d"}, @@ -1097,65 +1222,82 @@ atomicwrites = [ {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, ] attrs = [ - {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, - {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, + {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, + {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, ] beautifulsoup4 = [ - {file = "beautifulsoup4-4.9.3-py2-none-any.whl", hash = "sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35"}, - {file = "beautifulsoup4-4.9.3-py3-none-any.whl", hash = "sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"}, - {file = "beautifulsoup4-4.9.3.tar.gz", hash = "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25"}, + {file = "beautifulsoup4-4.10.0-py3-none-any.whl", hash = "sha256:9a315ce70049920ea4572a4055bc4bd700c940521d36fc858205ad4fcde149bf"}, + {file = "beautifulsoup4-4.10.0.tar.gz", hash = "sha256:c23ad23c521d818955a4151a67d81580319d4bf548d3d49f4223ae041ff98891"}, ] +bot-core = [] certifi = [ - {file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"}, - {file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"}, + {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, + {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, ] cffi = [ - {file = "cffi-1.14.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991"}, - {file = "cffi-1.14.5-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1"}, - {file = "cffi-1.14.5-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:99cd03ae7988a93dd00bcd9d0b75e1f6c426063d6f03d2f90b89e29b25b82dfa"}, - {file = "cffi-1.14.5-cp27-cp27m-win32.whl", hash = "sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3"}, - {file = "cffi-1.14.5-cp27-cp27m-win_amd64.whl", hash = "sha256:51182f8927c5af975fece87b1b369f722c570fe169f9880764b1ee3bca8347b5"}, - {file = "cffi-1.14.5-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482"}, - {file = "cffi-1.14.5-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6"}, - {file = "cffi-1.14.5-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045"}, - {file = "cffi-1.14.5-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:a465da611f6fa124963b91bf432d960a555563efe4ed1cc403ba5077b15370aa"}, - {file = "cffi-1.14.5-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406"}, - {file = "cffi-1.14.5-cp35-cp35m-win32.whl", hash = "sha256:72d8d3ef52c208ee1c7b2e341f7d71c6fd3157138abf1a95166e6165dd5d4369"}, - {file = "cffi-1.14.5-cp35-cp35m-win_amd64.whl", hash = "sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315"}, - {file = "cffi-1.14.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892"}, - {file = "cffi-1.14.5-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058"}, - {file = "cffi-1.14.5-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5"}, - {file = "cffi-1.14.5-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:9e93e79c2551ff263400e1e4be085a1210e12073a31c2011dbbda14bda0c6132"}, - {file = "cffi-1.14.5-cp36-cp36m-win32.whl", hash = "sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53"}, - {file = "cffi-1.14.5-cp36-cp36m-win_amd64.whl", hash = "sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813"}, - {file = "cffi-1.14.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73"}, - {file = "cffi-1.14.5-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06"}, - {file = "cffi-1.14.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1"}, - {file = "cffi-1.14.5-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49"}, - {file = "cffi-1.14.5-cp37-cp37m-win32.whl", hash = "sha256:9ff227395193126d82e60319a673a037d5de84633f11279e336f9c0f189ecc62"}, - {file = "cffi-1.14.5-cp37-cp37m-win_amd64.whl", hash = "sha256:9cf8022fb8d07a97c178b02327b284521c7708d7c71a9c9c355c178ac4bbd3d4"}, - {file = "cffi-1.14.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b198cec6c72df5289c05b05b8b0969819783f9418e0409865dac47288d2a053"}, - {file = "cffi-1.14.5-cp38-cp38-manylinux1_i686.whl", hash = "sha256:ad17025d226ee5beec591b52800c11680fca3df50b8b29fe51d882576e039ee0"}, - {file = "cffi-1.14.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e"}, - {file = "cffi-1.14.5-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8ae6299f6c68de06f136f1f9e69458eae58f1dacf10af5c17353eae03aa0d827"}, - {file = "cffi-1.14.5-cp38-cp38-win32.whl", hash = "sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e"}, - {file = "cffi-1.14.5-cp38-cp38-win_amd64.whl", hash = "sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396"}, - {file = "cffi-1.14.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea"}, - {file = "cffi-1.14.5-cp39-cp39-manylinux1_i686.whl", hash = "sha256:9de2e279153a443c656f2defd67769e6d1e4163952b3c622dcea5b08a6405322"}, - {file = "cffi-1.14.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c"}, - {file = "cffi-1.14.5-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee"}, - {file = "cffi-1.14.5-cp39-cp39-win32.whl", hash = "sha256:afb29c1ba2e5a3736f1c301d9d0abe3ec8b86957d04ddfa9d7a6a42b9367e396"}, - {file = "cffi-1.14.5-cp39-cp39-win_amd64.whl", hash = "sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d"}, - {file = "cffi-1.14.5.tar.gz", hash = "sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c"}, + {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"}, + {file = "cffi-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0"}, + {file = "cffi-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14"}, + {file = "cffi-1.15.0-cp27-cp27m-win32.whl", hash = "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474"}, + {file = "cffi-1.15.0-cp27-cp27m-win_amd64.whl", hash = "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6"}, + {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27"}, + {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023"}, + {file = "cffi-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2"}, + {file = "cffi-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382"}, + {file = "cffi-1.15.0-cp310-cp310-win32.whl", hash = "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55"}, + {file = "cffi-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0"}, + {file = "cffi-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605"}, + {file = "cffi-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e"}, + {file = "cffi-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc"}, + {file = "cffi-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7"}, + {file = "cffi-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66"}, + {file = "cffi-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029"}, + {file = "cffi-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6"}, + {file = "cffi-1.15.0-cp38-cp38-win32.whl", hash = "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c"}, + {file = "cffi-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443"}, + {file = "cffi-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a"}, + {file = "cffi-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8"}, + {file = "cffi-1.15.0-cp39-cp39-win32.whl", hash = "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a"}, + {file = "cffi-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139"}, + {file = "cffi-1.15.0.tar.gz", hash = "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954"}, ] cfgv = [ - {file = "cfgv-3.3.0-py2.py3-none-any.whl", hash = "sha256:b449c9c6118fe8cca7fa5e00b9ec60ba08145d281d52164230a69211c5d597a1"}, - {file = "cfgv-3.3.0.tar.gz", hash = "sha256:9e600479b3b99e8af981ecdfc80a0296104ee610cab48a5ae4ffd0b668650eb1"}, + {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, + {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, ] chardet = [ {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"}, {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, ] +charset-normalizer = [ + {file = "charset-normalizer-2.0.10.tar.gz", hash = "sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd"}, + {file = "charset_normalizer-2.0.10-py3-none-any.whl", hash = "sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455"}, +] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, @@ -1218,24 +1360,18 @@ coverage = [ {file = "coverage-5.5-pp37-none-any.whl", hash = "sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4"}, {file = "coverage-5.5.tar.gz", hash = "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c"}, ] -coveralls = [ - {file = "coveralls-2.2.0-py2.py3-none-any.whl", hash = "sha256:2301a19500b06649d2ec4f2858f9c69638d7699a4c63027c5d53daba666147cc"}, - {file = "coveralls-2.2.0.tar.gz", hash = "sha256:b990ba1f7bc4288e63340be0433698c1efe8217f78c689d254c2540af3d38617"}, -] deepdiff = [ {file = "deepdiff-4.3.2-py3-none-any.whl", hash = "sha256:59fc1e3e7a28dd0147b0f2b00e3e27181f0f0ef4286b251d5f214a5bcd9a9bc4"}, {file = "deepdiff-4.3.2.tar.gz", hash = "sha256:91360be1d9d93b1d9c13ae9c5048fa83d9cff17a88eb30afaa0d7ff2d0fee17d"}, ] -"discord.py" = [ - {file = "discord.py-1.7.3-py3-none-any.whl", hash = "sha256:c6f64db136de0e18e090f6752ea68bdd4ab0a61b82dfe7acecefa22d6477bb0c"}, - {file = "discord.py-1.7.3.tar.gz", hash = "sha256:462cd0fe307aef8b29cbfa8dd613e548ae4b2cb581d46da9ac0d46fb6ea19408"}, +deprecated = [ + {file = "Deprecated-1.2.13-py2.py3-none-any.whl", hash = "sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d"}, + {file = "Deprecated-1.2.13.tar.gz", hash = "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d"}, ] +"discord.py" = [] distlib = [ - {file = "distlib-0.3.2-py2.py3-none-any.whl", hash = "sha256:23e223426b28491b1ced97dc3bbe183027419dfc7982b4fa2f05d5f3ff10711c"}, - {file = "distlib-0.3.2.zip", hash = "sha256:106fef6dc37dd8c0e2c0a60d3fca3e77460a48907f335fa28420463a6f799736"}, -] -docopt = [ - {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"}, + {file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"}, + {file = "distlib-0.3.4.zip", hash = "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579"}, ] emoji = [ {file = "emoji-0.6.0.tar.gz", hash = "sha256:e42da4f8d648f8ef10691bc246f682a1ec6b18373abfd9be10ec0b398823bd11"}, @@ -1245,24 +1381,24 @@ execnet = [ {file = "execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"}, ] fakeredis = [ - {file = "fakeredis-1.5.2-py3-none-any.whl", hash = "sha256:f1ffdb134538e6d7c909ddfb4fc5edeb4a73d0ea07245bc69b8135fbc4144b04"}, - {file = "fakeredis-1.5.2.tar.gz", hash = "sha256:18fc1808d2ce72169d3f11acdb524a00ef96bd29970c6d34cfeb2edb3fc0c020"}, + {file = "fakeredis-1.7.0-py3-none-any.whl", hash = "sha256:6f1e04f64557ad3b6835bdc6e5a8d022cbace4bdc24a47ad58f6a72e0fbff760"}, + {file = "fakeredis-1.7.0.tar.gz", hash = "sha256:c9bd12e430336cbd3e189fae0e91eb99997b93e76dbfdd6ed67fa352dc684c71"}, ] feedparser = [ {file = "feedparser-6.0.8-py3-none-any.whl", hash = "sha256:1b7f57841d9cf85074deb316ed2c795091a238adb79846bc46dccdaf80f9c59a"}, {file = "feedparser-6.0.8.tar.gz", hash = "sha256:5ce0410a05ab248c8c7cfca3a0ea2203968ee9ff4486067379af4827a59f9661"}, ] filelock = [ - {file = "filelock-3.0.12-py3-none-any.whl", hash = "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836"}, - {file = "filelock-3.0.12.tar.gz", hash = "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59"}, + {file = "filelock-3.4.2-py3-none-any.whl", hash = "sha256:cf0fc6a2f8d26bd900f19bf33915ca70ba4dd8c56903eeb14e1e7a2fd7590146"}, + {file = "filelock-3.4.2.tar.gz", hash = "sha256:38b4f4c989f9d06d44524df1b24bd19e167d851f19b50bf3e3559952dddc5b80"}, ] flake8 = [ {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, ] flake8-annotations = [ - {file = "flake8-annotations-2.6.2.tar.gz", hash = "sha256:0d6cd2e770b5095f09689c9d84cc054c51b929c41a68969ea1beb4b825cac515"}, - {file = "flake8_annotations-2.6.2-py3-none-any.whl", hash = "sha256:d10c4638231f8a50c0a597c4efce42bd7b7d85df4f620a0ddaca526138936a4f"}, + {file = "flake8-annotations-2.7.0.tar.gz", hash = "sha256:52e53c05b0c06cac1c2dec192ea2c36e85081238add3bd99421d56f574b9479b"}, + {file = "flake8_annotations-2.7.0-py3-none-any.whl", hash = "sha256:3edfbbfb58e404868834fe6ec3eaf49c139f64f0701259f707d043185545151e"}, ] flake8-bugbear = [ {file = "flake8-bugbear-20.11.1.tar.gz", hash = "sha256:528020129fea2dea33a466b9d64ab650aa3e5f9ffc788b70ea4bc6cf18283538"}, @@ -1272,9 +1408,9 @@ flake8-docstrings = [ {file = "flake8-docstrings-1.6.0.tar.gz", hash = "sha256:9fe7c6a306064af8e62a055c2f61e9eb1da55f84bb39caef2b84ce53708ac34b"}, {file = "flake8_docstrings-1.6.0-py2.py3-none-any.whl", hash = "sha256:99cac583d6c7e32dd28bbfbef120a7c0d1b6dde4adb5a9fd441c4227a6534bde"}, ] -flake8-import-order = [ - {file = "flake8-import-order-0.18.1.tar.gz", hash = "sha256:a28dc39545ea4606c1ac3c24e9d05c849c6e5444a50fb7e9cdd430fc94de6e92"}, - {file = "flake8_import_order-0.18.1-py2.py3-none-any.whl", hash = "sha256:90a80e46886259b9c396b578d75c749801a41ee969a235e163cfe1be7afd2543"}, +flake8-isort = [ + {file = "flake8-isort-4.1.1.tar.gz", hash = "sha256:d814304ab70e6e58859bc5c3e221e2e6e71c958e7005239202fee19c24f82717"}, + {file = "flake8_isort-4.1.1-py3-none-any.whl", hash = "sha256:c4e8b6dcb7be9b71a02e6e5d4196cefcef0f3447be51e82730fb336fff164949"}, ] flake8-polyfill = [ {file = "flake8-polyfill-1.0.2.tar.gz", hash = "sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda"}, @@ -1285,8 +1421,8 @@ flake8-string-format = [ {file = "flake8_string_format-0.3.0-py2.py3-none-any.whl", hash = "sha256:812ff431f10576a74c89be4e85b8e075a705be39bc40c4b4278b5b13e2afa9af"}, ] flake8-tidy-imports = [ - {file = "flake8-tidy-imports-4.3.0.tar.gz", hash = "sha256:e66d46f58ed108f36da920e7781a728dc2d8e4f9269e7e764274105700c0a90c"}, - {file = "flake8_tidy_imports-4.3.0-py3-none-any.whl", hash = "sha256:d6e64cb565ca9474d13d5cb3f838b8deafb5fed15906998d4a674daf55bd6d89"}, + {file = "flake8-tidy-imports-4.5.0.tar.gz", hash = "sha256:ac637961d0f319012d099e49619f8c928e3221f74e00fe6eb89513bc64c40adb"}, + {file = "flake8_tidy_imports-4.5.0-py3-none-any.whl", hash = "sha256:87eed94ae6a2fda6a5918d109746feadf1311e0eb8274ab7a7920f6db00a41c9"}, ] flake8-todo = [ {file = "flake8-todo-0.7.tar.gz", hash = "sha256:6e4c5491ff838c06fe5a771b0e95ee15fc005ca57196011011280fc834a85915"}, @@ -1335,68 +1471,86 @@ hiredis = [ {file = "hiredis-2.0.0.tar.gz", hash = "sha256:81d6d8e39695f2c37954d1011c0480ef7cf444d4e3ae24bc5e89ee5de360139a"}, ] humanfriendly = [ - {file = "humanfriendly-9.2-py2.py3-none-any.whl", hash = "sha256:332da98c24cc150efcc91b5508b19115209272bfdf4b0764a56795932f854271"}, - {file = "humanfriendly-9.2.tar.gz", hash = "sha256:f7dba53ac7935fd0b4a2fc9a29e316ddd9ea135fb3052d3d0279d10c18ff9c48"}, + {file = "humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477"}, + {file = "humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"}, ] identify = [ - {file = "identify-2.2.10-py2.py3-none-any.whl", hash = "sha256:18d0c531ee3dbc112fa6181f34faa179de3f57ea57ae2899754f16a7e0ff6421"}, - {file = "identify-2.2.10.tar.gz", hash = "sha256:5b41f71471bc738e7b586308c3fca172f78940195cb3bf6734c1e66fdac49306"}, + {file = "identify-2.4.2-py2.py3-none-any.whl", hash = "sha256:67c1e66225870dce721228176637a8ef965e8dd58450bcc7592249d0dfc4da6c"}, + {file = "identify-2.4.2.tar.gz", hash = "sha256:93e8ec965e888f2212aa5c24b2b662f4832c39acb1d7196a70ea45acb626a05e"}, ] idna = [ - {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, - {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, + {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, + {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] +isort = [ + {file = "isort-5.10.1-py3-none-any.whl", hash = "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7"}, + {file = "isort-5.10.1.tar.gz", hash = "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951"}, +] lxml = [ - {file = "lxml-4.6.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:df7c53783a46febb0e70f6b05df2ba104610f2fb0d27023409734a3ecbb78fb2"}, - {file = "lxml-4.6.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:1b7584d421d254ab86d4f0b13ec662a9014397678a7c4265a02a6d7c2b18a75f"}, - {file = "lxml-4.6.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:079f3ae844f38982d156efce585bc540c16a926d4436712cf4baee0cce487a3d"}, - {file = "lxml-4.6.3-cp27-cp27m-win32.whl", hash = "sha256:bc4313cbeb0e7a416a488d72f9680fffffc645f8a838bd2193809881c67dd106"}, - {file = "lxml-4.6.3-cp27-cp27m-win_amd64.whl", hash = "sha256:8157dadbb09a34a6bd95a50690595e1fa0af1a99445e2744110e3dca7831c4ee"}, - {file = "lxml-4.6.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7728e05c35412ba36d3e9795ae8995e3c86958179c9770e65558ec3fdfd3724f"}, - {file = "lxml-4.6.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:4bff24dfeea62f2e56f5bab929b4428ae6caba2d1eea0c2d6eb618e30a71e6d4"}, - {file = "lxml-4.6.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:74f7d8d439b18fa4c385f3f5dfd11144bb87c1da034a466c5b5577d23a1d9b51"}, - {file = "lxml-4.6.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f90ba11136bfdd25cae3951af8da2e95121c9b9b93727b1b896e3fa105b2f586"}, - {file = "lxml-4.6.3-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:4c61b3a0db43a1607d6264166b230438f85bfed02e8cff20c22e564d0faff354"}, - {file = "lxml-4.6.3-cp35-cp35m-manylinux2014_x86_64.whl", hash = "sha256:5c8c163396cc0df3fd151b927e74f6e4acd67160d6c33304e805b84293351d16"}, - {file = "lxml-4.6.3-cp35-cp35m-win32.whl", hash = "sha256:f2380a6376dfa090227b663f9678150ef27543483055cc327555fb592c5967e2"}, - {file = "lxml-4.6.3-cp35-cp35m-win_amd64.whl", hash = "sha256:c4f05c5a7c49d2fb70223d0d5bcfbe474cf928310ac9fa6a7c6dddc831d0b1d4"}, - {file = "lxml-4.6.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d2e35d7bf1c1ac8c538f88d26b396e73dd81440d59c1ef8522e1ea77b345ede4"}, - {file = "lxml-4.6.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:289e9ca1a9287f08daaf796d96e06cb2bc2958891d7911ac7cae1c5f9e1e0ee3"}, - {file = "lxml-4.6.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:bccbfc27563652de7dc9bdc595cb25e90b59c5f8e23e806ed0fd623755b6565d"}, - {file = "lxml-4.6.3-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:d916d31fd85b2f78c76400d625076d9124de3e4bda8b016d25a050cc7d603f24"}, - {file = "lxml-4.6.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:820628b7b3135403540202e60551e741f9b6d3304371712521be939470b454ec"}, - {file = "lxml-4.6.3-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:c47ff7e0a36d4efac9fd692cfa33fbd0636674c102e9e8d9b26e1b93a94e7617"}, - {file = "lxml-4.6.3-cp36-cp36m-win32.whl", hash = "sha256:5a0a14e264069c03e46f926be0d8919f4105c1623d620e7ec0e612a2e9bf1c04"}, - {file = "lxml-4.6.3-cp36-cp36m-win_amd64.whl", hash = "sha256:92e821e43ad382332eade6812e298dc9701c75fe289f2a2d39c7960b43d1e92a"}, - {file = "lxml-4.6.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:efd7a09678fd8b53117f6bae4fa3825e0a22b03ef0a932e070c0bdbb3a35e654"}, - {file = "lxml-4.6.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:efac139c3f0bf4f0939f9375af4b02c5ad83a622de52d6dfa8e438e8e01d0eb0"}, - {file = "lxml-4.6.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:0fbcf5565ac01dff87cbfc0ff323515c823081c5777a9fc7703ff58388c258c3"}, - {file = "lxml-4.6.3-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:36108c73739985979bf302006527cf8a20515ce444ba916281d1c43938b8bb96"}, - {file = "lxml-4.6.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:122fba10466c7bd4178b07dba427aa516286b846b2cbd6f6169141917283aae2"}, - {file = "lxml-4.6.3-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:cdaf11d2bd275bf391b5308f86731e5194a21af45fbaaaf1d9e8147b9160ea92"}, - {file = "lxml-4.6.3-cp37-cp37m-win32.whl", hash = "sha256:3439c71103ef0e904ea0a1901611863e51f50b5cd5e8654a151740fde5e1cade"}, - {file = "lxml-4.6.3-cp37-cp37m-win_amd64.whl", hash = "sha256:4289728b5e2000a4ad4ab8da6e1db2e093c63c08bdc0414799ee776a3f78da4b"}, - {file = "lxml-4.6.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b007cbb845b28db4fb8b6a5cdcbf65bacb16a8bd328b53cbc0698688a68e1caa"}, - {file = "lxml-4.6.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:76fa7b1362d19f8fbd3e75fe2fb7c79359b0af8747e6f7141c338f0bee2f871a"}, - {file = "lxml-4.6.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:26e761ab5b07adf5f555ee82fb4bfc35bf93750499c6c7614bd64d12aaa67927"}, - {file = "lxml-4.6.3-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:e1cbd3f19a61e27e011e02f9600837b921ac661f0c40560eefb366e4e4fb275e"}, - {file = "lxml-4.6.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:66e575c62792c3f9ca47cb8b6fab9e35bab91360c783d1606f758761810c9791"}, - {file = "lxml-4.6.3-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:1b38116b6e628118dea5b2186ee6820ab138dbb1e24a13e478490c7db2f326ae"}, - {file = "lxml-4.6.3-cp38-cp38-win32.whl", hash = "sha256:89b8b22a5ff72d89d48d0e62abb14340d9e99fd637d046c27b8b257a01ffbe28"}, - {file = "lxml-4.6.3-cp38-cp38-win_amd64.whl", hash = "sha256:2a9d50e69aac3ebee695424f7dbd7b8c6d6eb7de2a2eb6b0f6c7db6aa41e02b7"}, - {file = "lxml-4.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ce256aaa50f6cc9a649c51be3cd4ff142d67295bfc4f490c9134d0f9f6d58ef0"}, - {file = "lxml-4.6.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:7610b8c31688f0b1be0ef882889817939490a36d0ee880ea562a4e1399c447a1"}, - {file = "lxml-4.6.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f8380c03e45cf09f8557bdaa41e1fa7c81f3ae22828e1db470ab2a6c96d8bc23"}, - {file = "lxml-4.6.3-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:3082c518be8e97324390614dacd041bb1358c882d77108ca1957ba47738d9d59"}, - {file = "lxml-4.6.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:884ab9b29feaca361f7f88d811b1eea9bfca36cf3da27768d28ad45c3ee6f969"}, - {file = "lxml-4.6.3-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:6f12e1427285008fd32a6025e38e977d44d6382cf28e7201ed10d6c1698d2a9a"}, - {file = "lxml-4.6.3-cp39-cp39-win32.whl", hash = "sha256:33bb934a044cf32157c12bfcfbb6649807da20aa92c062ef51903415c704704f"}, - {file = "lxml-4.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:542d454665a3e277f76954418124d67516c5f88e51a900365ed54a9806122b83"}, - {file = "lxml-4.6.3.tar.gz", hash = "sha256:39b78571b3b30645ac77b95f7c69d1bffc4cf8c3b157c435a34da72e78c82468"}, + {file = "lxml-4.7.1-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:d546431636edb1d6a608b348dd58cc9841b81f4116745857b6cb9f8dadb2725f"}, + {file = "lxml-4.7.1-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6308062534323f0d3edb4e702a0e26a76ca9e0e23ff99be5d82750772df32a9e"}, + {file = "lxml-4.7.1-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:f76dbe44e31abf516114f6347a46fa4e7c2e8bceaa4b6f7ee3a0a03c8eba3c17"}, + {file = "lxml-4.7.1-cp27-cp27m-win32.whl", hash = "sha256:d5618d49de6ba63fe4510bdada62d06a8acfca0b4b5c904956c777d28382b419"}, + {file = "lxml-4.7.1-cp27-cp27m-win_amd64.whl", hash = "sha256:9393a05b126a7e187f3e38758255e0edf948a65b22c377414002d488221fdaa2"}, + {file = "lxml-4.7.1-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:50d3dba341f1e583265c1a808e897b4159208d814ab07530202b6036a4d86da5"}, + {file = "lxml-4.7.1-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:44f552e0da3c8ee3c28e2eb82b0b784200631687fc6a71277ea8ab0828780e7d"}, + {file = "lxml-4.7.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e662c6266e3a275bdcb6bb049edc7cd77d0b0f7e119a53101d367c841afc66dc"}, + {file = "lxml-4.7.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:4c093c571bc3da9ebcd484e001ba18b8452903cd428c0bc926d9b0141bcb710e"}, + {file = "lxml-4.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:3e26ad9bc48d610bf6cc76c506b9e5ad9360ed7a945d9be3b5b2c8535a0145e3"}, + {file = "lxml-4.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a5f623aeaa24f71fce3177d7fee875371345eb9102b355b882243e33e04b7175"}, + {file = "lxml-4.7.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7b5e2acefd33c259c4a2e157119c4373c8773cf6793e225006a1649672ab47a6"}, + {file = "lxml-4.7.1-cp310-cp310-win32.whl", hash = "sha256:67fa5f028e8a01e1d7944a9fb616d1d0510d5d38b0c41708310bd1bc45ae89f6"}, + {file = "lxml-4.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:b1d381f58fcc3e63fcc0ea4f0a38335163883267f77e4c6e22d7a30877218a0e"}, + {file = "lxml-4.7.1-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:38d9759733aa04fb1697d717bfabbedb21398046bd07734be7cccc3d19ea8675"}, + {file = "lxml-4.7.1-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:dfd0d464f3d86a1460683cd742306d1138b4e99b79094f4e07e1ca85ee267fe7"}, + {file = "lxml-4.7.1-cp35-cp35m-win32.whl", hash = "sha256:534e946bce61fd162af02bad7bfd2daec1521b71d27238869c23a672146c34a5"}, + {file = "lxml-4.7.1-cp35-cp35m-win_amd64.whl", hash = "sha256:6ec829058785d028f467be70cd195cd0aaf1a763e4d09822584ede8c9eaa4b03"}, + {file = "lxml-4.7.1-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:ade74f5e3a0fd17df5782896ddca7ddb998845a5f7cd4b0be771e1ffc3b9aa5b"}, + {file = "lxml-4.7.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:41358bfd24425c1673f184d7c26c6ae91943fe51dfecc3603b5e08187b4bcc55"}, + {file = "lxml-4.7.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:6e56521538f19c4a6690f439fefed551f0b296bd785adc67c1777c348beb943d"}, + {file = "lxml-4.7.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5b0f782f0e03555c55e37d93d7a57454efe7495dab33ba0ccd2dbe25fc50f05d"}, + {file = "lxml-4.7.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:490712b91c65988012e866c411a40cc65b595929ececf75eeb4c79fcc3bc80a6"}, + {file = "lxml-4.7.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:34c22eb8c819d59cec4444d9eebe2e38b95d3dcdafe08965853f8799fd71161d"}, + {file = "lxml-4.7.1-cp36-cp36m-win32.whl", hash = "sha256:2a906c3890da6a63224d551c2967413b8790a6357a80bf6b257c9a7978c2c42d"}, + {file = "lxml-4.7.1-cp36-cp36m-win_amd64.whl", hash = "sha256:36b16fecb10246e599f178dd74f313cbdc9f41c56e77d52100d1361eed24f51a"}, + {file = "lxml-4.7.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:a5edc58d631170de90e50adc2cc0248083541affef82f8cd93bea458e4d96db8"}, + {file = "lxml-4.7.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:87c1b0496e8c87ec9db5383e30042357b4839b46c2d556abd49ec770ce2ad868"}, + {file = "lxml-4.7.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:0a5f0e4747f31cff87d1eb32a6000bde1e603107f632ef4666be0dc065889c7a"}, + {file = "lxml-4.7.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:bf6005708fc2e2c89a083f258b97709559a95f9a7a03e59f805dd23c93bc3986"}, + {file = "lxml-4.7.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc15874816b9320581133ddc2096b644582ab870cf6a6ed63684433e7af4b0d3"}, + {file = "lxml-4.7.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:0b5e96e25e70917b28a5391c2ed3ffc6156513d3db0e1476c5253fcd50f7a944"}, + {file = "lxml-4.7.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ec9027d0beb785a35aa9951d14e06d48cfbf876d8ff67519403a2522b181943b"}, + {file = "lxml-4.7.1-cp37-cp37m-win32.whl", hash = "sha256:9fbc0dee7ff5f15c4428775e6fa3ed20003140560ffa22b88326669d53b3c0f4"}, + {file = "lxml-4.7.1-cp37-cp37m-win_amd64.whl", hash = "sha256:1104a8d47967a414a436007c52f533e933e5d52574cab407b1e49a4e9b5ddbd1"}, + {file = "lxml-4.7.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:fc9fb11b65e7bc49f7f75aaba1b700f7181d95d4e151cf2f24d51bfd14410b77"}, + {file = "lxml-4.7.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:317bd63870b4d875af3c1be1b19202de34c32623609ec803b81c99193a788c1e"}, + {file = "lxml-4.7.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:610807cea990fd545b1559466971649e69302c8a9472cefe1d6d48a1dee97440"}, + {file = "lxml-4.7.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:09b738360af8cb2da275998a8bf79517a71225b0de41ab47339c2beebfff025f"}, + {file = "lxml-4.7.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6a2ab9d089324d77bb81745b01f4aeffe4094306d939e92ba5e71e9a6b99b71e"}, + {file = "lxml-4.7.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:eed394099a7792834f0cb4a8f615319152b9d801444c1c9e1b1a2c36d2239f9e"}, + {file = "lxml-4.7.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:735e3b4ce9c0616e85f302f109bdc6e425ba1670a73f962c9f6b98a6d51b77c9"}, + {file = "lxml-4.7.1-cp38-cp38-win32.whl", hash = "sha256:772057fba283c095db8c8ecde4634717a35c47061d24f889468dc67190327bcd"}, + {file = "lxml-4.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:13dbb5c7e8f3b6a2cf6e10b0948cacb2f4c9eb05029fe31c60592d08ac63180d"}, + {file = "lxml-4.7.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:718d7208b9c2d86aaf0294d9381a6acb0158b5ff0f3515902751404e318e02c9"}, + {file = "lxml-4.7.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:5bee1b0cbfdb87686a7fb0e46f1d8bd34d52d6932c0723a86de1cc532b1aa489"}, + {file = "lxml-4.7.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:e410cf3a2272d0a85526d700782a2fa92c1e304fdcc519ba74ac80b8297adf36"}, + {file = "lxml-4.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:585ea241ee4961dc18a95e2f5581dbc26285fcf330e007459688096f76be8c42"}, + {file = "lxml-4.7.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a555e06566c6dc167fbcd0ad507ff05fd9328502aefc963cb0a0547cfe7f00db"}, + {file = "lxml-4.7.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:adaab25be351fff0d8a691c4f09153647804d09a87a4e4ea2c3f9fe9e8651851"}, + {file = "lxml-4.7.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:82d16a64236970cb93c8d63ad18c5b9f138a704331e4b916b2737ddfad14e0c4"}, + {file = "lxml-4.7.1-cp39-cp39-win32.whl", hash = "sha256:59e7da839a1238807226f7143c68a479dee09244d1b3cf8c134f2fce777d12d0"}, + {file = "lxml-4.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:a1bbc4efa99ed1310b5009ce7f3a1784698082ed2c1ef3895332f5df9b3b92c2"}, + {file = "lxml-4.7.1-pp37-pypy37_pp73-macosx_10_14_x86_64.whl", hash = "sha256:0607ff0988ad7e173e5ddf7bf55ee65534bd18a5461183c33e8e41a59e89edf4"}, + {file = "lxml-4.7.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:6c198bfc169419c09b85ab10cb0f572744e686f40d1e7f4ed09061284fc1303f"}, + {file = "lxml-4.7.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a58d78653ae422df6837dd4ca0036610b8cb4962b5cfdbd337b7b24de9e5f98a"}, + {file = "lxml-4.7.1-pp38-pypy38_pp73-macosx_10_14_x86_64.whl", hash = "sha256:e18281a7d80d76b66a9f9e68a98cf7e1d153182772400d9a9ce855264d7d0ce7"}, + {file = "lxml-4.7.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:8e54945dd2eeb50925500957c7c579df3cd07c29db7810b83cf30495d79af267"}, + {file = "lxml-4.7.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:447d5009d6b5447b2f237395d0018901dcc673f7d9f82ba26c1b9f9c3b444b60"}, + {file = "lxml-4.7.1.tar.gz", hash = "sha256:a1613838aa6b89af4ba10a0f3a972836128801ed008078f8c1244e65958f1b24"}, ] markdownify = [ {file = "markdownify-0.6.1-py3-none-any.whl", hash = "sha256:7489fd5c601536996a376c4afbcd1dd034db7690af807120681461e82fbc0acc"}, @@ -1407,51 +1561,86 @@ mccabe = [ {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] more-itertools = [ - {file = "more-itertools-8.8.0.tar.gz", hash = "sha256:83f0308e05477c68f56ea3a888172c78ed5d5b3c282addb67508e7ba6c8f813a"}, - {file = "more_itertools-8.8.0-py3-none-any.whl", hash = "sha256:2cf89ec599962f2ddc4d568a05defc40e0a587fbc10d5989713638864c36be4d"}, + {file = "more-itertools-8.12.0.tar.gz", hash = "sha256:7dc6ad46f05f545f900dd59e8dfb4e84a4827b97b3cfecb175ea0c7d247f6064"}, + {file = "more_itertools-8.12.0-py3-none-any.whl", hash = "sha256:43e6dd9942dffd72661a2c4ef383ad7da1e6a3e968a927ad7a6083ab410a688b"}, ] mslex = [ {file = "mslex-0.3.0-py2.py3-none-any.whl", hash = "sha256:380cb14abf8fabf40e56df5c8b21a6d533dc5cbdcfe42406bbf08dda8f42e42a"}, {file = "mslex-0.3.0.tar.gz", hash = "sha256:4a1ac3f25025cad78ad2fe499dd16d42759f7a3801645399cce5c404415daa97"}, ] multidict = [ - {file = "multidict-5.1.0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:b7993704f1a4b204e71debe6095150d43b2ee6150fa4f44d6d966ec356a8d61f"}, - {file = "multidict-5.1.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:9dd6e9b1a913d096ac95d0399bd737e00f2af1e1594a787e00f7975778c8b2bf"}, - {file = "multidict-5.1.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281"}, - {file = "multidict-5.1.0-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:1ab820665e67373de5802acae069a6a05567ae234ddb129f31d290fc3d1aa56d"}, - {file = "multidict-5.1.0-cp36-cp36m-manylinux2014_ppc64le.whl", hash = "sha256:9436dc58c123f07b230383083855593550c4d301d2532045a17ccf6eca505f6d"}, - {file = "multidict-5.1.0-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:830f57206cc96ed0ccf68304141fec9481a096c4d2e2831f311bde1c404401da"}, - {file = "multidict-5.1.0-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:2e68965192c4ea61fff1b81c14ff712fc7dc15d2bd120602e4a3494ea6584224"}, - {file = "multidict-5.1.0-cp36-cp36m-win32.whl", hash = "sha256:2f1a132f1c88724674271d636e6b7351477c27722f2ed789f719f9e3545a3d26"}, - {file = "multidict-5.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:3a4f32116f8f72ecf2a29dabfb27b23ab7cdc0ba807e8459e59a93a9be9506f6"}, - {file = "multidict-5.1.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:46c73e09ad374a6d876c599f2328161bcd95e280f84d2060cf57991dec5cfe76"}, - {file = "multidict-5.1.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:018132dbd8688c7a69ad89c4a3f39ea2f9f33302ebe567a879da8f4ca73f0d0a"}, - {file = "multidict-5.1.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:4b186eb7d6ae7c06eb4392411189469e6a820da81447f46c0072a41c748ab73f"}, - {file = "multidict-5.1.0-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:3a041b76d13706b7fff23b9fc83117c7b8fe8d5fe9e6be45eee72b9baa75f348"}, - {file = "multidict-5.1.0-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:051012ccee979b2b06be928a6150d237aec75dd6bf2d1eeeb190baf2b05abc93"}, - {file = "multidict-5.1.0-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:6a4d5ce640e37b0efcc8441caeea8f43a06addace2335bd11151bc02d2ee31f9"}, - {file = "multidict-5.1.0-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:5cf3443199b83ed9e955f511b5b241fd3ae004e3cb81c58ec10f4fe47c7dce37"}, - {file = "multidict-5.1.0-cp37-cp37m-win32.whl", hash = "sha256:f200755768dc19c6f4e2b672421e0ebb3dd54c38d5a4f262b872d8cfcc9e93b5"}, - {file = "multidict-5.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:05c20b68e512166fddba59a918773ba002fdd77800cad9f55b59790030bab632"}, - {file = "multidict-5.1.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:54fd1e83a184e19c598d5e70ba508196fd0bbdd676ce159feb412a4a6664f952"}, - {file = "multidict-5.1.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:0e3c84e6c67eba89c2dbcee08504ba8644ab4284863452450520dad8f1e89b79"}, - {file = "multidict-5.1.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:dc862056f76443a0db4509116c5cd480fe1b6a2d45512a653f9a855cc0517456"}, - {file = "multidict-5.1.0-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:0e929169f9c090dae0646a011c8b058e5e5fb391466016b39d21745b48817fd7"}, - {file = "multidict-5.1.0-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:d81eddcb12d608cc08081fa88d046c78afb1bf8107e6feab5d43503fea74a635"}, - {file = "multidict-5.1.0-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:585fd452dd7782130d112f7ddf3473ffdd521414674c33876187e101b588738a"}, - {file = "multidict-5.1.0-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:37e5438e1c78931df5d3c0c78ae049092877e5e9c02dd1ff5abb9cf27a5914ea"}, - {file = "multidict-5.1.0-cp38-cp38-win32.whl", hash = "sha256:07b42215124aedecc6083f1ce6b7e5ec5b50047afa701f3442054373a6deb656"}, - {file = "multidict-5.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:929006d3c2d923788ba153ad0de8ed2e5ed39fdbe8e7be21e2f22ed06c6783d3"}, - {file = "multidict-5.1.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:b797515be8743b771aa868f83563f789bbd4b236659ba52243b735d80b29ed93"}, - {file = "multidict-5.1.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:d5c65bdf4484872c4af3150aeebe101ba560dcfb34488d9a8ff8dbcd21079647"}, - {file = "multidict-5.1.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b47a43177a5e65b771b80db71e7be76c0ba23cc8aa73eeeb089ed5219cdbe27d"}, - {file = "multidict-5.1.0-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:806068d4f86cb06af37cd65821554f98240a19ce646d3cd24e1c33587f313eb8"}, - {file = "multidict-5.1.0-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:46dd362c2f045095c920162e9307de5ffd0a1bfbba0a6e990b344366f55a30c1"}, - {file = "multidict-5.1.0-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:ace010325c787c378afd7f7c1ac66b26313b3344628652eacd149bdd23c68841"}, - {file = "multidict-5.1.0-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:ecc771ab628ea281517e24fd2c52e8f31c41e66652d07599ad8818abaad38cda"}, - {file = "multidict-5.1.0-cp39-cp39-win32.whl", hash = "sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80"}, - {file = "multidict-5.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359"}, - {file = "multidict-5.1.0.tar.gz", hash = "sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5"}, + {file = "multidict-5.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3822c5894c72e3b35aae9909bef66ec83e44522faf767c0ad39e0e2de11d3b55"}, + {file = "multidict-5.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:28e6d883acd8674887d7edc896b91751dc2d8e87fbdca8359591a13872799e4e"}, + {file = "multidict-5.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b61f85101ef08cbbc37846ac0e43f027f7844f3fade9b7f6dd087178caedeee7"}, + {file = "multidict-5.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9b668c065968c5979fe6b6fa6760bb6ab9aeb94b75b73c0a9c1acf6393ac3bf"}, + {file = "multidict-5.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:517d75522b7b18a3385726b54a081afd425d4f41144a5399e5abd97ccafdf36b"}, + {file = "multidict-5.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1b4ac3ba7a97b35a5ccf34f41b5a8642a01d1e55454b699e5e8e7a99b5a3acf5"}, + {file = "multidict-5.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:df23c83398715b26ab09574217ca21e14694917a0c857e356fd39e1c64f8283f"}, + {file = "multidict-5.2.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e58a9b5cc96e014ddf93c2227cbdeca94b56a7eb77300205d6e4001805391747"}, + {file = "multidict-5.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f76440e480c3b2ca7f843ff8a48dc82446b86ed4930552d736c0bac507498a52"}, + {file = "multidict-5.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cfde464ca4af42a629648c0b0d79b8f295cf5b695412451716531d6916461628"}, + {file = "multidict-5.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0fed465af2e0eb6357ba95795d003ac0bdb546305cc2366b1fc8f0ad67cc3fda"}, + {file = "multidict-5.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:b70913cbf2e14275013be98a06ef4b412329fe7b4f83d64eb70dce8269ed1e1a"}, + {file = "multidict-5.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a5635bcf1b75f0f6ef3c8a1ad07b500104a971e38d3683167b9454cb6465ac86"}, + {file = "multidict-5.2.0-cp310-cp310-win32.whl", hash = "sha256:77f0fb7200cc7dedda7a60912f2059086e29ff67cefbc58d2506638c1a9132d7"}, + {file = "multidict-5.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:9416cf11bcd73c861267e88aea71e9fcc35302b3943e45e1dbb4317f91a4b34f"}, + {file = "multidict-5.2.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:fd77c8f3cba815aa69cb97ee2b2ef385c7c12ada9c734b0f3b32e26bb88bbf1d"}, + {file = "multidict-5.2.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98ec9aea6223adf46999f22e2c0ab6cf33f5914be604a404f658386a8f1fba37"}, + {file = "multidict-5.2.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e5283c0a00f48e8cafcecadebfa0ed1dac8b39e295c7248c44c665c16dc1138b"}, + {file = "multidict-5.2.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5f79c19c6420962eb17c7e48878a03053b7ccd7b69f389d5831c0a4a7f1ac0a1"}, + {file = "multidict-5.2.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e4a67f1080123de76e4e97a18d10350df6a7182e243312426d508712e99988d4"}, + {file = "multidict-5.2.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:94b117e27efd8e08b4046c57461d5a114d26b40824995a2eb58372b94f9fca02"}, + {file = "multidict-5.2.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2e77282fd1d677c313ffcaddfec236bf23f273c4fba7cdf198108f5940ae10f5"}, + {file = "multidict-5.2.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:116347c63ba049c1ea56e157fa8aa6edaf5e92925c9b64f3da7769bdfa012858"}, + {file = "multidict-5.2.0-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:dc3a866cf6c13d59a01878cd806f219340f3e82eed514485e094321f24900677"}, + {file = "multidict-5.2.0-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ac42181292099d91217a82e3fa3ce0e0ddf3a74fd891b7c2b347a7f5aa0edded"}, + {file = "multidict-5.2.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:f0bb0973f42ffcb5e3537548e0767079420aefd94ba990b61cf7bb8d47f4916d"}, + {file = "multidict-5.2.0-cp36-cp36m-win32.whl", hash = "sha256:ea21d4d5104b4f840b91d9dc8cbc832aba9612121eaba503e54eaab1ad140eb9"}, + {file = "multidict-5.2.0-cp36-cp36m-win_amd64.whl", hash = "sha256:e6453f3cbeb78440747096f239d282cc57a2997a16b5197c9bc839099e1633d0"}, + {file = "multidict-5.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d3def943bfd5f1c47d51fd324df1e806d8da1f8e105cc7f1c76a1daf0f7e17b0"}, + {file = "multidict-5.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35591729668a303a02b06e8dba0eb8140c4a1bfd4c4b3209a436a02a5ac1de11"}, + {file = "multidict-5.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce8cacda0b679ebc25624d5de66c705bc53dcc7c6f02a7fb0f3ca5e227d80422"}, + {file = "multidict-5.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:baf1856fab8212bf35230c019cde7c641887e3fc08cadd39d32a421a30151ea3"}, + {file = "multidict-5.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a43616aec0f0d53c411582c451f5d3e1123a68cc7b3475d6f7d97a626f8ff90d"}, + {file = "multidict-5.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:25cbd39a9029b409167aa0a20d8a17f502d43f2efebfe9e3ac019fe6796c59ac"}, + {file = "multidict-5.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0a2cbcfbea6dc776782a444db819c8b78afe4db597211298dd8b2222f73e9cd0"}, + {file = "multidict-5.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3d2d7d1fff8e09d99354c04c3fd5b560fb04639fd45926b34e27cfdec678a704"}, + {file = "multidict-5.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a37e9a68349f6abe24130846e2f1d2e38f7ddab30b81b754e5a1fde32f782b23"}, + {file = "multidict-5.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:637c1896497ff19e1ee27c1c2c2ddaa9f2d134bbb5e0c52254361ea20486418d"}, + {file = "multidict-5.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:9815765f9dcda04921ba467957be543423e5ec6a1136135d84f2ae092c50d87b"}, + {file = "multidict-5.2.0-cp37-cp37m-win32.whl", hash = "sha256:8b911d74acdc1fe2941e59b4f1a278a330e9c34c6c8ca1ee21264c51ec9b67ef"}, + {file = "multidict-5.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:380b868f55f63d048a25931a1632818f90e4be71d2081c2338fcf656d299949a"}, + {file = "multidict-5.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e7d81ce5744757d2f05fc41896e3b2ae0458464b14b5a2c1e87a6a9d69aefaa8"}, + {file = "multidict-5.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2d1d55cdf706ddc62822d394d1df53573d32a7a07d4f099470d3cb9323b721b6"}, + {file = "multidict-5.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a4771d0d0ac9d9fe9e24e33bed482a13dfc1256d008d101485fe460359476065"}, + {file = "multidict-5.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da7d57ea65744d249427793c042094c4016789eb2562576fb831870f9c878d9e"}, + {file = "multidict-5.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cdd68778f96216596218b4e8882944d24a634d984ee1a5a049b300377878fa7c"}, + {file = "multidict-5.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ecc99bce8ee42dcad15848c7885197d26841cb24fa2ee6e89d23b8993c871c64"}, + {file = "multidict-5.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:067150fad08e6f2dd91a650c7a49ba65085303fcc3decbd64a57dc13a2733031"}, + {file = "multidict-5.2.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:78c106b2b506b4d895ddc801ff509f941119394b89c9115580014127414e6c2d"}, + {file = "multidict-5.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e6c4fa1ec16e01e292315ba76eb1d012c025b99d22896bd14a66628b245e3e01"}, + {file = "multidict-5.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b227345e4186809d31f22087d0265655114af7cda442ecaf72246275865bebe4"}, + {file = "multidict-5.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:06560fbdcf22c9387100979e65b26fba0816c162b888cb65b845d3def7a54c9b"}, + {file = "multidict-5.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:7878b61c867fb2df7a95e44b316f88d5a3742390c99dfba6c557a21b30180cac"}, + {file = "multidict-5.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:246145bff76cc4b19310f0ad28bd0769b940c2a49fc601b86bfd150cbd72bb22"}, + {file = "multidict-5.2.0-cp38-cp38-win32.whl", hash = "sha256:c30ac9f562106cd9e8071c23949a067b10211917fdcb75b4718cf5775356a940"}, + {file = "multidict-5.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:f19001e790013ed580abfde2a4465388950728861b52f0da73e8e8a9418533c0"}, + {file = "multidict-5.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c1ff762e2ee126e6f1258650ac641e2b8e1f3d927a925aafcfde943b77a36d24"}, + {file = "multidict-5.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bd6c9c50bf2ad3f0448edaa1a3b55b2e6866ef8feca5d8dbec10ec7c94371d21"}, + {file = "multidict-5.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fc66d4016f6e50ed36fb39cd287a3878ffcebfa90008535c62e0e90a7ab713ae"}, + {file = "multidict-5.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9acb76d5f3dd9421874923da2ed1e76041cb51b9337fd7f507edde1d86535d6"}, + {file = "multidict-5.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dfc924a7e946dd3c6360e50e8f750d51e3ef5395c95dc054bc9eab0f70df4f9c"}, + {file = "multidict-5.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32fdba7333eb2351fee2596b756d730d62b5827d5e1ab2f84e6cbb287cc67fe0"}, + {file = "multidict-5.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:b9aad49466b8d828b96b9e3630006234879c8d3e2b0a9d99219b3121bc5cdb17"}, + {file = "multidict-5.2.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:93de39267c4c676c9ebb2057e98a8138bade0d806aad4d864322eee0803140a0"}, + {file = "multidict-5.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f9bef5cff994ca3026fcc90680e326d1a19df9841c5e3d224076407cc21471a1"}, + {file = "multidict-5.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5f841c4f14331fd1e36cbf3336ed7be2cb2a8f110ce40ea253e5573387db7621"}, + {file = "multidict-5.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:38ba256ee9b310da6a1a0f013ef4e422fca30a685bcbec86a969bd520504e341"}, + {file = "multidict-5.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:3bc3b1621b979621cee9f7b09f024ec76ec03cc365e638126a056317470bde1b"}, + {file = "multidict-5.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6ee908c070020d682e9b42c8f621e8bb10c767d04416e2ebe44e37d0f44d9ad5"}, + {file = "multidict-5.2.0-cp39-cp39-win32.whl", hash = "sha256:1c7976cd1c157fa7ba5456ae5d31ccdf1479680dc9b8d8aa28afabc370df42b8"}, + {file = "multidict-5.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:c9631c642e08b9fff1c6255487e62971d8b8e821808ddd013d8ac058087591ac"}, + {file = "multidict-5.2.0.tar.gz", hash = "sha256:0dd1c93edb444b33ba2274b66f63def8a327d607c6c790772f448a53b6ea59ce"}, ] nodeenv = [ {file = "nodeenv-1.6.0-py2.py3-none-any.whl", hash = "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7"}, @@ -1461,101 +1650,114 @@ ordered-set = [ {file = "ordered-set-4.0.2.tar.gz", hash = "sha256:ba93b2df055bca202116ec44b9bead3df33ea63a7d5827ff8e16738b97f33a95"}, ] packaging = [ - {file = "packaging-21.0-py3-none-any.whl", hash = "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"}, - {file = "packaging-21.0.tar.gz", hash = "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7"}, + {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, + {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] pamqp = [ {file = "pamqp-2.3.0-py2.py3-none-any.whl", hash = "sha256:2f81b5c186f668a67f165193925b6bfd83db4363a6222f599517f29ecee60b02"}, {file = "pamqp-2.3.0.tar.gz", hash = "sha256:5cd0f5a85e89f20d5f8e19285a1507788031cfca4a9ea6f067e3cf18f5e294e8"}, ] pep8-naming = [ - {file = "pep8-naming-0.12.0.tar.gz", hash = "sha256:1f9a3ecb2f3fd83240fd40afdd70acc89695c49c333413e49788f93b61827e12"}, - {file = "pep8_naming-0.12.0-py2.py3-none-any.whl", hash = "sha256:2321ac2b7bf55383dd19a6a9c8ae2ebf05679699927a3af33e60dd7d337099d3"}, + {file = "pep8-naming-0.12.1.tar.gz", hash = "sha256:bb2455947757d162aa4cad55dba4ce029005cd1692f2899a21d51d8630ca7841"}, + {file = "pep8_naming-0.12.1-py2.py3-none-any.whl", hash = "sha256:4a8daeaeb33cfcde779309fc0c9c0a68a3bbe2ad8a8308b763c5068f86eb9f37"}, +] +pip-licenses = [ + {file = "pip-licenses-3.5.3.tar.gz", hash = "sha256:f44860e00957b791c6c6005a3328f2d5eaeee96ddb8e7d87d4b0aa25b02252e4"}, + {file = "pip_licenses-3.5.3-py3-none-any.whl", hash = "sha256:59c148d6a03784bf945d232c0dc0e9de4272a3675acaa0361ad7712398ca86ba"}, +] +platformdirs = [ + {file = "platformdirs-2.4.1-py3-none-any.whl", hash = "sha256:1d7385c7db91728b83efd0ca99a5afb296cab9d0ed8313a45ed8ba17967ecfca"}, + {file = "platformdirs-2.4.1.tar.gz", hash = "sha256:440633ddfebcc36264232365d7840a970e75e1018d15b4327d11f91909045fda"}, ] pluggy = [ - {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, - {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] pre-commit = [ - {file = "pre_commit-2.13.0-py2.py3-none-any.whl", hash = "sha256:b679d0fddd5b9d6d98783ae5f10fd0c4c59954f375b70a58cbe1ce9bcf9809a4"}, - {file = "pre_commit-2.13.0.tar.gz", hash = "sha256:764972c60693dc668ba8e86eb29654ec3144501310f7198742a767bec385a378"}, + {file = "pre_commit-2.16.0-py2.py3-none-any.whl", hash = "sha256:758d1dc9b62c2ed8881585c254976d66eae0889919ab9b859064fc2fe3c7743e"}, + {file = "pre_commit-2.16.0.tar.gz", hash = "sha256:fe9897cac830aa7164dbd02a4e7b90cae49630451ce88464bca73db486ba9f65"}, ] psutil = [ - {file = "psutil-5.8.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64"}, - {file = "psutil-5.8.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:0ae6f386d8d297177fd288be6e8d1afc05966878704dad9847719650e44fc49c"}, - {file = "psutil-5.8.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:12d844996d6c2b1d3881cfa6fa201fd635971869a9da945cf6756105af73d2df"}, - {file = "psutil-5.8.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:02b8292609b1f7fcb34173b25e48d0da8667bc85f81d7476584d889c6e0f2131"}, - {file = "psutil-5.8.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:6ffe81843131ee0ffa02c317186ed1e759a145267d54fdef1bc4ea5f5931ab60"}, - {file = "psutil-5.8.0-cp27-none-win32.whl", hash = "sha256:ea313bb02e5e25224e518e4352af4bf5e062755160f77e4b1767dd5ccb65f876"}, - {file = "psutil-5.8.0-cp27-none-win_amd64.whl", hash = "sha256:5da29e394bdedd9144c7331192e20c1f79283fb03b06e6abd3a8ae45ffecee65"}, - {file = "psutil-5.8.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:74fb2557d1430fff18ff0d72613c5ca30c45cdbfcddd6a5773e9fc1fe9364be8"}, - {file = "psutil-5.8.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:74f2d0be88db96ada78756cb3a3e1b107ce8ab79f65aa885f76d7664e56928f6"}, - {file = "psutil-5.8.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:99de3e8739258b3c3e8669cb9757c9a861b2a25ad0955f8e53ac662d66de61ac"}, - {file = "psutil-5.8.0-cp36-cp36m-win32.whl", hash = "sha256:36b3b6c9e2a34b7d7fbae330a85bf72c30b1c827a4366a07443fc4b6270449e2"}, - {file = "psutil-5.8.0-cp36-cp36m-win_amd64.whl", hash = "sha256:52de075468cd394ac98c66f9ca33b2f54ae1d9bff1ef6b67a212ee8f639ec06d"}, - {file = "psutil-5.8.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c6a5fd10ce6b6344e616cf01cc5b849fa8103fbb5ba507b6b2dee4c11e84c935"}, - {file = "psutil-5.8.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:61f05864b42fedc0771d6d8e49c35f07efd209ade09a5afe6a5059e7bb7bf83d"}, - {file = "psutil-5.8.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:0dd4465a039d343925cdc29023bb6960ccf4e74a65ad53e768403746a9207023"}, - {file = "psutil-5.8.0-cp37-cp37m-win32.whl", hash = "sha256:1bff0d07e76114ec24ee32e7f7f8d0c4b0514b3fae93e3d2aaafd65d22502394"}, - {file = "psutil-5.8.0-cp37-cp37m-win_amd64.whl", hash = "sha256:fcc01e900c1d7bee2a37e5d6e4f9194760a93597c97fee89c4ae51701de03563"}, - {file = "psutil-5.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6223d07a1ae93f86451d0198a0c361032c4c93ebd4bf6d25e2fb3edfad9571ef"}, - {file = "psutil-5.8.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d225cd8319aa1d3c85bf195c4e07d17d3cd68636b8fc97e6cf198f782f99af28"}, - {file = "psutil-5.8.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:28ff7c95293ae74bf1ca1a79e8805fcde005c18a122ca983abf676ea3466362b"}, - {file = "psutil-5.8.0-cp38-cp38-win32.whl", hash = "sha256:ce8b867423291cb65cfc6d9c4955ee9bfc1e21fe03bb50e177f2b957f1c2469d"}, - {file = "psutil-5.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:90f31c34d25b1b3ed6c40cdd34ff122b1887a825297c017e4cbd6796dd8b672d"}, - {file = "psutil-5.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6323d5d845c2785efb20aded4726636546b26d3b577aded22492908f7c1bdda7"}, - {file = "psutil-5.8.0-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:245b5509968ac0bd179287d91210cd3f37add77dad385ef238b275bad35fa1c4"}, - {file = "psutil-5.8.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:90d4091c2d30ddd0a03e0b97e6a33a48628469b99585e2ad6bf21f17423b112b"}, - {file = "psutil-5.8.0-cp39-cp39-win32.whl", hash = "sha256:ea372bcc129394485824ae3e3ddabe67dc0b118d262c568b4d2602a7070afdb0"}, - {file = "psutil-5.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:f4634b033faf0d968bb9220dd1c793b897ab7f1189956e1aa9eae752527127d3"}, - {file = "psutil-5.8.0.tar.gz", hash = "sha256:0c9ccb99ab76025f2f0bbecf341d4656e9c1351db8cc8a03ccd62e318ab4b5c6"}, + {file = "psutil-5.9.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:55ce319452e3d139e25d6c3f85a1acf12d1607ddedea5e35fb47a552c051161b"}, + {file = "psutil-5.9.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:7336292a13a80eb93c21f36bde4328aa748a04b68c13d01dfddd67fc13fd0618"}, + {file = "psutil-5.9.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:cb8d10461c1ceee0c25a64f2dd54872b70b89c26419e147a05a10b753ad36ec2"}, + {file = "psutil-5.9.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:7641300de73e4909e5d148e90cc3142fb890079e1525a840cf0dfd39195239fd"}, + {file = "psutil-5.9.0-cp27-none-win32.whl", hash = "sha256:ea42d747c5f71b5ccaa6897b216a7dadb9f52c72a0fe2b872ef7d3e1eacf3ba3"}, + {file = "psutil-5.9.0-cp27-none-win_amd64.whl", hash = "sha256:ef216cc9feb60634bda2f341a9559ac594e2eeaadd0ba187a4c2eb5b5d40b91c"}, + {file = "psutil-5.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90a58b9fcae2dbfe4ba852b57bd4a1dded6b990a33d6428c7614b7d48eccb492"}, + {file = "psutil-5.9.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff0d41f8b3e9ebb6b6110057e40019a432e96aae2008951121ba4e56040b84f3"}, + {file = "psutil-5.9.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:742c34fff804f34f62659279ed5c5b723bb0195e9d7bd9907591de9f8f6558e2"}, + {file = "psutil-5.9.0-cp310-cp310-win32.whl", hash = "sha256:8293942e4ce0c5689821f65ce6522ce4786d02af57f13c0195b40e1edb1db61d"}, + {file = "psutil-5.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:9b51917c1af3fa35a3f2dabd7ba96a2a4f19df3dec911da73875e1edaf22a40b"}, + {file = "psutil-5.9.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e9805fed4f2a81de98ae5fe38b75a74c6e6ad2df8a5c479594c7629a1fe35f56"}, + {file = "psutil-5.9.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c51f1af02334e4b516ec221ee26b8fdf105032418ca5a5ab9737e8c87dafe203"}, + {file = "psutil-5.9.0-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32acf55cb9a8cbfb29167cd005951df81b567099295291bcfd1027365b36591d"}, + {file = "psutil-5.9.0-cp36-cp36m-win32.whl", hash = "sha256:e5c783d0b1ad6ca8a5d3e7b680468c9c926b804be83a3a8e95141b05c39c9f64"}, + {file = "psutil-5.9.0-cp36-cp36m-win_amd64.whl", hash = "sha256:d62a2796e08dd024b8179bd441cb714e0f81226c352c802fca0fd3f89eeacd94"}, + {file = "psutil-5.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3d00a664e31921009a84367266b35ba0aac04a2a6cad09c550a89041034d19a0"}, + {file = "psutil-5.9.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7779be4025c540d1d65a2de3f30caeacc49ae7a2152108adeaf42c7534a115ce"}, + {file = "psutil-5.9.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:072664401ae6e7c1bfb878c65d7282d4b4391f1bc9a56d5e03b5a490403271b5"}, + {file = "psutil-5.9.0-cp37-cp37m-win32.whl", hash = "sha256:df2c8bd48fb83a8408c8390b143c6a6fa10cb1a674ca664954de193fdcab36a9"}, + {file = "psutil-5.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1d7b433519b9a38192dfda962dd8f44446668c009833e1429a52424624f408b4"}, + {file = "psutil-5.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c3400cae15bdb449d518545cbd5b649117de54e3596ded84aacabfbb3297ead2"}, + {file = "psutil-5.9.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b2237f35c4bbae932ee98902a08050a27821f8f6dfa880a47195e5993af4702d"}, + {file = "psutil-5.9.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1070a9b287846a21a5d572d6dddd369517510b68710fca56b0e9e02fd24bed9a"}, + {file = "psutil-5.9.0-cp38-cp38-win32.whl", hash = "sha256:76cebf84aac1d6da5b63df11fe0d377b46b7b500d892284068bacccf12f20666"}, + {file = "psutil-5.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:3151a58f0fbd8942ba94f7c31c7e6b310d2989f4da74fcbf28b934374e9bf841"}, + {file = "psutil-5.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:539e429da49c5d27d5a58e3563886057f8fc3868a5547b4f1876d9c0f007bccf"}, + {file = "psutil-5.9.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58c7d923dc209225600aec73aa2c4ae8ea33b1ab31bc11ef8a5933b027476f07"}, + {file = "psutil-5.9.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3611e87eea393f779a35b192b46a164b1d01167c9d323dda9b1e527ea69d697d"}, + {file = "psutil-5.9.0-cp39-cp39-win32.whl", hash = "sha256:4e2fb92e3aeae3ec3b7b66c528981fd327fb93fd906a77215200404444ec1845"}, + {file = "psutil-5.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:7d190ee2eaef7831163f254dc58f6d2e2a22e27382b936aab51c835fc080c3d3"}, + {file = "psutil-5.9.0.tar.gz", hash = "sha256:869842dbd66bb80c3217158e629d6fceaecc3a3166d3d1faee515b05dd26ca25"}, +] +ptable = [ + {file = "PTable-0.9.2.tar.gz", hash = "sha256:aa7fc151cb40f2dabcd2275ba6f7fd0ff8577a86be3365cd3fb297cbe09cc292"}, ] py = [ - {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, - {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] pycares = [ - {file = "pycares-4.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:db5a533111a3cfd481e7e4fb2bf8bef69f4fa100339803e0504dd5aecafb96a5"}, - {file = "pycares-4.0.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:fdff88393c25016f417770d82678423fc7a56995abb2df3d2a1e55725db6977d"}, - {file = "pycares-4.0.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0aa97f900a7ffb259be77d640006585e2a907b0cd4edeee0e85cf16605995d5a"}, - {file = "pycares-4.0.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:a34b0e3e693dceb60b8a1169668d606c75cb100ceba0a2df53c234a0eb067fbc"}, - {file = "pycares-4.0.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:7661d6bbd51a337e7373cb356efa8be9b4655fda484e068f9455e939aec8d54e"}, - {file = "pycares-4.0.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:57315b8eb8fdbc56b3ad4932bc4b17132bb7c7fd2bd590f7fb84b6b522098aa9"}, - {file = "pycares-4.0.0-cp36-cp36m-win32.whl", hash = "sha256:dca9dc58845a9d083f302732a3130c68ded845ad5d463865d464e53c75a3dd45"}, - {file = "pycares-4.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:c95c964d5dd307e104b44b193095c67bb6b10c9eda1ffe7d44ab7a9e84c476d9"}, - {file = "pycares-4.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:26e67e4f81c80a5955dcf6193f3d9bee3c491fc0056299b383b84d792252fba4"}, - {file = "pycares-4.0.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:cd3011ffd5e1ad55880f7256791dbab9c43ebeda260474a968f19cd0319e1aef"}, - {file = "pycares-4.0.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1b959dd5921d207d759d421eece1b60416df33a7f862465739d5f2c363c2f523"}, - {file = "pycares-4.0.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:6f258c1b74c048a9501a25f732f11b401564005e5e3c18f1ca6cad0c3dc0fb19"}, - {file = "pycares-4.0.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:b17ef48729786e62b574c6431f675f4cb02b27691b49e7428a605a50cd59c072"}, - {file = "pycares-4.0.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:82b3259cb590ddd107a6d2dc52da2a2e9a986bf242e893d58c786af2f8191047"}, - {file = "pycares-4.0.0-cp37-cp37m-win32.whl", hash = "sha256:4876fc790ae32832ae270c4a010a1a77e12ddf8d8e6ad70ad0b0a9d506c985f7"}, - {file = "pycares-4.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f60c04c5561b1ddf85ca4e626943cc09d7fb684e1adb22abb632095415a40fd7"}, - {file = "pycares-4.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:615406013cdcd1b445e5d1a551d276c6200b3abe77e534f8a7f7e1551208d14f"}, - {file = "pycares-4.0.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:6580aef5d1b29a88c3d72fe73c691eacfd454f86e74d3fdd18f4bad8e8def98b"}, - {file = "pycares-4.0.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8ebb3ba0485f66cae8eed7ce3e9ed6f2c0bfd5e7319d5d0fbbb511064f17e1d4"}, - {file = "pycares-4.0.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:c5362b7690ca481440f6b98395ac6df06aa50518ccb183c560464d1e5e2ab5d4"}, - {file = "pycares-4.0.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:eb60be66accc9a9ea1018b591a1f5800cba83491d07e9acc8c56bc6e6607ab54"}, - {file = "pycares-4.0.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:44896d6e191a6b5a914dbe3aa7c748481bf6ad19a9df33c1e76f8f2dc33fc8f0"}, - {file = "pycares-4.0.0-cp38-cp38-win32.whl", hash = "sha256:09b28fc7bc2cc05f7f69bf1636ddf46086e0a1837b62961e2092fcb40477320d"}, - {file = "pycares-4.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:d4a5081e232c1d181883dcac4675807f3a6cf33911c4173fbea00c0523687ed4"}, - {file = "pycares-4.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:103353577a6266a53e71bfee4cf83825f1401fefa60f0fb8bdec35f13be6a5f2"}, - {file = "pycares-4.0.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:ad6caf580ee69806fc6534be93ddbb6e99bf94296d79ab351c37b2992b17abfd"}, - {file = "pycares-4.0.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:3d5e50c95849f6905d2a9dbf02ed03f82580173e3c5604a39e2ad054185631f1"}, - {file = "pycares-4.0.0-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:53bc4f181b19576499b02cea4b45391e8dcbe30abd4cd01492f66bfc15615a13"}, - {file = "pycares-4.0.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:d52f9c725d2a826d5ffa37681eb07ffb996bfe21788590ef257664a3898fc0b5"}, - {file = "pycares-4.0.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:3c7fb8d34ee11971c39acfaf98d0fac66725385ccef3bfe1b174c92b210e1aa4"}, - {file = "pycares-4.0.0-cp39-cp39-win32.whl", hash = "sha256:e9773e07684a55f54657df05237267611a77b294ec3bacb5f851c4ffca38a465"}, - {file = "pycares-4.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:38e54037f36c149146ff15f17a4a963fbdd0f9871d4a21cd94ff9f368140f57e"}, - {file = "pycares-4.0.0.tar.gz", hash = "sha256:d0154fc5753b088758fbec9bc137e1b24bb84fc0c6a09725c8bac25a342311cd"}, + {file = "pycares-4.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:71b99b9e041ae3356b859822c511f286f84c8889ec9ed1fbf6ac30fb4da13e4c"}, + {file = "pycares-4.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c000942f5fc64e6e046aa61aa53b629b576ba11607d108909727c3c8f211a157"}, + {file = "pycares-4.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:b0e50ddc78252f2e2b6b5f2c73e5b2449dfb6bea7a5a0e21dfd1e2bcc9e17382"}, + {file = "pycares-4.1.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6831e963a910b0a8cbdd2750ffcdf5f2bb0edb3f53ca69ff18484de2cc3807c4"}, + {file = "pycares-4.1.2-cp310-cp310-win32.whl", hash = "sha256:ad7b28e1b6bc68edd3d678373fa3af84e39d287090434f25055d21b4716b2fc6"}, + {file = "pycares-4.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:27a6f09dbfb69bb79609724c0f90dfaa7c215876a7cd9f12d585574d1f922112"}, + {file = "pycares-4.1.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e5a060f5fa90ae245aa99a4a8ad13ec39c2340400de037c7e8d27b081e1a3c64"}, + {file = "pycares-4.1.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:056330275dea42b7199494047a745e1d9785d39fb8c4cd469dca043532240b80"}, + {file = "pycares-4.1.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0aa897543a786daba74ec5e19638bd38b2b432d179a0e248eac1e62de5756207"}, + {file = "pycares-4.1.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cbceaa9b2c416aa931627466d3240aecfc905c292c842252e3d77b8630072505"}, + {file = "pycares-4.1.2-cp36-cp36m-win32.whl", hash = "sha256:112e1385c451069112d6b5ea1f9c378544f3c6b89882ff964e9a64be3336d7e4"}, + {file = "pycares-4.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:c6680f7fdc0f1163e8f6c2a11d11b9a0b524a61000d2a71f9ccd410f154fb171"}, + {file = "pycares-4.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58a41a2baabcd95266db776c510d349d417919407f03510fc87ac7488730d913"}, + {file = "pycares-4.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a810d01c9a426ee8b0f36969c2aef5fb966712be9d7e466920beb328cd9cefa3"}, + {file = "pycares-4.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:b266cec81dcea2c3efbbd3dda00af8d7eb0693ae9e47e8706518334b21f27d4a"}, + {file = "pycares-4.1.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8319afe4838e09df267c421ca93da408f770b945ec6217dda72f1f6a493e37e4"}, + {file = "pycares-4.1.2-cp37-cp37m-win32.whl", hash = "sha256:4d5da840aa0d9b15fa51107f09270c563a348cb77b14ae9653d0bbdbe326fcc2"}, + {file = "pycares-4.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:5632f21d92cc0225ba5ff906e4e5dec415ef0b3df322c461d138190681cd5d89"}, + {file = "pycares-4.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8fd1ff17a26bb004f0f6bb902ba7dddd810059096ae0cc3b45e4f5be46315d19"}, + {file = "pycares-4.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:439799be4b7576e907139a7f9b3c8a01b90d3e38af4af9cd1fc6c1ee9a42b9e6"}, + {file = "pycares-4.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:40079ed58efa91747c50aac4edf8ecc7e570132ab57dc0a4030eb0d016a6cab8"}, + {file = "pycares-4.1.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e190471a015f8225fa38069617192e06122771cce2b169ac7a60bfdbd3d4ab2"}, + {file = "pycares-4.1.2-cp38-cp38-win32.whl", hash = "sha256:2b837315ed08c7df009b67725fe1f50489e99de9089f58ec1b243dc612f172aa"}, + {file = "pycares-4.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:c7eba3c8354b730a54d23237d0b6445a2f68570fa68d0848887da23a3f3b71f3"}, + {file = "pycares-4.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2f5f84fe9f83eab9cd68544b165b74ba6e3412d029cc9ab20098d9c332869fc5"}, + {file = "pycares-4.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569eef8597b5e02b1bc4644b9f272160304d8c9985357d7ecfcd054da97c0771"}, + {file = "pycares-4.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e1489aa25d14dbf7176110ead937c01176ed5a0ebefd3b092bbd6b202241814c"}, + {file = "pycares-4.1.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:dc942692fca0e27081b7bb414bb971d34609c80df5e953f6d0c62ecc8019acd9"}, + {file = "pycares-4.1.2-cp39-cp39-win32.whl", hash = "sha256:ed71dc4290d9c3353945965604ef1f6a4de631733e9819a7ebc747220b27e641"}, + {file = "pycares-4.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:ec00f3594ee775665167b1a1630edceefb1b1283af9ac57480dba2fb6fd6c360"}, + {file = "pycares-4.1.2.tar.gz", hash = "sha256:03490be0e7b51a0c8073f877bec347eff31003f64f57d9518d419d9369452837"}, ] pycodestyle = [ {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"}, {file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"}, ] pycparser = [ - {file = "pycparser-2.20-py2.py3-none-any.whl", hash = "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"}, - {file = "pycparser-2.20.tar.gz", hash = "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0"}, + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] pydocstyle = [ {file = "pydocstyle-6.1.1-py3-none-any.whl", hash = "sha256:6987826d6775056839940041beef5c08cc7e3d71d63149b48e36727f70144dc4"}, @@ -1566,33 +1768,32 @@ pyflakes = [ {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, ] pyparsing = [ - {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, - {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, + {file = "pyparsing-3.0.6-py3-none-any.whl", hash = "sha256:04ff808a5b90911829c55c4e26f75fa5ca8a2f5f36aa3a51f68e27033341d3e4"}, + {file = "pyparsing-3.0.6.tar.gz", hash = "sha256:d9bdec0013ef1eb5a84ab39a3b3868911598afa494f5faa038647101504e2b81"}, ] -pyreadline = [ - {file = "pyreadline-2.1.win-amd64.exe", hash = "sha256:9ce5fa65b8992dfa373bddc5b6e0864ead8f291c94fbfec05fbd5c836162e67b"}, - {file = "pyreadline-2.1.win32.exe", hash = "sha256:65540c21bfe14405a3a77e4c085ecfce88724743a4ead47c66b84defcf82c32e"}, - {file = "pyreadline-2.1.zip", hash = "sha256:4530592fc2e85b25b1a9f79664433da09237c1a270e4d78ea5aa3a2c7229e2d1"}, +pyreadline3 = [ + {file = "pyreadline3-3.3-py3-none-any.whl", hash = "sha256:0003fd0079d152ecbd8111202c5a7dfa6a5569ffd65b235e45f3c2ecbee337b4"}, + {file = "pyreadline3-3.3.tar.gz", hash = "sha256:ff3b5a1ac0010d0967869f723e687d42cabc7dccf33b14934c92aa5168d260b3"}, ] pytest = [ - {file = "pytest-6.2.4-py3-none-any.whl", hash = "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890"}, - {file = "pytest-6.2.4.tar.gz", hash = "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b"}, + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, ] pytest-cov = [ {file = "pytest-cov-2.12.1.tar.gz", hash = "sha256:261ceeb8c227b726249b376b8526b600f38667ee314f910353fa318caa01f4d7"}, {file = "pytest_cov-2.12.1-py2.py3-none-any.whl", hash = "sha256:261bb9e47e65bd099c89c3edf92972865210c36813f80ede5277dceb77a4a62a"}, ] pytest-forked = [ - {file = "pytest-forked-1.3.0.tar.gz", hash = "sha256:6aa9ac7e00ad1a539c41bec6d21011332de671e938c7637378ec9710204e37ca"}, - {file = "pytest_forked-1.3.0-py2.py3-none-any.whl", hash = "sha256:dc4147784048e70ef5d437951728825a131b81714b398d5d52f17c7c144d8815"}, + {file = "pytest-forked-1.4.0.tar.gz", hash = "sha256:8b67587c8f98cbbadfdd804539ed5455b6ed03802203485dd2f53c1422d7440e"}, + {file = "pytest_forked-1.4.0-py3-none-any.whl", hash = "sha256:bbbb6717efc886b9d64537b41fb1497cfaf3c9601276be8da2cccfea5a3c8ad8"}, ] pytest-xdist = [ {file = "pytest-xdist-2.3.0.tar.gz", hash = "sha256:e8ecde2f85d88fbcadb7d28cb33da0fa29bca5cf7d5967fa89fc0e97e5299ea5"}, {file = "pytest_xdist-2.3.0-py3-none-any.whl", hash = "sha256:ed3d7da961070fce2a01818b51f6888327fb88df4379edeb6b9d990e789d9c8d"}, ] python-dateutil = [ - {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, - {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"}, + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] python-dotenv = [ {file = "python-dotenv-0.17.1.tar.gz", hash = "sha256:b1ae5e9643d5ed987fc57cc2583021e38db531946518130777734f9589b3141f"}, @@ -1609,88 +1810,87 @@ pyyaml = [ {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, ] rapidfuzz = [ - {file = "rapidfuzz-1.4.1-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:72878878d6744883605b5453c382361716887e9e552f677922f76d93d622d8cb"}, - {file = "rapidfuzz-1.4.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:56a67a5b3f783e9af73940f6945366408b3a2060fc6ab18466e5a2894fd85617"}, - {file = "rapidfuzz-1.4.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f5d396b64f8ae3a793633911a1fb5d634ac25bf8f13d440139fa729131be42d8"}, - {file = "rapidfuzz-1.4.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:4990698233e7eda7face7c09f5874a09760c7524686045cbb10317e3a7f3225f"}, - {file = "rapidfuzz-1.4.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:a87e212855b18a951e79ec71d71dbd856d98cd2019d0c2bd46ec30688a8aa68a"}, - {file = "rapidfuzz-1.4.1-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1897d2ef03f5b51bc19bdb2d0398ae968766750fa319843733f0a8f12ddde986"}, - {file = "rapidfuzz-1.4.1-cp35-cp35m-manylinux2014_ppc64le.whl", hash = "sha256:e1fc4fd219057f5f1fa40bb9bc5e880f8ef45bf19350d4f5f15ca2ce7f61c99b"}, - {file = "rapidfuzz-1.4.1-cp35-cp35m-manylinux2014_s390x.whl", hash = "sha256:21300c4d048798985c271a8bf1ed1611902ebd4479fcacda1a3eaaebbad2f744"}, - {file = "rapidfuzz-1.4.1-cp35-cp35m-win32.whl", hash = "sha256:d2659967c6ac74211a87a1109e79253e4bc179641057c64800ef4e2dc0534fdb"}, - {file = "rapidfuzz-1.4.1-cp35-cp35m-win_amd64.whl", hash = "sha256:26ac4bfe564c516e053fc055f1543d2b2433338806738c7582e1f75ed0485f7e"}, - {file = "rapidfuzz-1.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:3b485c98ad1ce3c04556f65aaab5d6d6d72121cde656d43505169c71ae956476"}, - {file = "rapidfuzz-1.4.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:59db06356eaf22c83f44b0dded964736cbb137291cdf2cf7b4974c0983b94932"}, - {file = "rapidfuzz-1.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:fef95249af9a535854b617a68788c38cd96308d97ee14d44bc598cc73e986167"}, - {file = "rapidfuzz-1.4.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:7d8c186e8270e103d339b26ef498581cf3178470ccf238dfd5fd0e47d80e4c7d"}, - {file = "rapidfuzz-1.4.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:9246b9c5c8992a83a08ac7813c8bbff2e674ad0b681f9b3fb1ec7641eff6c21f"}, - {file = "rapidfuzz-1.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:f58c17f7a82b1bcc2ce304942cae14287223e6b6eead7071241273da7d9b9770"}, - {file = "rapidfuzz-1.4.1-cp36-cp36m-manylinux2014_ppc64le.whl", hash = "sha256:ed708620b23a09ac52eaaec0761943c1bbc9a62d19ecd2feb4da8c3f79ef9d37"}, - {file = "rapidfuzz-1.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:bdec9ae5fd8a8d4d8813b4aac3505c027b922b4033a32a7aab66a9b2f03a7b47"}, - {file = "rapidfuzz-1.4.1-cp36-cp36m-win32.whl", hash = "sha256:fc668fd706ad1162ce14f26ca2957b4690d47770d23609756536c918a855ced0"}, - {file = "rapidfuzz-1.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f9f35df5dd9b02669ff6b1d4a386607ff56982c86a7e57d95eb08c6afbab4ddd"}, - {file = "rapidfuzz-1.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8427310ea29ce2968e1c6f6779ae5a458b3a4984f9150fc4d16f92b96456f848"}, - {file = "rapidfuzz-1.4.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:1430dc745476e3798742ad835f61f6e6bf5d3e9a22cf9cd0288b28b7440a9872"}, - {file = "rapidfuzz-1.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1d20311da611c8f4638a09e2bc5e04b327bae010cb265ef9628d9c13c6d5da7b"}, - {file = "rapidfuzz-1.4.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:d7881965e428cf6fe248d6e702e6d5857da02278ab9b21313bee717c080e443e"}, - {file = "rapidfuzz-1.4.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:f76c965f15861ec4d39e904bd65b84a39121334439ac17bfb8b900d1e6779a93"}, - {file = "rapidfuzz-1.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:61167f989415e701ac379de247e6b0a21ea62afc86c54d8a79f485b4f0173c02"}, - {file = "rapidfuzz-1.4.1-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:645cfb9456229f0bd5752b3eda69f221d825fbb8cbb8855433516bc185111506"}, - {file = "rapidfuzz-1.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:c28be57c9bc47b3d7f484340fab1bec8ed4393dee1090892c2774a4584435eb8"}, - {file = "rapidfuzz-1.4.1-cp37-cp37m-win32.whl", hash = "sha256:3c94b6d3513c693f253ff762112cc4580d3bd377e4abacb96af31a3d606fbe14"}, - {file = "rapidfuzz-1.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:506d50a066451502ee2f8bf016bc3ba3e3b04eede7a4059d7956248e2dd96179"}, - {file = "rapidfuzz-1.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:80b375098658bb3db14215a975d354f6573d3943ac2ae0c4627c7760d57ce075"}, - {file = "rapidfuzz-1.4.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:ba8f7cbd8fdbd3ae115f4484888f3cb94bc2ac7cbd4eb1ca95a3d4f874261ff8"}, - {file = "rapidfuzz-1.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:5fa8570720b0fdfc52f24f5663d66c52ea88ba19cb8b1ff6a39a8bc0b925b33b"}, - {file = "rapidfuzz-1.4.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:f35c8a4c690447fd335bfd77df4da42dfea37cfa06a8ecbf22543d86dc720e12"}, - {file = "rapidfuzz-1.4.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:27f9eef48e212d73e78f0f5ceedc62180b68f6a25fa0752d2ccfaedc3a840bec"}, - {file = "rapidfuzz-1.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:31e99216e2a04aec4f281d472b28a683921f1f669a429cf605d11526623eaeed"}, - {file = "rapidfuzz-1.4.1-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:f22bf7ba6eddd59764457f74c637ab5c3ed976c5fcfaf827e1d320cc0478e12b"}, - {file = "rapidfuzz-1.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:c43ddb354abd00e56f024ce80affb3023fa23206239bb81916d5877cba7f2d1e"}, - {file = "rapidfuzz-1.4.1-cp38-cp38-win32.whl", hash = "sha256:62c1f4ac20c8019ce8d481fb27235306ef3912a8d0b9a60b17905699f43ff072"}, - {file = "rapidfuzz-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:2963f356c70b710dc6337b012ec976ce2fc2b81c2a9918a686838fead6eb4e1d"}, - {file = "rapidfuzz-1.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c07f301fd549b266410654850c6918318d7dcde8201350e9ac0819f0542cf147"}, - {file = "rapidfuzz-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fa4c8b6fc7e93e3a3fb9be9566f1fe7ef920735eadcee248a0d70f3ca8941341"}, - {file = "rapidfuzz-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c200bd813bbd3b146ba0fd284a9ad314bbad9d95ed542813273bdb9d0ee4e796"}, - {file = "rapidfuzz-1.4.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:2cccc84e1f0c6217747c09cafe93164e57d3644e18a334845a2dfbdd2073cd2c"}, - {file = "rapidfuzz-1.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f2033e3d61d1e498f618123b54dc7436d50510b0d18fd678d867720e8d7b2f23"}, - {file = "rapidfuzz-1.4.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:26b7f48b3ddd9d97cf8482a88f0f6cba47ac13ff16e63386ea7ce06178174770"}, - {file = "rapidfuzz-1.4.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:bf18614f87fe3bfff783f0a3d0fad0eb59c92391e52555976e55570a651d2330"}, - {file = "rapidfuzz-1.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:8cb5c2502ff06028a1468bdf61323b53cc3a37f54b5d62d62c5371795b81086a"}, - {file = "rapidfuzz-1.4.1-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:f37f80c1541d6e0a30547261900086b8c0bac519ebc12c9cd6b61a9a43a7e195"}, - {file = "rapidfuzz-1.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:c13cd1e840aa93639ac1d131fbfa740a609fd20dfc2a462d5cd7bce747a2398d"}, - {file = "rapidfuzz-1.4.1-cp39-cp39-win32.whl", hash = "sha256:0ec346f271e96c485716c091c8b0b78ba52da33f7c6ebb52a349d64094566c2d"}, - {file = "rapidfuzz-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:5208ce1b1989a10e6fc5b5ef5d0bb7d1ffe5408838f3106abde241aff4dab08c"}, - {file = "rapidfuzz-1.4.1-pp36-pypy36_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4fa195ea9ca35bacfa2a4319c6d4ab03aa6a283ad2089b70d2dfa0f6a7d9c1bc"}, - {file = "rapidfuzz-1.4.1-pp36-pypy36_pp73-manylinux1_x86_64.whl", hash = "sha256:6e336cfd8103b0b38e107e01502e9d6bf7c7f04e49b970fb11a4bf6c7a932b94"}, - {file = "rapidfuzz-1.4.1-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:c798c5b87efe8a7e63f408e07ff3bc03ba8b94f4498a89b48eaab3a9f439d52c"}, - {file = "rapidfuzz-1.4.1-pp36-pypy36_pp73-win32.whl", hash = "sha256:bb16a10b40f5bd3c645f7748fbd36f49699a03f550c010a2c665905cc8937de8"}, - {file = "rapidfuzz-1.4.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2278001924031d9d75f821bff2c5fef565c8376f252562e04d8eec8857475c36"}, - {file = "rapidfuzz-1.4.1-pp37-pypy37_pp73-manylinux1_x86_64.whl", hash = "sha256:a89d11f3b5da35fdf3e839186203b9367d56e2be792e8dccb098f47634ec6eb9"}, - {file = "rapidfuzz-1.4.1-pp37-pypy37_pp73-manylinux2010_x86_64.whl", hash = "sha256:f8c79cd11b4778d387366a59aa747f5268433f9d68be37b00d16f4fb08fdf850"}, - {file = "rapidfuzz-1.4.1-pp37-pypy37_pp73-win32.whl", hash = "sha256:4364db793ed4b439f9dd28a335bee14e2a828283d3b93c2d2686cc645eeafdd5"}, - {file = "rapidfuzz-1.4.1.tar.gz", hash = "sha256:de20550178376d21bfe1b34a7dc42ab107bb282ef82069cf6dfe2805a0029e26"}, + {file = "rapidfuzz-1.9.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:68227a8b25291d6a2140aef049271ea30a77be5ef672a58e582a55a5cc1fce93"}, + {file = "rapidfuzz-1.9.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:c33541995b96ff40025c1456b8c74b7dd2ab9cbf91943fc35a7bb621f48940e2"}, + {file = "rapidfuzz-1.9.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:c2fafbbf97a4632822248f4201601b691e2eac5fdb30e5d7a96d07a6d058a7d4"}, + {file = "rapidfuzz-1.9.1-cp27-cp27m-win32.whl", hash = "sha256:364795f617a99e1dbb55ac3947ab8366588b72531cb2d6152666287d20610706"}, + {file = "rapidfuzz-1.9.1-cp27-cp27m-win_amd64.whl", hash = "sha256:f171d9e66144b0647f9b998ef10bdd919a640e4b1357250c8ef6259deb5ffe0d"}, + {file = "rapidfuzz-1.9.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:c83801a7c5209663aa120b815a4f2c39e95fe8e0b774ec58a1e0affd6a2fcfc6"}, + {file = "rapidfuzz-1.9.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:67e61c2baa6bb1848c4a33752f1781124dcc90bf3f31b18b44db1ae4e4e26634"}, + {file = "rapidfuzz-1.9.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8ab7eb003a18991347174910f11d38ff40399081185d9e3199ec277535f7828b"}, + {file = "rapidfuzz-1.9.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5ad450badf06ddf98a246140b5059ba895ee8445e8102a5a289908327f551f81"}, + {file = "rapidfuzz-1.9.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:402b2174bded62a793c5f7d9aec16bc32c661402360a934819ae72b54cfbce1e"}, + {file = "rapidfuzz-1.9.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:92066ccb054efc2e17afb4049c98b550969653cd58f71dd756cfcc8e6864630a"}, + {file = "rapidfuzz-1.9.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8dc0bf1814accee08a9c9bace6672ef06eae6b0446fce88e3e97e23dfaf3ea10"}, + {file = "rapidfuzz-1.9.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bdbd387efb8478605951344f327dd03bf053c138d757369a43404305b99e55db"}, + {file = "rapidfuzz-1.9.1-cp310-cp310-win32.whl", hash = "sha256:b1c54807e556dbcc6caf4ce0f24446c01b195f3cc46e2a6e74b82d3a21eaa45d"}, + {file = "rapidfuzz-1.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:ac3273364cd1619cab3bf0ba731efea5405833f9eba362da7dcd70bd42073d8e"}, + {file = "rapidfuzz-1.9.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:d9faf62606c08a0a6992dd480c72b6a068733ae02688dc35f2e36ba0d44673f4"}, + {file = "rapidfuzz-1.9.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:f6a56a48be047637b1b0b2459a11cf7cd5aa7bbe16a439bd4f73b4af39e620e4"}, + {file = "rapidfuzz-1.9.1-cp35-cp35m-win32.whl", hash = "sha256:aa91609979e9d2700f0ff100df99b36e7d700b70169ee385d43d5de9e471ae97"}, + {file = "rapidfuzz-1.9.1-cp35-cp35m-win_amd64.whl", hash = "sha256:b4cfdd0915ab4cec86c2ff6bab9f01b03454f3de0963c37f9f219df2ddf42b95"}, + {file = "rapidfuzz-1.9.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c6bfa4ad0158a093cd304f795ceefdc3861ae6942a61432b2a50858be6de88ca"}, + {file = "rapidfuzz-1.9.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:eb0ea02295d9278bd2dcd2df4760b0f2887b6c3f2f374005ec5af320d8d3a37e"}, + {file = "rapidfuzz-1.9.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d5187cd5cd6273e9fee07de493a42a2153134a4914df74cb1abb0744551c548a"}, + {file = "rapidfuzz-1.9.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6e5b8af63f9c05b64454460759ed84a715d581d598ec4484f4ec512f398e8b1"}, + {file = "rapidfuzz-1.9.1-cp36-cp36m-win32.whl", hash = "sha256:36137f88f2b28115af506118e64e11c816611eab2434293af7fdacd1290ffb9d"}, + {file = "rapidfuzz-1.9.1-cp36-cp36m-win_amd64.whl", hash = "sha256:fcc420cad46be7c9887110edf04cdee545f26dbf22650a443d89790fc35f7b88"}, + {file = "rapidfuzz-1.9.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b06de314f426aebff8a44319016bbe2b22f7848c84e44224f80b0690b7b08b18"}, + {file = "rapidfuzz-1.9.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e5de44e719faea79e45322b037f0d4a141d750b80d2204fa68f43a42a24f0fbc"}, + {file = "rapidfuzz-1.9.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f9439df09a782afd01b67005a3b110c70bbf9e1cf06d2ac9b293ce2d02d3c549"}, + {file = "rapidfuzz-1.9.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e903d4702647465721e2d0431c95f04fd56a06577f06f41e2960c83fd63c1bad"}, + {file = "rapidfuzz-1.9.1-cp37-cp37m-win32.whl", hash = "sha256:a5298f4ac1975edcbb15583eab659a44b33aebaf3bccf172e185cfea68771c08"}, + {file = "rapidfuzz-1.9.1-cp37-cp37m-win_amd64.whl", hash = "sha256:103193a01921b54fcdad6b01cfda3a68e00aeafca236b7ecd5b1b2c2e7e96337"}, + {file = "rapidfuzz-1.9.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:1d98a3187040dca855e02179a35c137f72ef83ce243783d44ea59efa86b94b3a"}, + {file = "rapidfuzz-1.9.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cb92bf7fc911b787055a88d9295ca3b4fe8576e3b59271f070f1b1b181eb087d"}, + {file = "rapidfuzz-1.9.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3f014a0f5f8159a94c6ee884fedd1c30e07fb866a5d76ff2c18091bc6363b76f"}, + {file = "rapidfuzz-1.9.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:31474074a99f72289ac325fbd77983e7d355d48860bfe7a4f6f6396fdb24410a"}, + {file = "rapidfuzz-1.9.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ec67d79af5a2d7b0cf67b570a5579710e461cadda4120478e813b63491f394dd"}, + {file = "rapidfuzz-1.9.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ebc0d3d15ed32f98f0052cf6e3e9c9b8010fb93c04fb74d2022e3c51ec540e2"}, + {file = "rapidfuzz-1.9.1-cp38-cp38-win32.whl", hash = "sha256:477ab1a3044bab89db45caabc562b158f68765ecaa638b73ba17e92f09dfa5ff"}, + {file = "rapidfuzz-1.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:8e872763dc0367d7544aa585d2e8b27af233323b8a7cd2f9b78cafa05bae5018"}, + {file = "rapidfuzz-1.9.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8401c41e219ae36ca7a88762776a6270511650d4cc70d024ae61561e96d67e47"}, + {file = "rapidfuzz-1.9.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ea10bd8e0436801c3264f7084a5ea194f12ba9fe1ba898aa4a2107d276501292"}, + {file = "rapidfuzz-1.9.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:433737914b46c1ffa0c678eceae1c260dc6b7fb5b6cad4c725d3e3607c764b32"}, + {file = "rapidfuzz-1.9.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8c3b08e90e45acbc469d1f456681643256e952bf84ec7714f58979baba0c8a1c"}, + {file = "rapidfuzz-1.9.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:bbcd265b3c86176e5db4cbba7b4364d7333c214ee80e2d259c7085929934ca9d"}, + {file = "rapidfuzz-1.9.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d69fabcd635783cd842e7d5ee4b77164314c5124b82df5a0c436ab3d698f8a9"}, + {file = "rapidfuzz-1.9.1-cp39-cp39-win32.whl", hash = "sha256:01f16b6f3fa5d1a26c12f5da5de0032f1e12c919d876005b57492a8ec9a5c043"}, + {file = "rapidfuzz-1.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:0bcc5bbfdbe6068cc2cf0029ab6cde08dceac498d232fa3a61dd34fbfa0b3f36"}, + {file = "rapidfuzz-1.9.1-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:de869c8f4e8edb9b2f7b8232a04896645501defcbd9d85bc0202ff3ec6285f6b"}, + {file = "rapidfuzz-1.9.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:db5978e970fb0955974d51021da4b929e2e4890fef17792989ee32658e2b159c"}, + {file = "rapidfuzz-1.9.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:33479f75f36ac3a1d8421365d4fa906e013490790730a89caba31d06e6f71738"}, + {file = "rapidfuzz-1.9.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:af991cb333ec526d894923163050931b3a870b7694bf7687aaa6154d341a98f5"}, + {file = "rapidfuzz-1.9.1.tar.gz", hash = "sha256:bd7a4fe33ba49db3417f0f57a8af02462554f1296dedcf35b026cd3525efef74"}, ] redis = [ - {file = "redis-3.5.3-py2.py3-none-any.whl", hash = "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24"}, - {file = "redis-3.5.3.tar.gz", hash = "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2"}, + {file = "redis-4.0.2-py3-none-any.whl", hash = "sha256:c8481cf414474e3497ec7971a1ba9b998c8efad0f0d289a009a5bbef040894f9"}, + {file = "redis-4.0.2.tar.gz", hash = "sha256:ccf692811f2c1fc7a92b466aa2599e4a6d2d73d5f736a2c70be600657c0da34a"}, ] regex = [ {file = "regex-2021.4.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:619d71c59a78b84d7f18891fe914446d07edd48dc8328c8e149cbe0929b4e000"}, @@ -1736,12 +1936,16 @@ regex = [ {file = "regex-2021.4.4.tar.gz", hash = "sha256:52ba3d3f9b942c49d7e4bc105bb28551c44065f139a65062ab7912bef10c9afb"}, ] requests = [ - {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, - {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, + {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, + {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, +] +requests-file = [ + {file = "requests-file-1.5.1.tar.gz", hash = "sha256:07d74208d3389d01c38ab89ef403af0cfec63957d53a0081d8eca738d0247d8e"}, + {file = "requests_file-1.5.1-py2.py3-none-any.whl", hash = "sha256:dfe5dae75c12481f68ba353183c53a65e6044c923e64c24b2209f6c7570ca953"}, ] sentry-sdk = [ - {file = "sentry-sdk-0.20.3.tar.gz", hash = "sha256:4ae8d1ced6c67f1c8ea51d82a16721c166c489b76876c9f2c202b8a50334b237"}, - {file = "sentry_sdk-0.20.3-py2.py3-none-any.whl", hash = "sha256:e75c8c58932bda8cd293ea8e4b242527129e1caaec91433d21b8b2f20fee030b"}, + {file = "sentry-sdk-1.5.1.tar.gz", hash = "sha256:2a1757d6611e4bec7d672c2b7ef45afef79fed201d064f53994753303944f5a8"}, + {file = "sentry_sdk-1.5.1-py2.py3-none-any.whl", hash = "sha256:e4cb107e305b2c1b919414775fa73a9997f996447417d22b98e7610ded1e9eb5"}, ] sgmllib3k = [ {file = "sgmllib3k-1.0.0.tar.gz", hash = "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9"}, @@ -1751,16 +1955,16 @@ six = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] snowballstemmer = [ - {file = "snowballstemmer-2.1.0-py2.py3-none-any.whl", hash = "sha256:b51b447bea85f9968c13b650126a888aabd4cb4463fca868ec596826325dedc2"}, - {file = "snowballstemmer-2.1.0.tar.gz", hash = "sha256:e997baa4f2e9139951b6f4c631bad912dfd3c792467e2f03d7239464af90e914"}, + {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, + {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, ] sortedcontainers = [ {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, ] soupsieve = [ - {file = "soupsieve-2.2.1-py3-none-any.whl", hash = "sha256:c2c1c2d44f158cdbddab7824a9af8c4f83c76b1e23e049479aa432feb6c4c23b"}, - {file = "soupsieve-2.2.1.tar.gz", hash = "sha256:052774848f448cf19c7e959adf5566904d525f33a3f8b6ba6f6f8f26ec7de0cc"}, + {file = "soupsieve-2.3.1-py3-none-any.whl", hash = "sha256:1a3cca2617c6b38c0343ed661b1fa5de5637f257d4fe22bd9f1338010a1efefb"}, + {file = "soupsieve-2.3.1.tar.gz", hash = "sha256:b8d49b1cd4f037c7082a9683dfa1801aa2597fb11c3a1155b7a5b94829b4f1f9"}, ] statsd = [ {file = "statsd-3.3.0-py2.py3-none-any.whl", hash = "sha256:c610fb80347fca0ef62666d241bce64184bd7cc1efe582f9690e045c25535eaa"}, @@ -1770,59 +1974,154 @@ taskipy = [ {file = "taskipy-1.7.0-py3-none-any.whl", hash = "sha256:9e284c10898e9dee01a3e72220b94b192b1daa0f560271503a6df1da53d03844"}, {file = "taskipy-1.7.0.tar.gz", hash = "sha256:960e480b1004971e76454ecd1a0484e640744a30073a1069894a311467f85ed8"}, ] +testfixtures = [ + {file = "testfixtures-6.18.3-py2.py3-none-any.whl", hash = "sha256:6ddb7f56a123e1a9339f130a200359092bd0a6455e31838d6c477e8729bb7763"}, + {file = "testfixtures-6.18.3.tar.gz", hash = "sha256:2600100ae96ffd082334b378e355550fef8b4a529a6fa4c34f47130905c7426d"}, +] +tldextract = [ + {file = "tldextract-3.1.2-py2.py3-none-any.whl", hash = "sha256:f55e05f6bf4cc952a87d13594386d32ad2dd265630a8bdfc3df03bd60425c6b0"}, + {file = "tldextract-3.1.2.tar.gz", hash = "sha256:d2034c3558651f7d8fdadea83fb681050b2d662dc67a00d950326dc902029444"}, +] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] typing-extensions = [ - {file = "typing_extensions-3.10.0.0-py2-none-any.whl", hash = "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497"}, - {file = "typing_extensions-3.10.0.0-py3-none-any.whl", hash = "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"}, - {file = "typing_extensions-3.10.0.0.tar.gz", hash = "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342"}, + {file = "typing_extensions-4.0.1-py3-none-any.whl", hash = "sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b"}, + {file = "typing_extensions-4.0.1.tar.gz", hash = "sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e"}, ] urllib3 = [ - {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, - {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, + {file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"}, + {file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"}, ] virtualenv = [ - {file = "virtualenv-20.4.7-py2.py3-none-any.whl", hash = "sha256:2b0126166ea7c9c3661f5b8e06773d28f83322de7a3ff7d06f0aed18c9de6a76"}, - {file = "virtualenv-20.4.7.tar.gz", hash = "sha256:14fdf849f80dbb29a4eb6caa9875d476ee2a5cf76a5f5415fa2f1606010ab467"}, + {file = "virtualenv-20.13.0-py2.py3-none-any.whl", hash = "sha256:339f16c4a86b44240ba7223d0f93a7887c3ca04b5f9c8129da7958447d079b09"}, + {file = "virtualenv-20.13.0.tar.gz", hash = "sha256:d8458cf8d59d0ea495ad9b34c2599487f8a7772d796f9910858376d1600dd2dd"}, +] +wrapt = [ + {file = "wrapt-1.13.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:e05e60ff3b2b0342153be4d1b597bbcfd8330890056b9619f4ad6b8d5c96a81a"}, + {file = "wrapt-1.13.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:85148f4225287b6a0665eef08a178c15097366d46b210574a658c1ff5b377489"}, + {file = "wrapt-1.13.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:2dded5496e8f1592ec27079b28b6ad2a1ef0b9296d270f77b8e4a3a796cf6909"}, + {file = "wrapt-1.13.3-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:e94b7d9deaa4cc7bac9198a58a7240aaf87fe56c6277ee25fa5b3aa1edebd229"}, + {file = "wrapt-1.13.3-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:498e6217523111d07cd67e87a791f5e9ee769f9241fcf8a379696e25806965af"}, + {file = "wrapt-1.13.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:ec7e20258ecc5174029a0f391e1b948bf2906cd64c198a9b8b281b811cbc04de"}, + {file = "wrapt-1.13.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:87883690cae293541e08ba2da22cacaae0a092e0ed56bbba8d018cc486fbafbb"}, + {file = "wrapt-1.13.3-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:f99c0489258086308aad4ae57da9e8ecf9e1f3f30fa35d5e170b4d4896554d80"}, + {file = "wrapt-1.13.3-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:6a03d9917aee887690aa3f1747ce634e610f6db6f6b332b35c2dd89412912bca"}, + {file = "wrapt-1.13.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:936503cb0a6ed28dbfa87e8fcd0a56458822144e9d11a49ccee6d9a8adb2ac44"}, + {file = "wrapt-1.13.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f9c51d9af9abb899bd34ace878fbec8bf357b3194a10c4e8e0a25512826ef056"}, + {file = "wrapt-1.13.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:220a869982ea9023e163ba915077816ca439489de6d2c09089b219f4e11b6785"}, + {file = "wrapt-1.13.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0877fe981fd76b183711d767500e6b3111378ed2043c145e21816ee589d91096"}, + {file = "wrapt-1.13.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:43e69ffe47e3609a6aec0fe723001c60c65305784d964f5007d5b4fb1bc6bf33"}, + {file = "wrapt-1.13.3-cp310-cp310-win32.whl", hash = "sha256:78dea98c81915bbf510eb6a3c9c24915e4660302937b9ae05a0947164248020f"}, + {file = "wrapt-1.13.3-cp310-cp310-win_amd64.whl", hash = "sha256:ea3e746e29d4000cd98d572f3ee2a6050a4f784bb536f4ac1f035987fc1ed83e"}, + {file = "wrapt-1.13.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:8c73c1a2ec7c98d7eaded149f6d225a692caa1bd7b2401a14125446e9e90410d"}, + {file = "wrapt-1.13.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:086218a72ec7d986a3eddb7707c8c4526d677c7b35e355875a0fe2918b059179"}, + {file = "wrapt-1.13.3-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:e92d0d4fa68ea0c02d39f1e2f9cb5bc4b4a71e8c442207433d8db47ee79d7aa3"}, + {file = "wrapt-1.13.3-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:d4a5f6146cfa5c7ba0134249665acd322a70d1ea61732723c7d3e8cc0fa80755"}, + {file = "wrapt-1.13.3-cp35-cp35m-win32.whl", hash = "sha256:8aab36778fa9bba1a8f06a4919556f9f8c7b33102bd71b3ab307bb3fecb21851"}, + {file = "wrapt-1.13.3-cp35-cp35m-win_amd64.whl", hash = "sha256:944b180f61f5e36c0634d3202ba8509b986b5fbaf57db3e94df11abee244ba13"}, + {file = "wrapt-1.13.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:2ebdde19cd3c8cdf8df3fc165bc7827334bc4e353465048b36f7deeae8ee0918"}, + {file = "wrapt-1.13.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:610f5f83dd1e0ad40254c306f4764fcdc846641f120c3cf424ff57a19d5f7ade"}, + {file = "wrapt-1.13.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5601f44a0f38fed36cc07db004f0eedeaadbdcec90e4e90509480e7e6060a5bc"}, + {file = "wrapt-1.13.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:e6906d6f48437dfd80464f7d7af1740eadc572b9f7a4301e7dd3d65db285cacf"}, + {file = "wrapt-1.13.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:766b32c762e07e26f50d8a3468e3b4228b3736c805018e4b0ec8cc01ecd88125"}, + {file = "wrapt-1.13.3-cp36-cp36m-win32.whl", hash = "sha256:5f223101f21cfd41deec8ce3889dc59f88a59b409db028c469c9b20cfeefbe36"}, + {file = "wrapt-1.13.3-cp36-cp36m-win_amd64.whl", hash = "sha256:f122ccd12fdc69628786d0c947bdd9cb2733be8f800d88b5a37c57f1f1d73c10"}, + {file = "wrapt-1.13.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:46f7f3af321a573fc0c3586612db4decb7eb37172af1bc6173d81f5b66c2e068"}, + {file = "wrapt-1.13.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:778fd096ee96890c10ce96187c76b3e99b2da44e08c9e24d5652f356873f6709"}, + {file = "wrapt-1.13.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0cb23d36ed03bf46b894cfec777eec754146d68429c30431c99ef28482b5c1df"}, + {file = "wrapt-1.13.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:96b81ae75591a795d8c90edc0bfaab44d3d41ffc1aae4d994c5aa21d9b8e19a2"}, + {file = "wrapt-1.13.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7dd215e4e8514004c8d810a73e342c536547038fb130205ec4bba9f5de35d45b"}, + {file = "wrapt-1.13.3-cp37-cp37m-win32.whl", hash = "sha256:47f0a183743e7f71f29e4e21574ad3fa95676136f45b91afcf83f6a050914829"}, + {file = "wrapt-1.13.3-cp37-cp37m-win_amd64.whl", hash = "sha256:fd76c47f20984b43d93de9a82011bb6e5f8325df6c9ed4d8310029a55fa361ea"}, + {file = "wrapt-1.13.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b73d4b78807bd299b38e4598b8e7bd34ed55d480160d2e7fdaabd9931afa65f9"}, + {file = "wrapt-1.13.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ec9465dd69d5657b5d2fa6133b3e1e989ae27d29471a672416fd729b429eb554"}, + {file = "wrapt-1.13.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:dd91006848eb55af2159375134d724032a2d1d13bcc6f81cd8d3ed9f2b8e846c"}, + {file = "wrapt-1.13.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ae9de71eb60940e58207f8e71fe113c639da42adb02fb2bcbcaccc1ccecd092b"}, + {file = "wrapt-1.13.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:51799ca950cfee9396a87f4a1240622ac38973b6df5ef7a41e7f0b98797099ce"}, + {file = "wrapt-1.13.3-cp38-cp38-win32.whl", hash = "sha256:4b9c458732450ec42578b5642ac53e312092acf8c0bfce140ada5ca1ac556f79"}, + {file = "wrapt-1.13.3-cp38-cp38-win_amd64.whl", hash = "sha256:7dde79d007cd6dfa65afe404766057c2409316135cb892be4b1c768e3f3a11cb"}, + {file = "wrapt-1.13.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:981da26722bebb9247a0601e2922cedf8bb7a600e89c852d063313102de6f2cb"}, + {file = "wrapt-1.13.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:705e2af1f7be4707e49ced9153f8d72131090e52be9278b5dbb1498c749a1e32"}, + {file = "wrapt-1.13.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:25b1b1d5df495d82be1c9d2fad408f7ce5ca8a38085e2da41bb63c914baadff7"}, + {file = "wrapt-1.13.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:77416e6b17926d953b5c666a3cb718d5945df63ecf922af0ee576206d7033b5e"}, + {file = "wrapt-1.13.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:865c0b50003616f05858b22174c40ffc27a38e67359fa1495605f96125f76640"}, + {file = "wrapt-1.13.3-cp39-cp39-win32.whl", hash = "sha256:0a017a667d1f7411816e4bf214646d0ad5b1da2c1ea13dec6c162736ff25a374"}, + {file = "wrapt-1.13.3-cp39-cp39-win_amd64.whl", hash = "sha256:81bd7c90d28a4b2e1df135bfbd7c23aee3050078ca6441bead44c42483f9ebfb"}, + {file = "wrapt-1.13.3.tar.gz", hash = "sha256:1fea9cd438686e6682271d36f3481a9f3636195578bab9ca3382e2f5f01fc185"}, ] yarl = [ - {file = "yarl-1.6.3-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434"}, - {file = "yarl-1.6.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478"}, - {file = "yarl-1.6.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:547f7665ad50fa8563150ed079f8e805e63dd85def6674c97efd78eed6c224a6"}, - {file = "yarl-1.6.3-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:63f90b20ca654b3ecc7a8d62c03ffa46999595f0167d6450fa8383bab252987e"}, - {file = "yarl-1.6.3-cp36-cp36m-manylinux2014_ppc64le.whl", hash = "sha256:97b5bdc450d63c3ba30a127d018b866ea94e65655efaf889ebeabc20f7d12406"}, - {file = "yarl-1.6.3-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:d8d07d102f17b68966e2de0e07bfd6e139c7c02ef06d3a0f8d2f0f055e13bb76"}, - {file = "yarl-1.6.3-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:15263c3b0b47968c1d90daa89f21fcc889bb4b1aac5555580d74565de6836366"}, - {file = "yarl-1.6.3-cp36-cp36m-win32.whl", hash = "sha256:b5dfc9a40c198334f4f3f55880ecf910adebdcb2a0b9a9c23c9345faa9185721"}, - {file = "yarl-1.6.3-cp36-cp36m-win_amd64.whl", hash = "sha256:b2e9a456c121e26d13c29251f8267541bd75e6a1ccf9e859179701c36a078643"}, - {file = "yarl-1.6.3-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:ce3beb46a72d9f2190f9e1027886bfc513702d748047b548b05dab7dfb584d2e"}, - {file = "yarl-1.6.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:2ce4c621d21326a4a5500c25031e102af589edb50c09b321049e388b3934eec3"}, - {file = "yarl-1.6.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:d26608cf178efb8faa5ff0f2d2e77c208f471c5a3709e577a7b3fd0445703ac8"}, - {file = "yarl-1.6.3-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:4c5bcfc3ed226bf6419f7a33982fb4b8ec2e45785a0561eb99274ebbf09fdd6a"}, - {file = "yarl-1.6.3-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:4736eaee5626db8d9cda9eb5282028cc834e2aeb194e0d8b50217d707e98bb5c"}, - {file = "yarl-1.6.3-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:68dc568889b1c13f1e4745c96b931cc94fdd0defe92a72c2b8ce01091b22e35f"}, - {file = "yarl-1.6.3-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:7356644cbed76119d0b6bd32ffba704d30d747e0c217109d7979a7bc36c4d970"}, - {file = "yarl-1.6.3-cp37-cp37m-win32.whl", hash = "sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e"}, - {file = "yarl-1.6.3-cp37-cp37m-win_amd64.whl", hash = "sha256:69ee97c71fee1f63d04c945f56d5d726483c4762845400a6795a3b75d56b6c50"}, - {file = "yarl-1.6.3-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:e46fba844f4895b36f4c398c5af062a9808d1f26b2999c58909517384d5deda2"}, - {file = "yarl-1.6.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:31ede6e8c4329fb81c86706ba8f6bf661a924b53ba191b27aa5fcee5714d18ec"}, - {file = "yarl-1.6.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71"}, - {file = "yarl-1.6.3-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:72a660bdd24497e3e84f5519e57a9ee9220b6f3ac4d45056961bf22838ce20cc"}, - {file = "yarl-1.6.3-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:324ba3d3c6fee56e2e0b0d09bf5c73824b9f08234339d2b788af65e60040c959"}, - {file = "yarl-1.6.3-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:e6b5460dc5ad42ad2b36cca524491dfcaffbfd9c8df50508bddc354e787b8dc2"}, - {file = "yarl-1.6.3-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:6d6283d8e0631b617edf0fd726353cb76630b83a089a40933043894e7f6721e2"}, - {file = "yarl-1.6.3-cp38-cp38-win32.whl", hash = "sha256:9ede61b0854e267fd565e7527e2f2eb3ef8858b301319be0604177690e1a3896"}, - {file = "yarl-1.6.3-cp38-cp38-win_amd64.whl", hash = "sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a"}, - {file = "yarl-1.6.3-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:329412812ecfc94a57cd37c9d547579510a9e83c516bc069470db5f75684629e"}, - {file = "yarl-1.6.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:c49ff66d479d38ab863c50f7bb27dee97c6627c5fe60697de15529da9c3de724"}, - {file = "yarl-1.6.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:f040bcc6725c821a4c0665f3aa96a4d0805a7aaf2caf266d256b8ed71b9f041c"}, - {file = "yarl-1.6.3-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:d5c32c82990e4ac4d8150fd7652b972216b204de4e83a122546dce571c1bdf25"}, - {file = "yarl-1.6.3-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:d597767fcd2c3dc49d6eea360c458b65643d1e4dbed91361cf5e36e53c1f8c96"}, - {file = "yarl-1.6.3-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:8aa3decd5e0e852dc68335abf5478a518b41bf2ab2f330fe44916399efedfae0"}, - {file = "yarl-1.6.3-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:73494d5b71099ae8cb8754f1df131c11d433b387efab7b51849e7e1e851f07a4"}, - {file = "yarl-1.6.3-cp39-cp39-win32.whl", hash = "sha256:5b883e458058f8d6099e4420f0cc2567989032b5f34b271c0827de9f1079a424"}, - {file = "yarl-1.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6"}, - {file = "yarl-1.6.3.tar.gz", hash = "sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10"}, + {file = "yarl-1.7.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f2a8508f7350512434e41065684076f640ecce176d262a7d54f0da41d99c5a95"}, + {file = "yarl-1.7.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:da6df107b9ccfe52d3a48165e48d72db0eca3e3029b5b8cb4fe6ee3cb870ba8b"}, + {file = "yarl-1.7.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1d0894f238763717bdcfea74558c94e3bc34aeacd3351d769460c1a586a8b05"}, + {file = "yarl-1.7.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfe4b95b7e00c6635a72e2d00b478e8a28bfb122dc76349a06e20792eb53a523"}, + {file = "yarl-1.7.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c145ab54702334c42237a6c6c4cc08703b6aa9b94e2f227ceb3d477d20c36c63"}, + {file = "yarl-1.7.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1ca56f002eaf7998b5fcf73b2421790da9d2586331805f38acd9997743114e98"}, + {file = "yarl-1.7.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1d3d5ad8ea96bd6d643d80c7b8d5977b4e2fb1bab6c9da7322616fd26203d125"}, + {file = "yarl-1.7.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:167ab7f64e409e9bdd99333fe8c67b5574a1f0495dcfd905bc7454e766729b9e"}, + {file = "yarl-1.7.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:95a1873b6c0dd1c437fb3bb4a4aaa699a48c218ac7ca1e74b0bee0ab16c7d60d"}, + {file = "yarl-1.7.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6152224d0a1eb254f97df3997d79dadd8bb2c1a02ef283dbb34b97d4f8492d23"}, + {file = "yarl-1.7.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5bb7d54b8f61ba6eee541fba4b83d22b8a046b4ef4d8eb7f15a7e35db2e1e245"}, + {file = "yarl-1.7.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:9c1f083e7e71b2dd01f7cd7434a5f88c15213194df38bc29b388ccdf1492b739"}, + {file = "yarl-1.7.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f44477ae29025d8ea87ec308539f95963ffdc31a82f42ca9deecf2d505242e72"}, + {file = "yarl-1.7.2-cp310-cp310-win32.whl", hash = "sha256:cff3ba513db55cc6a35076f32c4cdc27032bd075c9faef31fec749e64b45d26c"}, + {file = "yarl-1.7.2-cp310-cp310-win_amd64.whl", hash = "sha256:c9c6d927e098c2d360695f2e9d38870b2e92e0919be07dbe339aefa32a090265"}, + {file = "yarl-1.7.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9b4c77d92d56a4c5027572752aa35082e40c561eec776048330d2907aead891d"}, + {file = "yarl-1.7.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c01a89a44bb672c38f42b49cdb0ad667b116d731b3f4c896f72302ff77d71656"}, + {file = "yarl-1.7.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c19324a1c5399b602f3b6e7db9478e5b1adf5cf58901996fc973fe4fccd73eed"}, + {file = "yarl-1.7.2-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3abddf0b8e41445426d29f955b24aeecc83fa1072be1be4e0d194134a7d9baee"}, + {file = "yarl-1.7.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6a1a9fe17621af43e9b9fcea8bd088ba682c8192d744b386ee3c47b56eaabb2c"}, + {file = "yarl-1.7.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8b0915ee85150963a9504c10de4e4729ae700af11df0dc5550e6587ed7891e92"}, + {file = "yarl-1.7.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:29e0656d5497733dcddc21797da5a2ab990c0cb9719f1f969e58a4abac66234d"}, + {file = "yarl-1.7.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:bf19725fec28452474d9887a128e98dd67eee7b7d52e932e6949c532d820dc3b"}, + {file = "yarl-1.7.2-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:d6f3d62e16c10e88d2168ba2d065aa374e3c538998ed04996cd373ff2036d64c"}, + {file = "yarl-1.7.2-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ac10bbac36cd89eac19f4e51c032ba6b412b3892b685076f4acd2de18ca990aa"}, + {file = "yarl-1.7.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:aa32aaa97d8b2ed4e54dc65d241a0da1c627454950f7d7b1f95b13985afd6c5d"}, + {file = "yarl-1.7.2-cp36-cp36m-win32.whl", hash = "sha256:87f6e082bce21464857ba58b569370e7b547d239ca22248be68ea5d6b51464a1"}, + {file = "yarl-1.7.2-cp36-cp36m-win_amd64.whl", hash = "sha256:ac35ccde589ab6a1870a484ed136d49a26bcd06b6a1c6397b1967ca13ceb3913"}, + {file = "yarl-1.7.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a467a431a0817a292121c13cbe637348b546e6ef47ca14a790aa2fa8cc93df63"}, + {file = "yarl-1.7.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ab0c3274d0a846840bf6c27d2c60ba771a12e4d7586bf550eefc2df0b56b3b4"}, + {file = "yarl-1.7.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d260d4dc495c05d6600264a197d9d6f7fc9347f21d2594926202fd08cf89a8ba"}, + {file = "yarl-1.7.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fc4dd8b01a8112809e6b636b00f487846956402834a7fd59d46d4f4267181c41"}, + {file = "yarl-1.7.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c1164a2eac148d85bbdd23e07dfcc930f2e633220f3eb3c3e2a25f6148c2819e"}, + {file = "yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:67e94028817defe5e705079b10a8438b8cb56e7115fa01640e9c0bb3edf67332"}, + {file = "yarl-1.7.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:89ccbf58e6a0ab89d487c92a490cb5660d06c3a47ca08872859672f9c511fc52"}, + {file = "yarl-1.7.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8cce6f9fa3df25f55521fbb5c7e4a736683148bcc0c75b21863789e5185f9185"}, + {file = "yarl-1.7.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:211fcd65c58bf250fb994b53bc45a442ddc9f441f6fec53e65de8cba48ded986"}, + {file = "yarl-1.7.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c10ea1e80a697cf7d80d1ed414b5cb8f1eec07d618f54637067ae3c0334133c4"}, + {file = "yarl-1.7.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:52690eb521d690ab041c3919666bea13ab9fbff80d615ec16fa81a297131276b"}, + {file = "yarl-1.7.2-cp37-cp37m-win32.whl", hash = "sha256:695ba021a9e04418507fa930d5f0704edbce47076bdcfeeaba1c83683e5649d1"}, + {file = "yarl-1.7.2-cp37-cp37m-win_amd64.whl", hash = "sha256:c17965ff3706beedafd458c452bf15bac693ecd146a60a06a214614dc097a271"}, + {file = "yarl-1.7.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:fce78593346c014d0d986b7ebc80d782b7f5e19843ca798ed62f8e3ba8728576"}, + {file = "yarl-1.7.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c2a1ac41a6aa980db03d098a5531f13985edcb451bcd9d00670b03129922cd0d"}, + {file = "yarl-1.7.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:39d5493c5ecd75c8093fa7700a2fb5c94fe28c839c8e40144b7ab7ccba6938c8"}, + {file = "yarl-1.7.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1eb6480ef366d75b54c68164094a6a560c247370a68c02dddb11f20c4c6d3c9d"}, + {file = "yarl-1.7.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ba63585a89c9885f18331a55d25fe81dc2d82b71311ff8bd378fc8004202ff6"}, + {file = "yarl-1.7.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e39378894ee6ae9f555ae2de332d513a5763276a9265f8e7cbaeb1b1ee74623a"}, + {file = "yarl-1.7.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c0910c6b6c31359d2f6184828888c983d54d09d581a4a23547a35f1d0b9484b1"}, + {file = "yarl-1.7.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6feca8b6bfb9eef6ee057628e71e1734caf520a907b6ec0d62839e8293e945c0"}, + {file = "yarl-1.7.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8300401dc88cad23f5b4e4c1226f44a5aa696436a4026e456fe0e5d2f7f486e6"}, + {file = "yarl-1.7.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:788713c2896f426a4e166b11f4ec538b5736294ebf7d5f654ae445fd44270832"}, + {file = "yarl-1.7.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:fd547ec596d90c8676e369dd8a581a21227fe9b4ad37d0dc7feb4ccf544c2d59"}, + {file = "yarl-1.7.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:737e401cd0c493f7e3dd4db72aca11cfe069531c9761b8ea474926936b3c57c8"}, + {file = "yarl-1.7.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:baf81561f2972fb895e7844882898bda1eef4b07b5b385bcd308d2098f1a767b"}, + {file = "yarl-1.7.2-cp38-cp38-win32.whl", hash = "sha256:ede3b46cdb719c794427dcce9d8beb4abe8b9aa1e97526cc20de9bd6583ad1ef"}, + {file = "yarl-1.7.2-cp38-cp38-win_amd64.whl", hash = "sha256:cc8b7a7254c0fc3187d43d6cb54b5032d2365efd1df0cd1749c0c4df5f0ad45f"}, + {file = "yarl-1.7.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:580c1f15500e137a8c37053e4cbf6058944d4c114701fa59944607505c2fe3a0"}, + {file = "yarl-1.7.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3ec1d9a0d7780416e657f1e405ba35ec1ba453a4f1511eb8b9fbab81cb8b3ce1"}, + {file = "yarl-1.7.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3bf8cfe8856708ede6a73907bf0501f2dc4e104085e070a41f5d88e7faf237f3"}, + {file = "yarl-1.7.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1be4bbb3d27a4e9aa5f3df2ab61e3701ce8fcbd3e9846dbce7c033a7e8136746"}, + {file = "yarl-1.7.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:534b047277a9a19d858cde163aba93f3e1677d5acd92f7d10ace419d478540de"}, + {file = "yarl-1.7.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c6ddcd80d79c96eb19c354d9dca95291589c5954099836b7c8d29278a7ec0bda"}, + {file = "yarl-1.7.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9bfcd43c65fbb339dc7086b5315750efa42a34eefad0256ba114cd8ad3896f4b"}, + {file = "yarl-1.7.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f64394bd7ceef1237cc604b5a89bf748c95982a84bcd3c4bbeb40f685c810794"}, + {file = "yarl-1.7.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:044daf3012e43d4b3538562da94a88fb12a6490652dbc29fb19adfa02cf72eac"}, + {file = "yarl-1.7.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:368bcf400247318382cc150aaa632582d0780b28ee6053cd80268c7e72796dec"}, + {file = "yarl-1.7.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:bab827163113177aee910adb1f48ff7af31ee0289f434f7e22d10baf624a6dfe"}, + {file = "yarl-1.7.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0cba38120db72123db7c58322fa69e3c0efa933040ffb586c3a87c063ec7cae8"}, + {file = "yarl-1.7.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:59218fef177296451b23214c91ea3aba7858b4ae3306dde120224cfe0f7a6ee8"}, + {file = "yarl-1.7.2-cp39-cp39-win32.whl", hash = "sha256:1edc172dcca3f11b38a9d5c7505c83c1913c0addc99cd28e993efeaafdfaa18d"}, + {file = "yarl-1.7.2-cp39-cp39-win_amd64.whl", hash = "sha256:797c2c412b04403d2da075fb93c123df35239cd7b4cc4e0cd9e5839b73f52c58"}, + {file = "yarl-1.7.2.tar.gz", hash = "sha256:45399b46d60c253327a460e99856752009fcee5f5d3c80b2f7c0cae1c38d56dd"}, ] diff --git a/pyproject.toml b/pyproject.toml index 8eac504c5..c764910c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,9 @@ license = "MIT" [tool.poetry.dependencies] python = "3.9.*" +"discord.py" = {url = "https://github.com/Rapptz/discord.py/archive/45d498c1b76deaf3b394d17ccf56112fa691d160.zip"} +# See https://bot-core.pythondiscord.com/ for docs. +bot-core = {url = "https://github.com/python-discord/bot-core/archive/511bcba1b0196cd498c707a525ea56921bd971db.zip"} aio-pika = "~=6.1" aiodns = "~=2.0" aiohttp = "~=3.7" @@ -17,34 +20,34 @@ beautifulsoup4 = "~=4.9" colorama = { version = "~=0.4.3", markers = "sys_platform == 'win32'" } coloredlogs = "~=14.0" deepdiff = "~=4.0" -"discord.py" = "~=1.7.3" emoji = "~=0.6" feedparser = "~=6.0.2" rapidfuzz = "~=1.4" -lxml = "~=4.4" +lxml = "~=4.6" markdownify = "==0.6.1" more_itertools = "~=8.2" python-dateutil = "~=2.8" python-frontmatter = "~=1.0.0" pyyaml = "~=5.1" regex = "==2021.4.4" -sentry-sdk = "~=0.19" +sentry-sdk = "~=1.3" statsd = "~=3.3" +tldextract = "^3.1.2" [tool.poetry.dev-dependencies] coverage = "~=5.0" -coveralls = "~=2.1" flake8 = "~=3.8" flake8-annotations = "~=2.0" flake8-bugbear = "~=20.1" flake8-docstrings = "~=1.4" -flake8-import-order = "~=0.18" flake8-string-format = "~=0.2" flake8-tidy-imports = "~=4.0" flake8-todo = "~=0.7" +flake8-isort = "~=4.0" pep8-naming = "~=0.9" pre-commit = "~=2.1" taskipy = "~=1.7.0" +pip-licenses = "~=3.5.3" python-dotenv = "~=0.17.1" pytest = "~=6.2.4" pytest-cov = "~=2.12.1" @@ -61,11 +64,21 @@ precommit = "pre-commit install" build = "docker build -t ghcr.io/python-discord/bot:latest -f Dockerfile ." push = "docker push ghcr.io/python-discord/bot:latest" test-nocov = "pytest -n auto" -test = "pytest -n auto --cov-report= --cov" +test = "pytest -n auto --cov-report= --cov --ff" +retest = "pytest -n auto --cov-report= --cov --lf" html = "coverage html" report = "coverage report" +isort = "isort ." [tool.coverage.run] branch = true source_pkgs = ["bot"] source = ["tests"] + +[tool.isort] +multi_line_output = 6 +order_by_type = false +case_sensitive = true +combine_as_imports = true +line_length = 120 +atomic = true diff --git a/tests/__init__.py b/tests/__init__.py index 2228110ad..c2b9d12dc 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,5 +1,6 @@ import logging +from bot.log import get_logger -log = logging.getLogger() +log = get_logger() log.setLevel(logging.CRITICAL) diff --git a/tests/base.py b/tests/base.py index d99b9ac31..5e304ea9d 100644 --- a/tests/base.py +++ b/tests/base.py @@ -6,6 +6,7 @@ from typing import Dict import discord from discord.ext import commands +from bot.log import get_logger from tests import helpers @@ -42,7 +43,7 @@ class LoggingTestsMixin: manager when we're testing under the assumption that no log records will be emitted. """ if not isinstance(logger, logging.Logger): - logger = logging.getLogger(logger) + logger = get_logger(logger) if level: level = logging._nameToLevel.get(level, level) @@ -102,4 +103,4 @@ class CommandTestCase(unittest.IsolatedAsyncioTestCase): with self.assertRaises(commands.MissingPermissions) as cm: await cmd.can_run(ctx) - self.assertCountEqual(permissions.keys(), cm.exception.missing_perms) + self.assertCountEqual(permissions.keys(), cm.exception.missing_permissions) diff --git a/tests/bot/exts/backend/sync/test_base.py b/tests/bot/exts/backend/sync/test_base.py index 3ad9db9c3..9dc46005b 100644 --- a/tests/bot/exts/backend/sync/test_base.py +++ b/tests/bot/exts/backend/sync/test_base.py @@ -1,7 +1,6 @@ import unittest from unittest import mock - from bot.api import ResponseCodeError from bot.exts.backend.sync._syncers import Syncer from tests import helpers diff --git a/tests/bot/exts/backend/sync/test_cog.py b/tests/bot/exts/backend/sync/test_cog.py index 22a07313e..fdd0ab74a 100644 --- a/tests/bot/exts/backend/sync/test_cog.py +++ b/tests/bot/exts/backend/sync/test_cog.py @@ -60,13 +60,13 @@ class SyncCogTestCase(unittest.IsolatedAsyncioTestCase): class SyncCogTests(SyncCogTestCase): """Tests for the Sync cog.""" + @mock.patch("bot.utils.scheduling.create_task") @mock.patch.object(Sync, "sync_guild", new_callable=mock.MagicMock) - def test_sync_cog_init(self, sync_guild): + def test_sync_cog_init(self, sync_guild, create_task): """Should instantiate syncers and run a sync for the guild.""" # Reset because a Sync cog was already instantiated in setUp. self.RoleSyncer.reset_mock() self.UserSyncer.reset_mock() - self.bot.loop.create_task = mock.MagicMock() mock_sync_guild_coro = mock.MagicMock() sync_guild.return_value = mock_sync_guild_coro @@ -74,7 +74,8 @@ class SyncCogTests(SyncCogTestCase): Sync(self.bot) sync_guild.assert_called_once_with() - self.bot.loop.create_task.assert_called_once_with(mock_sync_guild_coro) + create_task.assert_called_once() + self.assertEqual(create_task.call_args.args[0], mock_sync_guild_coro) async def test_sync_cog_sync_guild(self): """Roles and users should be synced only if a guild is successfully retrieved.""" diff --git a/tests/bot/exts/backend/sync/test_users.py b/tests/bot/exts/backend/sync/test_users.py index 27932be95..2fc97af2d 100644 --- a/tests/bot/exts/backend/sync/test_users.py +++ b/tests/bot/exts/backend/sync/test_users.py @@ -1,6 +1,8 @@ import unittest from unittest import mock +from discord.errors import NotFound + from bot.exts.backend.sync._syncers import UserSyncer, _Diff from tests import helpers @@ -10,7 +12,7 @@ def fake_user(**kwargs): kwargs.setdefault("id", 43) kwargs.setdefault("name", "bob the test man") kwargs.setdefault("discriminator", 1337) - kwargs.setdefault("roles", [666]) + kwargs.setdefault("roles", [helpers.MockRole(id=666)]) kwargs.setdefault("in_guild", True) return kwargs @@ -134,6 +136,7 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): self.get_mock_member(fake_user()), None ] + guild.fetch_member.side_effect = NotFound(mock.Mock(status=404), "Not found") actual_diff = await UserSyncer._get_diff(guild) expected_diff = ([], [{"id": 63, "in_guild": False}], None) @@ -158,6 +161,7 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): self.get_mock_member(updated_user), None ] + guild.fetch_member.side_effect = NotFound(mock.Mock(status=404), "Not found") actual_diff = await UserSyncer._get_diff(guild) expected_diff = ([new_user], [{"id": 55, "name": "updated"}, {"id": 63, "in_guild": False}], None) @@ -177,6 +181,7 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): self.get_mock_member(fake_user()), None ] + guild.fetch_member.side_effect = NotFound(mock.Mock(status=404), "Not found") actual_diff = await UserSyncer._get_diff(guild) expected_diff = ([], [], None) diff --git a/tests/bot/exts/backend/test_error_handler.py b/tests/bot/exts/backend/test_error_handler.py index 2b0549b98..35fa0ee59 100644 --- a/tests/bot/exts/backend/test_error_handler.py +++ b/tests/bot/exts/backend/test_error_handler.py @@ -107,7 +107,7 @@ class ErrorHandlerTests(unittest.IsolatedAsyncioTestCase): """Should send error with `ctx.send` when error is `CommandOnCooldown`.""" self.ctx.reset_mock() cog = ErrorHandler(self.bot) - error = errors.CommandOnCooldown(10, 9) + error = errors.CommandOnCooldown(10, 9, type=None) self.assertIsNone(await cog.on_command_error(self.ctx, error)) self.ctx.send.assert_awaited_once_with(error) @@ -337,14 +337,12 @@ class TryGetTagTests(unittest.IsolatedAsyncioTestCase): async def test_try_get_tag_get_command(self): """Should call `Bot.get_command` with `tags get` argument.""" self.bot.get_command.reset_mock() - self.ctx.invoked_with = "foo" await self.cog.try_get_tag(self.ctx) self.bot.get_command.assert_called_once_with("tags get") async def test_try_get_tag_invoked_from_error_handler(self): """`self.ctx` should have `invoked_from_error_handler` `True`.""" self.ctx.invoked_from_error_handler = False - self.ctx.invoked_with = "foo" await self.cog.try_get_tag(self.ctx) self.assertTrue(self.ctx.invoked_from_error_handler) @@ -359,38 +357,12 @@ class TryGetTagTests(unittest.IsolatedAsyncioTestCase): err = errors.CommandError() self.tag.get_command.can_run = AsyncMock(side_effect=err) self.cog.on_command_error = AsyncMock() - self.ctx.invoked_with = "foo" self.assertIsNone(await self.cog.try_get_tag(self.ctx)) self.cog.on_command_error.assert_awaited_once_with(self.ctx, err) - @patch("bot.exts.backend.error_handler.TagNameConverter") - async def test_try_get_tag_convert_success(self, tag_converter): - """Converting tag should successful.""" - self.ctx.invoked_with = "foo" - tag_converter.convert = AsyncMock(return_value="foo") - self.assertIsNone(await self.cog.try_get_tag(self.ctx)) - tag_converter.convert.assert_awaited_once_with(self.ctx, "foo") - self.ctx.invoke.assert_awaited_once() - - @patch("bot.exts.backend.error_handler.TagNameConverter") - async def test_try_get_tag_convert_fail(self, tag_converter): - """Converting tag should raise `BadArgument`.""" - self.ctx.reset_mock() - self.ctx.invoked_with = "bar" - tag_converter.convert = AsyncMock(side_effect=errors.BadArgument()) - self.assertIsNone(await self.cog.try_get_tag(self.ctx)) - self.ctx.invoke.assert_not_awaited() - - async def test_try_get_tag_ctx_invoke(self): - """Should call `ctx.invoke` with proper args/kwargs.""" - self.ctx.reset_mock() - self.ctx.invoked_with = "foo" - self.assertIsNone(await self.cog.try_get_tag(self.ctx)) - self.ctx.invoke.assert_awaited_once_with(self.tag.get_command, tag_name="foo") - async def test_dont_call_suggestion_tag_sent(self): """Should never call command suggestion if tag is already sent.""" - self.ctx.invoked_with = "foo" + self.ctx.message = MagicMock(content="foo") self.ctx.invoke = AsyncMock(return_value=True) self.cog.send_command_suggestion = AsyncMock() @@ -572,38 +544,6 @@ class IndividualErrorHandlerTests(unittest.IsolatedAsyncioTestCase): push_scope_mock.set_extra.has_calls(set_extra_calls) -class OtherErrorHandlerTests(unittest.IsolatedAsyncioTestCase): - """Other `ErrorHandler` tests.""" - - def setUp(self): - self.bot = MockBot() - self.ctx = MockContext() - - async def test_get_help_command_command_specified(self): - """Should return coroutine of help command of specified command.""" - self.ctx.command = "foo" - result = ErrorHandler.get_help_command(self.ctx) - expected = self.ctx.send_help("foo") - self.assertEqual(result.__qualname__, expected.__qualname__) - self.assertEqual(result.cr_frame.f_locals, expected.cr_frame.f_locals) - - # Await coroutines to avoid warnings - await result - await expected - - async def test_get_help_command_no_command_specified(self): - """Should return coroutine of help command.""" - self.ctx.command = None - result = ErrorHandler.get_help_command(self.ctx) - expected = self.ctx.send_help() - self.assertEqual(result.__qualname__, expected.__qualname__) - self.assertEqual(result.cr_frame.f_locals, expected.cr_frame.f_locals) - - # Await coroutines to avoid warnings - await result - await expected - - class ErrorHandlerSetupTests(unittest.TestCase): """Tests for `ErrorHandler` `setup` function.""" diff --git a/tests/bot/exts/events/__init__.py b/tests/bot/exts/events/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/bot/exts/events/__init__.py diff --git a/tests/bot/exts/utils/test_jams.py b/tests/bot/exts/events/test_code_jams.py index 368a15476..0856546af 100644 --- a/tests/bot/exts/utils/test_jams.py +++ b/tests/bot/exts/events/test_code_jams.py @@ -1,14 +1,15 @@ import unittest -from unittest.mock import AsyncMock, MagicMock, create_autospec +from unittest.mock import AsyncMock, MagicMock, create_autospec, patch from discord import CategoryChannel from discord.ext.commands import BadArgument from bot.constants import Roles -from bot.exts.utils import jams +from bot.exts.events import code_jams +from bot.exts.events.code_jams import _channels, _cog from tests.helpers import ( - MockAttachment, MockBot, MockCategoryChannel, MockContext, - MockGuild, MockMember, MockRole, MockTextChannel + MockAttachment, MockBot, MockCategoryChannel, MockContext, MockGuild, MockMember, MockRole, MockTextChannel, + autospec ) TEST_CSV = b"""\ @@ -40,7 +41,7 @@ class JamCodejamCreateTests(unittest.IsolatedAsyncioTestCase): self.command_user = MockMember([self.admin_role]) self.guild = MockGuild([self.admin_role]) self.ctx = MockContext(bot=self.bot, author=self.command_user, guild=self.guild) - self.cog = jams.CodeJams(self.bot) + self.cog = _cog.CodeJams(self.bot) async def test_message_without_attachments(self): """If no link or attachments are provided, commands.BadArgument should be raised.""" @@ -49,7 +50,9 @@ class JamCodejamCreateTests(unittest.IsolatedAsyncioTestCase): with self.assertRaises(BadArgument): await self.cog.create(self.cog, self.ctx, None) - async def test_result_sending(self): + @patch.object(_channels, "create_team_channel") + @patch.object(_channels, "create_team_leader_channel") + async def test_result_sending(self, create_leader_channel, create_team_channel): """Should call `ctx.send` when everything goes right.""" self.ctx.message.attachments = [MockAttachment()] self.ctx.message.attachments[0].read = AsyncMock() @@ -61,14 +64,12 @@ class JamCodejamCreateTests(unittest.IsolatedAsyncioTestCase): self.ctx.guild.create_role = AsyncMock() self.ctx.guild.create_role.return_value = team_leaders - self.cog.create_team_channel = AsyncMock() - self.cog.create_team_leader_channel = AsyncMock() self.cog.add_roles = AsyncMock() await self.cog.create(self.cog, self.ctx, None) - self.cog.create_team_channel.assert_awaited() - self.cog.create_team_leader_channel.assert_awaited_once_with( + create_team_channel.assert_awaited() + create_leader_channel.assert_awaited_once_with( self.ctx.guild, team_leaders ) self.ctx.send.assert_awaited_once() @@ -81,25 +82,24 @@ class JamCodejamCreateTests(unittest.IsolatedAsyncioTestCase): self.ctx.send.assert_awaited_once() - async def test_category_doesnt_exist(self): + @patch.object(_channels, "_send_status_update") + async def test_category_doesnt_exist(self, update): """Should create a new code jam category.""" subtests = ( [], - [get_mock_category(jams.MAX_CHANNELS, jams.CATEGORY_NAME)], - [get_mock_category(jams.MAX_CHANNELS - 2, "other")], + [get_mock_category(_channels.MAX_CHANNELS, _channels.CATEGORY_NAME)], + [get_mock_category(_channels.MAX_CHANNELS - 2, "other")], ) - self.cog.send_status_update = AsyncMock() - for categories in subtests: - self.cog.send_status_update.reset_mock() + update.reset_mock() self.guild.reset_mock() self.guild.categories = categories with self.subTest(categories=categories): - actual_category = await self.cog.get_category(self.guild) + actual_category = await _channels._get_category(self.guild) - self.cog.send_status_update.assert_called_once() + update.assert_called_once() self.guild.create_category_channel.assert_awaited_once() category_overwrites = self.guild.create_category_channel.call_args[1]["overwrites"] @@ -109,45 +109,41 @@ class JamCodejamCreateTests(unittest.IsolatedAsyncioTestCase): async def test_category_channel_exist(self): """Should not try to create category channel.""" - expected_category = get_mock_category(jams.MAX_CHANNELS - 2, jams.CATEGORY_NAME) + expected_category = get_mock_category(_channels.MAX_CHANNELS - 2, _channels.CATEGORY_NAME) self.guild.categories = [ - get_mock_category(jams.MAX_CHANNELS - 2, "other"), + get_mock_category(_channels.MAX_CHANNELS - 2, "other"), expected_category, - get_mock_category(0, jams.CATEGORY_NAME), + get_mock_category(0, _channels.CATEGORY_NAME), ] - actual_category = await self.cog.get_category(self.guild) + actual_category = await _channels._get_category(self.guild) self.assertEqual(expected_category, actual_category) async def test_channel_overwrites(self): """Should have correct permission overwrites for users and roles.""" leader = (MockMember(), True) members = [leader] + [(MockMember(), False) for _ in range(4)] - overwrites = self.cog.get_overwrites(members, self.guild) + overwrites = _channels._get_overwrites(members, self.guild) for member, _ in members: self.assertTrue(overwrites[member].read_messages) - async def test_team_channels_creation(self): + @patch.object(_channels, "_get_overwrites") + @patch.object(_channels, "_get_category") + @autospec(_channels, "_add_team_leader_roles", pass_mocks=False) + async def test_team_channels_creation(self, get_category, get_overwrites): """Should create a text channel for a team.""" team_leaders = MockRole() members = [(MockMember(), True)] + [(MockMember(), False) for _ in range(5)] category = MockCategoryChannel() category.create_text_channel = AsyncMock() - self.cog.get_overwrites = MagicMock() - self.cog.get_category = AsyncMock() - self.cog.get_category.return_value = category - self.cog.add_team_leader_roles = AsyncMock() - - await self.cog.create_team_channel(self.guild, "my-team", members, team_leaders) - self.cog.add_team_leader_roles.assert_awaited_once_with(members, team_leaders) - self.cog.get_overwrites.assert_called_once_with(members, self.guild) - self.cog.get_category.assert_awaited_once_with(self.guild) + get_category.return_value = category + await _channels.create_team_channel(self.guild, "my-team", members, team_leaders) category.create_text_channel.assert_awaited_once_with( "my-team", - overwrites=self.cog.get_overwrites.return_value + overwrites=get_overwrites.return_value ) async def test_jam_roles_adding(self): @@ -156,7 +152,7 @@ class JamCodejamCreateTests(unittest.IsolatedAsyncioTestCase): leader = MockMember() members = [(leader, True)] + [(MockMember(), False) for _ in range(4)] - await self.cog.add_team_leader_roles(members, leader_role) + await _channels._add_team_leader_roles(members, leader_role) leader.add_roles.assert_awaited_once_with(leader_role) for member, is_leader in members: @@ -170,5 +166,5 @@ class CodeJamSetup(unittest.TestCase): def test_setup(self): """Should call `bot.add_cog`.""" bot = MockBot() - jams.setup(bot) + code_jams.setup(bot) bot.add_cog.assert_called_once() diff --git a/tests/bot/exts/filters/test_filtering.py b/tests/bot/exts/filters/test_filtering.py new file mode 100644 index 000000000..8ae59c1f1 --- /dev/null +++ b/tests/bot/exts/filters/test_filtering.py @@ -0,0 +1,40 @@ +import unittest +from unittest.mock import patch + +from bot.exts.filters import filtering +from tests.helpers import MockBot, autospec + + +class FilteringCogTests(unittest.IsolatedAsyncioTestCase): + """Tests the `Filtering` cog.""" + + def setUp(self): + """Instantiate the bot and cog.""" + self.bot = MockBot() + with patch("bot.utils.scheduling.create_task", new=lambda task, **_: task.close()): + self.cog = filtering.Filtering(self.bot) + + @autospec(filtering.Filtering, "_get_filterlist_items", pass_mocks=False, return_value=["TOKEN"]) + async def test_token_filter(self): + """Ensure that a filter token is correctly detected in a message.""" + messages = { + "": False, + "no matches": False, + "TOKEN": True, + + # See advisory https://github.com/python-discord/bot/security/advisories/GHSA-j8c3-8x46-8pp6 + "https://google.com TOKEN": True, + "https://google.com something else": False, + } + + for message, match in messages.items(): + with self.subTest(input=message, match=match): + result, _ = await self.cog._has_watch_regex_match(message) + + self.assertEqual( + match, + bool(result), + msg=f"Hit was {'expected' if match else 'not expected'} for this input." + ) + if result: + self.assertEqual("TOKEN", result.group()) diff --git a/tests/bot/exts/filters/test_token_remover.py b/tests/bot/exts/filters/test_token_remover.py index 51feae9cb..4db27269a 100644 --- a/tests/bot/exts/filters/test_token_remover.py +++ b/tests/bot/exts/filters/test_token_remover.py @@ -26,7 +26,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): self.msg.guild.get_member.return_value.bot = False self.msg.guild.get_member.return_value.__str__.return_value = "Woody" self.msg.author.__str__ = MagicMock(return_value=self.msg.author.name) - self.msg.author.avatar_url_as.return_value = "picture-lemon.png" + self.msg.author.display_avatar.url = "picture-lemon.png" def test_extract_user_id_valid(self): """Should consider user IDs valid if they decode into an integer ID.""" @@ -295,20 +295,21 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): ) @autospec("bot.exts.filters.token_remover", "UNKNOWN_USER_LOG_MESSAGE") - def test_format_userid_log_message_unknown(self, unknown_user_log_message): + async def test_format_userid_log_message_unknown(self, unknown_user_log_message,): """Should correctly format the user ID portion when the actual user it belongs to is unknown.""" token = Token("NDcyMjY1OTQzMDYyNDEzMzMy", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") unknown_user_log_message.format.return_value = " Partner" msg = MockMessage(id=555, content="hello world") msg.guild.get_member.return_value = None + msg.guild.fetch_member.side_effect = NotFound(mock.Mock(status=404), "Not found") - return_value = TokenRemover.format_userid_log_message(msg, token) + return_value = await TokenRemover.format_userid_log_message(msg, token) self.assertEqual(return_value, (unknown_user_log_message.format.return_value, False)) unknown_user_log_message.format.assert_called_once_with(user_id=472265943062413332) @autospec("bot.exts.filters.token_remover", "KNOWN_USER_LOG_MESSAGE") - def test_format_userid_log_message_bot(self, known_user_log_message): + async def test_format_userid_log_message_bot(self, known_user_log_message): """Should correctly format the user ID portion when the ID belongs to a known bot.""" token = Token("NDcyMjY1OTQzMDYyNDEzMzMy", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") known_user_log_message.format.return_value = " Partner" @@ -316,7 +317,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): msg.guild.get_member.return_value.__str__.return_value = "Sam" msg.guild.get_member.return_value.bot = True - return_value = TokenRemover.format_userid_log_message(msg, token) + return_value = await TokenRemover.format_userid_log_message(msg, token) self.assertEqual(return_value, (known_user_log_message.format.return_value, True)) @@ -327,12 +328,12 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): ) @autospec("bot.exts.filters.token_remover", "KNOWN_USER_LOG_MESSAGE") - def test_format_log_message_user_token_user(self, user_token_message): + async def test_format_log_message_user_token_user(self, user_token_message): """Should correctly format the user ID portion when the ID belongs to a known user.""" token = Token("NDY3MjIzMjMwNjUwNzc3NjQx", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") user_token_message.format.return_value = "Partner" - return_value = TokenRemover.format_userid_log_message(self.msg, token) + return_value = await TokenRemover.format_userid_log_message(self.msg, token) self.assertEqual(return_value, (user_token_message.format.return_value, True)) user_token_message.format.assert_called_once_with( @@ -375,7 +376,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): colour=Colour(constants.Colours.soft_red), title="Token removed!", text=log_msg + "\n" + userid_log_message, - thumbnail=self.msg.author.avatar_url_as.return_value, + thumbnail=self.msg.author.display_avatar.url, channel_id=constants.Channels.mod_alerts, ping_everyone=True, ) diff --git a/tests/bot/exts/info/test_information.py b/tests/bot/exts/info/test_information.py index 770660fe3..d896b7652 100644 --- a/tests/bot/exts/info/test_information.py +++ b/tests/bot/exts/info/test_information.py @@ -1,6 +1,7 @@ import textwrap import unittest import unittest.mock +from datetime import datetime import discord @@ -42,7 +43,7 @@ class InformationCogTests(unittest.IsolatedAsyncioTestCase): embed = kwargs.pop('embed') self.assertEqual(embed.title, "Role information (Total 1 role)") - self.assertEqual(embed.colour, discord.Colour.blurple()) + self.assertEqual(embed.colour, discord.Colour.og_blurple()) self.assertEqual(embed.description, f"\n`{self.moderator_role.id}` - {self.moderator_role.mention}\n") async def test_role_info_command(self): @@ -50,7 +51,7 @@ class InformationCogTests(unittest.IsolatedAsyncioTestCase): dummy_role = helpers.MockRole( name="Dummy", id=112233445566778899, - colour=discord.Colour.blurple(), + colour=discord.Colour.og_blurple(), position=10, members=[self.ctx.author], permissions=discord.Permissions(0) @@ -80,7 +81,7 @@ class InformationCogTests(unittest.IsolatedAsyncioTestCase): admin_embed = admin_kwargs["embed"] self.assertEqual(dummy_embed.title, "Dummy info") - self.assertEqual(dummy_embed.colour, discord.Colour.blurple()) + self.assertEqual(dummy_embed.colour, discord.Colour.og_blurple()) self.assertEqual(dummy_embed.fields[0].value, str(dummy_role.id)) self.assertEqual(dummy_embed.fields[1].value, f"#{dummy_role.colour.value:0>6x}") @@ -262,7 +263,6 @@ class UserInfractionHelperMethodTests(unittest.IsolatedAsyncioTestCase): await self._method_subtests(self.cog.user_nomination_counts, test_values, header) [email protected]("bot.exts.info.information.time_since", new=unittest.mock.MagicMock(return_value="1 year ago")) @unittest.mock.patch("bot.exts.info.information.constants.MODERATION_CHANNELS", new=[50]) class UserEmbedTests(unittest.IsolatedAsyncioTestCase): """Tests for the creation of the `!user` embed.""" @@ -277,6 +277,10 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) ) + @unittest.mock.patch( + f"{COG_PATH}.user_messages", + new=unittest.mock.AsyncMock(return_value=("Messsages", "user message count")) + ) async def test_create_user_embed_uses_string_representation_of_user_in_title_if_nick_is_not_available(self): """The embed should use the string representation of the user if they don't have a nick.""" ctx = helpers.MockContext(channel=helpers.MockTextChannel(id=1)) @@ -285,8 +289,9 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): user.nick = None user.__str__ = unittest.mock.Mock(return_value="Mr. Hemlock") user.colour = 0 + user.created_at = user.joined_at = datetime.utcnow() - embed = await self.cog.create_user_embed(ctx, user) + embed = await self.cog.create_user_embed(ctx, user, False) self.assertEqual(embed.title, "Mr. Hemlock") @@ -294,6 +299,10 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) ) + @unittest.mock.patch( + f"{COG_PATH}.user_messages", + new=unittest.mock.AsyncMock(return_value=("Messsages", "user message count")) + ) async def test_create_user_embed_uses_nick_in_title_if_available(self): """The embed should use the nick if it's available.""" ctx = helpers.MockContext(channel=helpers.MockTextChannel(id=1)) @@ -302,8 +311,9 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): user.nick = "Cat lover" user.__str__ = unittest.mock.Mock(return_value="Mr. Hemlock") user.colour = 0 + user.created_at = user.joined_at = datetime.utcnow() - embed = await self.cog.create_user_embed(ctx, user) + embed = await self.cog.create_user_embed(ctx, user, False) self.assertEqual(embed.title, "Cat lover (Mr. Hemlock)") @@ -311,6 +321,10 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) ) + @unittest.mock.patch( + f"{COG_PATH}.user_messages", + new=unittest.mock.AsyncMock(return_value=("Messsages", "user message count")) + ) async def test_create_user_embed_ignores_everyone_role(self): """Created `!user` embeds should not contain mention of the @everyone-role.""" ctx = helpers.MockContext(channel=helpers.MockTextChannel(id=1)) @@ -318,14 +332,19 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): # A `MockMember` has the @Everyone role by default; we add the Admins to that. user = helpers.MockMember(roles=[admins_role], colour=100) + user.created_at = user.joined_at = datetime.utcnow() - embed = await self.cog.create_user_embed(ctx, user) + embed = await self.cog.create_user_embed(ctx, user, False) self.assertIn("&Admins", embed.fields[1].value) self.assertNotIn("&Everyone", embed.fields[1].value) @unittest.mock.patch(f"{COG_PATH}.expanded_user_infraction_counts", new_callable=unittest.mock.AsyncMock) @unittest.mock.patch(f"{COG_PATH}.user_nomination_counts", new_callable=unittest.mock.AsyncMock) + @unittest.mock.patch( + f"{COG_PATH}.user_messages", + new=unittest.mock.AsyncMock(return_value=("Messsages", "user message count")) + ) async def test_create_user_embed_expanded_information_in_moderation_channels( self, nomination_counts, @@ -340,14 +359,15 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): nomination_counts.return_value = ("Nominations", "nomination info") user = helpers.MockMember(id=314, roles=[moderators_role], colour=100) - embed = await self.cog.create_user_embed(ctx, user) + user.created_at = user.joined_at = datetime.utcfromtimestamp(1) + embed = await self.cog.create_user_embed(ctx, user, False) infraction_counts.assert_called_once_with(user) nomination_counts.assert_called_once_with(user) self.assertEqual( textwrap.dedent(f""" - Created: {"1 year ago"} + Created: {"<t:1:R>"} Profile: {user.mention} ID: {user.id} """).strip(), @@ -356,7 +376,7 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): self.assertEqual( textwrap.dedent(f""" - Joined: {"1 year ago"} + Joined: {"<t:1:R>"} Verified: {"True"} Roles: &Moderators """).strip(), @@ -364,22 +384,29 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): ) @unittest.mock.patch(f"{COG_PATH}.basic_user_infraction_counts", new_callable=unittest.mock.AsyncMock) - async def test_create_user_embed_basic_information_outside_of_moderation_channels(self, infraction_counts): + @unittest.mock.patch(f"{COG_PATH}.user_messages", new_callable=unittest.mock.AsyncMock) + async def test_create_user_embed_basic_information_outside_of_moderation_channels( + self, + user_messages, + infraction_counts, + ): """The embed should contain only basic infraction data outside of mod channels.""" ctx = helpers.MockContext(channel=helpers.MockTextChannel(id=100)) moderators_role = helpers.MockRole(name='Moderators') infraction_counts.return_value = ("Infractions", "basic infractions info") + user_messages.return_value = ("Messages", "user message counts") user = helpers.MockMember(id=314, roles=[moderators_role], colour=100) - embed = await self.cog.create_user_embed(ctx, user) + user.created_at = user.joined_at = datetime.utcfromtimestamp(1) + embed = await self.cog.create_user_embed(ctx, user, False) infraction_counts.assert_called_once_with(user) self.assertEqual( textwrap.dedent(f""" - Created: {"1 year ago"} + Created: {"<t:1:R>"} Profile: {user.mention} ID: {user.id} """).strip(), @@ -388,21 +415,30 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): self.assertEqual( textwrap.dedent(f""" - Joined: {"1 year ago"} + Joined: {"<t:1:R>"} Roles: &Moderators """).strip(), embed.fields[1].value ) self.assertEqual( - "basic infractions info", + "user message counts", embed.fields[2].value ) + self.assertEqual( + "basic infractions info", + embed.fields[3].value + ) + @unittest.mock.patch( f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) ) + @unittest.mock.patch( + f"{COG_PATH}.user_messages", + new=unittest.mock.AsyncMock(return_value=("Messsages", "user message count")) + ) async def test_create_user_embed_uses_top_role_colour_when_user_has_roles(self): """The embed should be created with the colour of the top role, if a top role is available.""" ctx = helpers.MockContext() @@ -410,7 +446,8 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): moderators_role = helpers.MockRole(name='Moderators') user = helpers.MockMember(id=314, roles=[moderators_role], colour=100) - embed = await self.cog.create_user_embed(ctx, user) + user.created_at = user.joined_at = datetime.utcnow() + embed = await self.cog.create_user_embed(ctx, user, False) self.assertEqual(embed.colour, discord.Colour(100)) @@ -418,28 +455,37 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) ) - async def test_create_user_embed_uses_blurple_colour_when_user_has_no_roles(self): - """The embed should be created with a blurple colour if the user has no assigned roles.""" + @unittest.mock.patch( + f"{COG_PATH}.user_messages", + new=unittest.mock.AsyncMock(return_value=("Messsages", "user message count")) + ) + async def test_create_user_embed_uses_og_blurple_colour_when_user_has_no_roles(self): + """The embed should be created with the og blurple colour if the user has no assigned roles.""" ctx = helpers.MockContext() user = helpers.MockMember(id=217, colour=discord.Colour.default()) - embed = await self.cog.create_user_embed(ctx, user) + user.created_at = user.joined_at = datetime.utcnow() + embed = await self.cog.create_user_embed(ctx, user, False) - self.assertEqual(embed.colour, discord.Colour.blurple()) + self.assertEqual(embed.colour, discord.Colour.og_blurple()) @unittest.mock.patch( f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) ) + @unittest.mock.patch( + f"{COG_PATH}.user_messages", + new=unittest.mock.AsyncMock(return_value=("Messsages", "user message count")) + ) async def test_create_user_embed_uses_png_format_of_user_avatar_as_thumbnail(self): """The embed thumbnail should be set to the user's avatar in `png` format.""" ctx = helpers.MockContext() user = helpers.MockMember(id=217, colour=0) - user.avatar_url_as.return_value = "avatar url" - embed = await self.cog.create_user_embed(ctx, user) + user.created_at = user.joined_at = datetime.utcnow() + user.display_avatar.url = "avatar url" + embed = await self.cog.create_user_embed(ctx, user, False) - user.avatar_url_as.assert_called_once_with(static_format="png") self.assertEqual(embed.thumbnail.url, "avatar url") @@ -491,7 +537,7 @@ class UserCommandTests(unittest.IsolatedAsyncioTestCase): await self.cog.user_info(self.cog, ctx) - create_embed.assert_called_once_with(ctx, self.author) + create_embed.assert_called_once_with(ctx, self.author, False) ctx.send.assert_called_once() @unittest.mock.patch("bot.exts.info.information.Information.create_user_embed") @@ -502,28 +548,28 @@ class UserCommandTests(unittest.IsolatedAsyncioTestCase): await self.cog.user_info(self.cog, ctx, self.author) - create_embed.assert_called_once_with(ctx, self.author) + create_embed.assert_called_once_with(ctx, self.author, False) ctx.send.assert_called_once() @unittest.mock.patch("bot.exts.info.information.Information.create_user_embed") async def test_staff_members_can_bypass_channel_restriction(self, create_embed, constants): """Staff members should be able to bypass the bot-commands channel restriction.""" - constants.STAFF_ROLES = [self.moderator_role.id] + constants.STAFF_PARTNERS_COMMUNITY_ROLES = [self.moderator_role.id] ctx = helpers.MockContext(author=self.moderator, channel=helpers.MockTextChannel(id=200)) await self.cog.user_info(self.cog, ctx) - create_embed.assert_called_once_with(ctx, self.moderator) + create_embed.assert_called_once_with(ctx, self.moderator, False) ctx.send.assert_called_once() @unittest.mock.patch("bot.exts.info.information.Information.create_user_embed") async def test_moderators_can_target_another_member(self, create_embed, constants): """A moderator should be able to use `!user` targeting another user.""" constants.MODERATION_ROLES = [self.moderator_role.id] - constants.STAFF_ROLES = [self.moderator_role.id] + constants.STAFF_PARTNERS_COMMUNITY_ROLES = [self.moderator_role.id] ctx = helpers.MockContext(author=self.moderator, channel=helpers.MockTextChannel(id=50)) await self.cog.user_info(self.cog, ctx, self.target) - create_embed.assert_called_once_with(ctx, self.target) + create_embed.assert_called_once_with(ctx, self.target, False) ctx.send.assert_called_once() diff --git a/tests/bot/exts/moderation/infraction/test_infractions.py b/tests/bot/exts/moderation/infraction/test_infractions.py index b9d527770..052048053 100644 --- a/tests/bot/exts/moderation/infraction/test_infractions.py +++ b/tests/bot/exts/moderation/infraction/test_infractions.py @@ -1,11 +1,15 @@ import inspect import textwrap import unittest -from unittest.mock import ANY, AsyncMock, MagicMock, Mock, patch +from unittest.mock import ANY, AsyncMock, DEFAULT, MagicMock, Mock, patch + +from discord.errors import NotFound from bot.constants import Event +from bot.exts.moderation.clean import Clean from bot.exts.moderation.infraction import _utils from bot.exts.moderation.infraction.infractions import Infractions +from bot.exts.moderation.infraction.management import ModManagement from tests.helpers import MockBot, MockContext, MockGuild, MockMember, MockRole, MockUser, autospec @@ -13,12 +17,13 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): """Tests for ban and kick command reason truncation.""" def setUp(self): + self.me = MockMember(id=7890, roles=[MockRole(id=7890, position=5)]) self.bot = MockBot() self.cog = Infractions(self.bot) - self.user = MockMember(id=1234, top_role=MockRole(id=3577, position=10)) - self.target = MockMember(id=1265, top_role=MockRole(id=9876, position=0)) + self.user = MockMember(id=1234, roles=[MockRole(id=3577, position=10)]) + self.target = MockMember(id=1265, roles=[MockRole(id=9876, position=1)]) self.guild = MockGuild(id=4567) - self.ctx = MockContext(bot=self.bot, author=self.user, guild=self.guild) + self.ctx = MockContext(me=self.me, bot=self.bot, author=self.user, guild=self.guild) @patch("bot.exts.moderation.infraction._utils.get_active_infraction") @patch("bot.exts.moderation.infraction._utils.post_infraction") @@ -59,70 +64,70 @@ class TruncationTests(unittest.IsolatedAsyncioTestCase): @patch("bot.exts.moderation.infraction.infractions.constants.Roles.voice_verified", new=123456) -class VoiceBanTests(unittest.IsolatedAsyncioTestCase): - """Tests for voice ban related functions and commands.""" +class VoiceMuteTests(unittest.IsolatedAsyncioTestCase): + """Tests for voice mute related functions and commands.""" def setUp(self): self.bot = MockBot() - self.mod = MockMember(top_role=10) - self.user = MockMember(top_role=1, roles=[MockRole(id=123456)]) + self.mod = MockMember(roles=[MockRole(id=7890123, position=10)]) + self.user = MockMember(roles=[MockRole(id=123456, position=1)]) self.guild = MockGuild() self.ctx = MockContext(bot=self.bot, author=self.mod) self.cog = Infractions(self.bot) - async def test_permanent_voice_ban(self): - """Should call voice ban applying function without expiry.""" - self.cog.apply_voice_ban = AsyncMock() - self.assertIsNone(await self.cog.voiceban(self.cog, self.ctx, self.user, reason="foobar")) - self.cog.apply_voice_ban.assert_awaited_once_with(self.ctx, self.user, "foobar", expires_at=None) + async def test_permanent_voice_mute(self): + """Should call voice mute applying function without expiry.""" + self.cog.apply_voice_mute = AsyncMock() + self.assertIsNone(await self.cog.voicemute(self.cog, self.ctx, self.user, reason="foobar")) + self.cog.apply_voice_mute.assert_awaited_once_with(self.ctx, self.user, "foobar", expires_at=None) - async def test_temporary_voice_ban(self): - """Should call voice ban applying function with expiry.""" - self.cog.apply_voice_ban = AsyncMock() - self.assertIsNone(await self.cog.tempvoiceban(self.cog, self.ctx, self.user, "baz", reason="foobar")) - self.cog.apply_voice_ban.assert_awaited_once_with(self.ctx, self.user, "foobar", expires_at="baz") + async def test_temporary_voice_mute(self): + """Should call voice mute applying function with expiry.""" + self.cog.apply_voice_mute = AsyncMock() + self.assertIsNone(await self.cog.tempvoicemute(self.cog, self.ctx, self.user, "baz", reason="foobar")) + self.cog.apply_voice_mute.assert_awaited_once_with(self.ctx, self.user, "foobar", expires_at="baz") - async def test_voice_unban(self): + async def test_voice_unmute(self): """Should call infraction pardoning function.""" self.cog.pardon_infraction = AsyncMock() - self.assertIsNone(await self.cog.unvoiceban(self.cog, self.ctx, self.user)) - self.cog.pardon_infraction.assert_awaited_once_with(self.ctx, "voice_ban", self.user) + self.assertIsNone(await self.cog.unvoicemute(self.cog, self.ctx, self.user)) + self.cog.pardon_infraction.assert_awaited_once_with(self.ctx, "voice_mute", self.user) @patch("bot.exts.moderation.infraction.infractions._utils.post_infraction") @patch("bot.exts.moderation.infraction.infractions._utils.get_active_infraction") - async def test_voice_ban_user_have_active_infraction(self, get_active_infraction, post_infraction_mock): - """Should return early when user already have Voice Ban infraction.""" + async def test_voice_mute_user_have_active_infraction(self, get_active_infraction, post_infraction_mock): + """Should return early when user already have Voice Mute infraction.""" get_active_infraction.return_value = {"foo": "bar"} - self.assertIsNone(await self.cog.apply_voice_ban(self.ctx, self.user, "foobar")) - get_active_infraction.assert_awaited_once_with(self.ctx, self.user, "voice_ban") + self.assertIsNone(await self.cog.apply_voice_mute(self.ctx, self.user, "foobar")) + get_active_infraction.assert_awaited_once_with(self.ctx, self.user, "voice_mute") post_infraction_mock.assert_not_awaited() @patch("bot.exts.moderation.infraction.infractions._utils.post_infraction") @patch("bot.exts.moderation.infraction.infractions._utils.get_active_infraction") - async def test_voice_ban_infraction_post_failed(self, get_active_infraction, post_infraction_mock): + async def test_voice_mute_infraction_post_failed(self, get_active_infraction, post_infraction_mock): """Should return early when posting infraction fails.""" self.cog.mod_log.ignore = MagicMock() get_active_infraction.return_value = None post_infraction_mock.return_value = None - self.assertIsNone(await self.cog.apply_voice_ban(self.ctx, self.user, "foobar")) + self.assertIsNone(await self.cog.apply_voice_mute(self.ctx, self.user, "foobar")) post_infraction_mock.assert_awaited_once() self.cog.mod_log.ignore.assert_not_called() @patch("bot.exts.moderation.infraction.infractions._utils.post_infraction") @patch("bot.exts.moderation.infraction.infractions._utils.get_active_infraction") - async def test_voice_ban_infraction_post_add_kwargs(self, get_active_infraction, post_infraction_mock): - """Should pass all kwargs passed to apply_voice_ban to post_infraction.""" + async def test_voice_mute_infraction_post_add_kwargs(self, get_active_infraction, post_infraction_mock): + """Should pass all kwargs passed to apply_voice_mute to post_infraction.""" get_active_infraction.return_value = None # We don't want that this continue yet post_infraction_mock.return_value = None - self.assertIsNone(await self.cog.apply_voice_ban(self.ctx, self.user, "foobar", my_kwarg=23)) + self.assertIsNone(await self.cog.apply_voice_mute(self.ctx, self.user, "foobar", my_kwarg=23)) post_infraction_mock.assert_awaited_once_with( - self.ctx, self.user, "voice_ban", "foobar", active=True, my_kwarg=23 + self.ctx, self.user, "voice_mute", "foobar", active=True, my_kwarg=23 ) @patch("bot.exts.moderation.infraction.infractions._utils.post_infraction") @patch("bot.exts.moderation.infraction.infractions._utils.get_active_infraction") - async def test_voice_ban_mod_log_ignore(self, get_active_infraction, post_infraction_mock): + async def test_voice_mute_mod_log_ignore(self, get_active_infraction, post_infraction_mock): """Should ignore Voice Verified role removing.""" self.cog.mod_log.ignore = MagicMock() self.cog.apply_infraction = AsyncMock() @@ -131,11 +136,11 @@ class VoiceBanTests(unittest.IsolatedAsyncioTestCase): get_active_infraction.return_value = None post_infraction_mock.return_value = {"foo": "bar"} - self.assertIsNone(await self.cog.apply_voice_ban(self.ctx, self.user, "foobar")) + self.assertIsNone(await self.cog.apply_voice_mute(self.ctx, self.user, "foobar")) self.cog.mod_log.ignore.assert_called_once_with(Event.member_update, self.user.id) async def action_tester(self, action, reason: str) -> None: - """Helper method to test voice ban action.""" + """Helper method to test voice mute action.""" self.assertTrue(inspect.iscoroutine(action)) await action @@ -144,7 +149,7 @@ class VoiceBanTests(unittest.IsolatedAsyncioTestCase): @patch("bot.exts.moderation.infraction.infractions._utils.post_infraction") @patch("bot.exts.moderation.infraction.infractions._utils.get_active_infraction") - async def test_voice_ban_apply_infraction(self, get_active_infraction, post_infraction_mock): + async def test_voice_mute_apply_infraction(self, get_active_infraction, post_infraction_mock): """Should ignore Voice Verified role removing.""" self.cog.mod_log.ignore = MagicMock() self.cog.apply_infraction = AsyncMock() @@ -153,22 +158,22 @@ class VoiceBanTests(unittest.IsolatedAsyncioTestCase): post_infraction_mock.return_value = {"foo": "bar"} reason = "foobar" - self.assertIsNone(await self.cog.apply_voice_ban(self.ctx, self.user, reason)) + self.assertIsNone(await self.cog.apply_voice_mute(self.ctx, self.user, reason)) self.cog.apply_infraction.assert_awaited_once_with(self.ctx, {"foo": "bar"}, self.user, ANY) await self.action_tester(self.cog.apply_infraction.call_args[0][-1], reason) @patch("bot.exts.moderation.infraction.infractions._utils.post_infraction") @patch("bot.exts.moderation.infraction.infractions._utils.get_active_infraction") - async def test_voice_ban_truncate_reason(self, get_active_infraction, post_infraction_mock): - """Should truncate reason for voice ban.""" + async def test_voice_mute_truncate_reason(self, get_active_infraction, post_infraction_mock): + """Should truncate reason for voice mute.""" self.cog.mod_log.ignore = MagicMock() self.cog.apply_infraction = AsyncMock() get_active_infraction.return_value = None post_infraction_mock.return_value = {"foo": "bar"} - self.assertIsNone(await self.cog.apply_voice_ban(self.ctx, self.user, "foobar" * 3000)) + self.assertIsNone(await self.cog.apply_voice_mute(self.ctx, self.user, "foobar" * 3000)) self.cog.apply_infraction.assert_awaited_once_with(self.ctx, {"foo": "bar"}, self.user, ANY) # Test action @@ -177,14 +182,14 @@ class VoiceBanTests(unittest.IsolatedAsyncioTestCase): @autospec(_utils, "post_infraction", "get_active_infraction", return_value=None) @autospec(Infractions, "apply_infraction") - async def test_voice_ban_user_left_guild(self, apply_infraction_mock, post_infraction_mock, _): - """Should voice ban user that left the guild without throwing an error.""" + async def test_voice_mute_user_left_guild(self, apply_infraction_mock, post_infraction_mock, _): + """Should voice mute user that left the guild without throwing an error.""" infraction = {"foo": "bar"} post_infraction_mock.return_value = {"foo": "bar"} user = MockUser() - await self.cog.voiceban(self.cog, self.ctx, user, reason=None) - post_infraction_mock.assert_called_once_with(self.ctx, user, "voice_ban", None, active=True, expires_at=None) + await self.cog.voicemute(self.cog, self.ctx, user, reason=None) + post_infraction_mock.assert_called_once_with(self.ctx, user, "voice_mute", None, active=True, expires_at=None) apply_infraction_mock.assert_called_once_with(self.cog, self.ctx, infraction, user, ANY) # Test action @@ -192,21 +197,22 @@ class VoiceBanTests(unittest.IsolatedAsyncioTestCase): self.assertTrue(inspect.iscoroutine(action)) await action - async def test_voice_unban_user_not_found(self): + async def test_voice_unmute_user_not_found(self): """Should include info to return dict when user was not found from guild.""" self.guild.get_member.return_value = None - result = await self.cog.pardon_voice_ban(self.user.id, self.guild, "foobar") + self.guild.fetch_member.side_effect = NotFound(Mock(status=404), "Not found") + result = await self.cog.pardon_voice_mute(self.user.id, self.guild) self.assertEqual(result, {"Info": "User was not found in the guild."}) @patch("bot.exts.moderation.infraction.infractions._utils.notify_pardon") @patch("bot.exts.moderation.infraction.infractions.format_user") - async def test_voice_unban_user_found(self, format_user_mock, notify_pardon_mock): + async def test_voice_unmute_user_found(self, format_user_mock, notify_pardon_mock): """Should add role back with ignoring, notify user and return log dictionary..""" self.guild.get_member.return_value = self.user notify_pardon_mock.return_value = True format_user_mock.return_value = "my-user" - result = await self.cog.pardon_voice_ban(self.user.id, self.guild, "foobar") + result = await self.cog.pardon_voice_mute(self.user.id, self.guild) self.assertEqual(result, { "Member": "my-user", "DM": "Sent" @@ -215,15 +221,100 @@ class VoiceBanTests(unittest.IsolatedAsyncioTestCase): @patch("bot.exts.moderation.infraction.infractions._utils.notify_pardon") @patch("bot.exts.moderation.infraction.infractions.format_user") - async def test_voice_unban_dm_fail(self, format_user_mock, notify_pardon_mock): + async def test_voice_unmute_dm_fail(self, format_user_mock, notify_pardon_mock): """Should add role back with ignoring, notify user and return log dictionary..""" self.guild.get_member.return_value = self.user notify_pardon_mock.return_value = False format_user_mock.return_value = "my-user" - result = await self.cog.pardon_voice_ban(self.user.id, self.guild, "foobar") + result = await self.cog.pardon_voice_mute(self.user.id, self.guild) self.assertEqual(result, { "Member": "my-user", "DM": "**Failed**" }) notify_pardon_mock.assert_awaited_once() + + +class CleanBanTests(unittest.IsolatedAsyncioTestCase): + """Tests for cleanban functionality.""" + + def setUp(self): + self.bot = MockBot() + self.mod = MockMember(roles=[MockRole(id=7890123, position=10)]) + self.user = MockMember(roles=[MockRole(id=123456, position=1)]) + self.guild = MockGuild() + self.ctx = MockContext(bot=self.bot, author=self.mod) + self.cog = Infractions(self.bot) + self.clean_cog = Clean(self.bot) + self.management_cog = ModManagement(self.bot) + + self.cog.apply_ban = AsyncMock(return_value={"id": 42}) + self.log_url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + self.clean_cog._clean_messages = AsyncMock(return_value=self.log_url) + + def mock_get_cog(self, enable_clean, enable_manage): + """Mock get cog factory that allows the user to specify whether clean and manage cogs are enabled.""" + def inner(name): + if name == "ModManagement": + return self.management_cog if enable_manage else None + elif name == "Clean": + return self.clean_cog if enable_clean else None + else: + return DEFAULT + return inner + + async def test_cleanban_falls_back_to_native_purge_without_clean_cog(self): + """Should fallback to native purge if the Clean cog is not available.""" + self.bot.get_cog.side_effect = self.mock_get_cog(False, False) + + self.assertIsNone(await self.cog.cleanban(self.cog, self.ctx, self.user, None, reason="FooBar")) + self.cog.apply_ban.assert_awaited_once_with( + self.ctx, + self.user, + "FooBar", + purge_days=1, + expires_at=None, + ) + + async def test_cleanban_doesnt_purge_messages_if_clean_cog_available(self): + """Cleanban command should use the native purge messages if the clean cog is available.""" + self.bot.get_cog.side_effect = self.mock_get_cog(True, False) + + self.assertIsNone(await self.cog.cleanban(self.cog, self.ctx, self.user, None, reason="FooBar")) + self.cog.apply_ban.assert_awaited_once_with( + self.ctx, + self.user, + "FooBar", + expires_at=None, + ) + + @patch("bot.exts.moderation.infraction.infractions.Age") + async def test_cleanban_uses_clean_cog_when_available(self, mocked_age_converter): + """Test cleanban uses the clean cog to clean messages if it's available.""" + self.bot.api_client.patch = AsyncMock() + self.bot.get_cog.side_effect = self.mock_get_cog(True, False) + + mocked_age_converter.return_value.convert = AsyncMock(return_value="81M") + self.assertIsNone(await self.cog.cleanban(self.cog, self.ctx, self.user, None, reason="FooBar")) + + self.clean_cog._clean_messages.assert_awaited_once_with( + self.ctx, + users=[self.user], + channels="*", + first_limit="81M", + attempt_delete_invocation=False, + ) + + async def test_cleanban_edits_infraction_reason(self): + """Ensure cleanban edits the ban reason with a link to the clean log.""" + self.bot.get_cog.side_effect = self.mock_get_cog(True, True) + + self.management_cog.infraction_append = AsyncMock() + self.assertIsNone(await self.cog.cleanban(self.cog, self.ctx, self.user, None, reason="FooBar")) + + self.management_cog.infraction_append.assert_awaited_once_with( + self.ctx, + {"id": 42}, + None, + reason=f"[Clean log]({self.log_url})" + ) diff --git a/tests/bot/exts/moderation/infraction/test_utils.py b/tests/bot/exts/moderation/infraction/test_utils.py index d35120992..ff81ddd65 100644 --- a/tests/bot/exts/moderation/infraction/test_utils.py +++ b/tests/bot/exts/moderation/infraction/test_utils.py @@ -15,7 +15,10 @@ class ModerationUtilsTests(unittest.IsolatedAsyncioTestCase): """Tests Moderation utils.""" def setUp(self): - self.bot = MockBot() + patcher = patch("bot.instance", new=MockBot()) + self.bot = patcher.start() + self.addCleanup(patcher.stop) + self.member = MockMember(id=1234) self.user = MockUser(id=1234) self.ctx = MockContext(bot=self.bot, author=self.member) @@ -94,8 +97,8 @@ class ModerationUtilsTests(unittest.IsolatedAsyncioTestCase): test_case = namedtuple("test_case", ["get_return_value", "expected_output", "infraction_nr", "send_msg"]) test_cases = [ test_case([], None, None, True), - test_case([{"id": 123987}], {"id": 123987}, "123987", False), - test_case([{"id": 123987}], {"id": 123987}, "123987", True) + test_case([{"id": 123987, "type": "ban"}], {"id": 123987, "type": "ban"}, "123987", False), + test_case([{"id": 123987, "type": "ban"}], {"id": 123987, "type": "ban"}, "123987", True) ] for case in test_cases: @@ -123,6 +126,7 @@ class ModerationUtilsTests(unittest.IsolatedAsyncioTestCase): else: self.ctx.send.assert_not_awaited() + @unittest.skip("Current time needs to be patched so infraction duration is correct.") @patch("bot.exts.moderation.infraction._utils.send_private_embed") async def test_send_infraction_embed(self, send_private_embed_mock): """ @@ -132,95 +136,95 @@ class ModerationUtilsTests(unittest.IsolatedAsyncioTestCase): """ test_cases = [ { - "args": (self.user, "ban", "2020-02-26 09:20 (23 hours and 59 minutes)"), + "args": (dict(id=0, type="ban", reason=None, expires_at=datetime(2020, 2, 26, 9, 20)), self.user), "expected_output": Embed( title=utils.INFRACTION_TITLE, description=utils.INFRACTION_DESCRIPTION_TEMPLATE.format( type="Ban", - expires="2020-02-26 09:20 (23 hours and 59 minutes) UTC", + expires="2020-02-26 09:20 (23 hours and 59 minutes)", reason="No reason provided." - ), + ) + utils.INFRACTION_APPEAL_SERVER_FOOTER, colour=Colours.soft_red, url=utils.RULES_URL ).set_author( name=utils.INFRACTION_AUTHOR_NAME, url=utils.RULES_URL, - icon_url=Icons.token_removed - ).set_footer(text=utils.INFRACTION_APPEAL_MODMAIL_FOOTER), + icon_url=Icons.user_ban + ), "send_result": True }, { - "args": (self.user, "warning", None, "Test reason."), + "args": (dict(id=0, type="warning", reason="Test reason.", expires_at=None), self.user), "expected_output": Embed( title=utils.INFRACTION_TITLE, description=utils.INFRACTION_DESCRIPTION_TEMPLATE.format( type="Warning", expires="N/A", reason="Test reason." - ), + ) + utils.INFRACTION_APPEAL_MODMAIL_FOOTER, colour=Colours.soft_red, url=utils.RULES_URL ).set_author( name=utils.INFRACTION_AUTHOR_NAME, url=utils.RULES_URL, - icon_url=Icons.token_removed - ).set_footer(text=utils.INFRACTION_APPEAL_MODMAIL_FOOTER), + icon_url=Icons.user_warn + ), "send_result": False }, # Note that this test case asserts that the DM that *would* get sent to the user is formatted # correctly, even though that message is deliberately never sent. { - "args": (self.user, "note", None, None, Icons.defcon_denied), + "args": (dict(id=0, type="note", reason=None, expires_at=None), self.user), "expected_output": Embed( title=utils.INFRACTION_TITLE, description=utils.INFRACTION_DESCRIPTION_TEMPLATE.format( type="Note", expires="N/A", reason="No reason provided." - ), + ) + utils.INFRACTION_APPEAL_MODMAIL_FOOTER, colour=Colours.soft_red, url=utils.RULES_URL ).set_author( name=utils.INFRACTION_AUTHOR_NAME, url=utils.RULES_URL, - icon_url=Icons.defcon_denied - ).set_footer(text=utils.INFRACTION_APPEAL_MODMAIL_FOOTER), + icon_url=Icons.user_warn + ), "send_result": False }, { - "args": (self.user, "mute", "2020-02-26 09:20 (23 hours and 59 minutes)", "Test", Icons.defcon_denied), + "args": (dict(id=0, type="mute", reason="Test", expires_at=datetime(2020, 2, 26, 9, 20)), self.user), "expected_output": Embed( title=utils.INFRACTION_TITLE, description=utils.INFRACTION_DESCRIPTION_TEMPLATE.format( type="Mute", - expires="2020-02-26 09:20 (23 hours and 59 minutes) UTC", + expires="2020-02-26 09:20 (23 hours and 59 minutes)", reason="Test" - ), + ) + utils.INFRACTION_APPEAL_MODMAIL_FOOTER, colour=Colours.soft_red, url=utils.RULES_URL ).set_author( name=utils.INFRACTION_AUTHOR_NAME, url=utils.RULES_URL, - icon_url=Icons.defcon_denied - ).set_footer(text=utils.INFRACTION_APPEAL_MODMAIL_FOOTER), + icon_url=Icons.user_mute + ), "send_result": False }, { - "args": (self.user, "mute", None, "foo bar" * 4000, Icons.defcon_denied), + "args": (dict(id=0, type="mute", reason="foo bar" * 4000, expires_at=None), self.user), "expected_output": Embed( title=utils.INFRACTION_TITLE, description=utils.INFRACTION_DESCRIPTION_TEMPLATE.format( type="Mute", expires="N/A", reason="foo bar" * 4000 - )[:2045] + "...", + )[:4093-utils.LONGEST_EXTRAS] + "..." + utils.INFRACTION_APPEAL_MODMAIL_FOOTER, colour=Colours.soft_red, url=utils.RULES_URL ).set_author( name=utils.INFRACTION_AUTHOR_NAME, url=utils.RULES_URL, - icon_url=Icons.defcon_denied - ).set_footer(text=utils.INFRACTION_APPEAL_MODMAIL_FOOTER), + icon_url=Icons.user_mute + ), "send_result": True } ] @@ -230,7 +234,7 @@ class ModerationUtilsTests(unittest.IsolatedAsyncioTestCase): send_private_embed_mock.reset_mock() send_private_embed_mock.return_value = case["send_result"] - result = await utils.send_infraction_embed(*case["args"]) + result = await utils.notify_infraction(*case["args"]) self.assertEqual(case["send_result"], result) @@ -238,7 +242,7 @@ class ModerationUtilsTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(embed.to_dict(), case["expected_output"].to_dict()) - send_private_embed_mock.assert_awaited_once_with(case["args"][0], embed) + send_private_embed_mock.assert_awaited_once_with(case["args"][1], embed) @patch("bot.exts.moderation.infraction._utils.send_private_embed") async def test_notify_pardon(self, send_private_embed_mock): @@ -313,7 +317,8 @@ class TestPostInfraction(unittest.IsolatedAsyncioTestCase): "type": "ban", "user": self.member.id, "active": False, - "expires_at": now.isoformat() + "expires_at": now.isoformat(), + "dm_sent": False } self.ctx.bot.api_client.post.return_value = "foo" @@ -350,7 +355,8 @@ class TestPostInfraction(unittest.IsolatedAsyncioTestCase): "reason": "Test reason", "type": "mute", "user": self.user.id, - "active": True + "active": True, + "dm_sent": False } self.bot.api_client.post.side_effect = [ResponseCodeError(MagicMock(status=400), {"user": "foo"}), "foo"] diff --git a/tests/bot/exts/moderation/test_clean.py b/tests/bot/exts/moderation/test_clean.py new file mode 100644 index 000000000..d7647fa48 --- /dev/null +++ b/tests/bot/exts/moderation/test_clean.py @@ -0,0 +1,104 @@ +import unittest +from unittest.mock import AsyncMock, MagicMock, patch + +from bot.exts.moderation.clean import Clean +from tests.helpers import MockBot, MockContext, MockGuild, MockMember, MockMessage, MockRole, MockTextChannel + + +class CleanTests(unittest.IsolatedAsyncioTestCase): + """Tests for clean cog functionality.""" + + def setUp(self): + self.bot = MockBot() + self.mod = MockMember(roles=[MockRole(id=7890123, position=10)]) + self.user = MockMember(roles=[MockRole(id=123456, position=1)]) + self.guild = MockGuild() + self.ctx = MockContext(bot=self.bot, author=self.mod) + self.cog = Clean(self.bot) + + self.log_url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + self.cog._modlog_cleaned_messages = AsyncMock(return_value=self.log_url) + + self.cog._use_cache = MagicMock(return_value=True) + self.cog._delete_found = AsyncMock(return_value=[42, 84]) + + @patch("bot.exts.moderation.clean.is_mod_channel") + async def test_clean_deletes_invocation_in_non_mod_channel(self, mod_channel_check): + """Clean command should delete the invocation message if ran in a non mod channel.""" + mod_channel_check.return_value = False + self.ctx.message.delete = AsyncMock() + + self.assertIsNone(await self.cog._delete_invocation(self.ctx)) + + self.ctx.message.delete.assert_awaited_once() + + @patch("bot.exts.moderation.clean.is_mod_channel") + async def test_clean_doesnt_delete_invocation_in_mod_channel(self, mod_channel_check): + """Clean command should not delete the invocation message if ran in a mod channel.""" + mod_channel_check.return_value = True + self.ctx.message.delete = AsyncMock() + + self.assertIsNone(await self.cog._delete_invocation(self.ctx)) + + self.ctx.message.delete.assert_not_awaited() + + async def test_clean_doesnt_attempt_deletion_when_attempt_delete_invocation_is_false(self): + """Clean command should not attempt to delete the invocation message if attempt_delete_invocation is false.""" + self.cog._delete_invocation = AsyncMock() + self.bot.get_channel = MagicMock(return_value=False) + + self.assertEqual( + await self.cog._clean_messages( + self.ctx, + None, + first_limit=MockMessage(), + attempt_delete_invocation=False, + ), + self.log_url, + ) + + self.cog._delete_invocation.assert_not_awaited() + + @patch("bot.exts.moderation.clean.is_mod_channel") + async def test_clean_replies_with_success_message_when_ran_in_mod_channel(self, mod_channel_check): + """Clean command should reply to the message with a confirmation message if invoked in a mod channel.""" + mod_channel_check.return_value = True + self.ctx.reply = AsyncMock() + + self.assertEqual( + await self.cog._clean_messages( + self.ctx, + None, + first_limit=MockMessage(), + attempt_delete_invocation=False, + ), + self.log_url, + ) + + self.ctx.reply.assert_awaited_once() + sent_message = self.ctx.reply.await_args[0][0] + self.assertIn(self.log_url, sent_message) + self.assertIn("2 messages", sent_message) + + @patch("bot.exts.moderation.clean.is_mod_channel") + async def test_clean_send_success_message_to_mods_when_ran_in_non_mod_channel(self, mod_channel_check): + """Clean command should send a confirmation message to #mods if invoked in a non-mod channel.""" + mod_channel_check.return_value = False + mocked_mods = MockTextChannel(id=1234567) + mocked_mods.send = AsyncMock() + self.bot.get_channel = MagicMock(return_value=mocked_mods) + + self.assertEqual( + await self.cog._clean_messages( + self.ctx, + None, + first_limit=MockMessage(), + attempt_delete_invocation=False, + ), + self.log_url, + ) + + mocked_mods.send.assert_awaited_once() + sent_message = mocked_mods.send.await_args[0][0] + self.assertIn(self.log_url, sent_message) + self.assertIn("2 messages", sent_message) diff --git a/tests/bot/exts/moderation/test_incidents.py b/tests/bot/exts/moderation/test_incidents.py index cbf7f7bcf..cfe0c4b03 100644 --- a/tests/bot/exts/moderation/test_incidents.py +++ b/tests/bot/exts/moderation/test_incidents.py @@ -3,23 +3,19 @@ import enum import logging import typing as t import unittest -from unittest.mock import AsyncMock, MagicMock, call, patch +from unittest import mock +from unittest.mock import AsyncMock, MagicMock, Mock, call, patch import aiohttp import discord +from async_rediscache import RedisSession from bot.constants import Colours from bot.exts.moderation import incidents +from bot.utils.messages import format_user from tests.helpers import ( - MockAsyncWebhook, - MockAttachment, - MockBot, - MockMember, - MockMessage, - MockReaction, - MockRole, - MockTextChannel, - MockUser, + MockAsyncWebhook, MockAttachment, MockBot, MockMember, MockMessage, MockReaction, MockRole, MockTextChannel, + MockUser ) @@ -283,6 +279,22 @@ class TestIncidents(unittest.IsolatedAsyncioTestCase): the instance as they wish. """ + session = None + + async def flush(self): + """Flush everything from the database to prevent carry-overs between tests.""" + with await self.session.pool as connection: + await connection.flushall() + + async def asyncSetUp(self): # noqa: N802 + self.session = RedisSession(use_fakeredis=True) + await self.session.connect() + await self.flush() + + async def asyncTearDown(self): # noqa: N802 + if self.session: + await self.session.close() + def setUp(self): """ Prepare a fresh `Incidents` instance for each test. @@ -379,7 +391,7 @@ class TestArchive(TestIncidents): # Define our own `incident` to be archived incident = MockMessage( content="this is an incident", - author=MockUser(name="author_name", avatar_url="author_avatar"), + author=MockUser(name="author_name", display_avatar=Mock(url="author_avatar")), id=123, ) built_embed = MagicMock(discord.Embed, id=123) # We patch `make_embed` to return this @@ -513,7 +525,7 @@ class TestProcessEvent(TestIncidents): with patch("bot.exts.moderation.incidents.Incidents.make_confirmation_task", mock_task): await self.cog_instance.process_event( reaction=incidents.Signal.ACTIONED.value, - incident=MockMessage(), + incident=MockMessage(id=123), member=MockMember(roles=[MockRole(id=1)]) ) @@ -533,7 +545,7 @@ class TestProcessEvent(TestIncidents): with patch("bot.exts.moderation.incidents.Incidents.make_confirmation_task", mock_task): await self.cog_instance.process_event( reaction=incidents.Signal.ACTIONED.value, - incident=MockMessage(), + incident=MockMessage(id=123), member=MockMember(roles=[MockRole(id=1)]) ) except asyncio.TimeoutError: @@ -768,3 +780,74 @@ class TestOnMessage(TestIncidents): await self.cog_instance.on_message(MockMessage()) mock_add_signals.assert_not_called() + + +class TestMessageLinkEmbeds(TestIncidents): + """Tests for `extract_message_links` coroutine.""" + + async def test_shorten_text(self): + """Test all cases of text shortening by mocking messages.""" + tests = { + "thisisasingleword"*10: "thisisasinglewordthisisasinglewordthisisasinglewor...", + + "\n".join("Lets make a new line test".split()): "Lets\nmake\na...", + + 'Hello, World!' * 300: ( + "Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!" + "Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!" + "Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!" + "Hello, World!Hello, World!H..." + ) + } + + for content, expected_conversion in tests.items(): + with self.subTest(content=content, expected_conversion=expected_conversion): + conversion = incidents.shorten_text(content) + self.assertEqual(conversion, expected_conversion) + + async def extract_and_form_message_link_embeds(self): + """ + Extract message links from a mocked message and form the message link embed. + + Considers all types of message links, discord supports. + """ + self.guild_id_patcher = mock.patch("bot.exts.backend.sync._cog.constants.Guild.id", 5) + self.guild_id = self.guild_id_patcher.start() + + msg = MockMessage(id=555, content="Hello, World!" * 3000) + msg.channel.mention = "#lemonade-stand" + + msg_links = [ + # Valid Message links + f"https://discord.com/channels/{self.guild_id}/{msg.channel.discord_id}/{msg.discord_id}", + f"http://canary.discord.com/channels/{self.guild_id}/{msg.channel.discord_id}/{msg.discord_id}", + + # Invalid Message links + f"https://discord.com/channels/{msg.channel.discord_id}/{msg.discord_id}", + f"https://discord.com/channels/{self.guild_id}/{msg.channel.discord_id}000/{msg.discord_id}", + ] + + incident_msg = MockMessage( + id=777, + content=( + f"I would like to report the following messages, " + f"as they break our rules: \n{', '.join(msg_links)}" + ) + ) + + with patch( + "bot.exts.moderation.incidents.Incidents.extract_message_links", AsyncMock() + ) as mock_extract_message_links: + embeds = mock_extract_message_links(incident_msg) + description = ( + f"**Author:** {format_user(msg.author)}\n" + f"**Channel:** {msg.channel.mention} ({msg.channel.category}/#{msg.channel.name})\n" + f"**Content:** {('Hello, World!' * 3000)[:300] + '...'}\n" + ) + + # Check number of embeds returned with number of valid links + self.assertEqual(len(embeds), 2) + + # Check for the embed descriptions + for embed in embeds: + self.assertEqual(embed.description, description) diff --git a/tests/bot/exts/moderation/test_modlog.py b/tests/bot/exts/moderation/test_modlog.py index f8f142484..79e04837d 100644 --- a/tests/bot/exts/moderation/test_modlog.py +++ b/tests/bot/exts/moderation/test_modlog.py @@ -25,5 +25,5 @@ class ModLogTests(unittest.IsolatedAsyncioTestCase): ) embed = self.channel.send.call_args[1]["embed"] self.assertEqual( - embed.description, ("foo bar" * 3000)[:2045] + "..." + embed.description, ("foo bar" * 3000)[:4093] + "..." ) diff --git a/tests/bot/exts/moderation/test_silence.py b/tests/bot/exts/moderation/test_silence.py index 59a5893ef..92ce3418a 100644 --- a/tests/bot/exts/moderation/test_silence.py +++ b/tests/bot/exts/moderation/test_silence.py @@ -12,14 +12,7 @@ from discord import PermissionOverwrite from bot.constants import Channels, Guild, MODERATION_ROLES, Roles from bot.exts.moderation import silence from tests.helpers import ( - MockBot, - MockContext, - MockGuild, - MockMember, - MockRole, - MockTextChannel, - MockVoiceChannel, - autospec + MockBot, MockContext, MockGuild, MockMember, MockRole, MockTextChannel, MockVoiceChannel, autospec ) redis_session = None @@ -438,7 +431,13 @@ class SilenceTests(unittest.IsolatedAsyncioTestCase): asyncio.run(self.cog._async_init()) # Populate instance attributes. self.text_channel = MockTextChannel() - self.text_overwrite = PermissionOverwrite(send_messages=True, add_reactions=False) + self.text_overwrite = PermissionOverwrite( + send_messages=True, + add_reactions=False, + create_private_threads=True, + create_public_threads=False, + send_messages_in_threads=True + ) self.text_channel.overwrites_for.return_value = self.text_overwrite self.voice_channel = MockVoiceChannel() @@ -509,9 +508,39 @@ class SilenceTests(unittest.IsolatedAsyncioTestCase): async def test_skipped_already_silenced(self): """Permissions were not set and `False` was returned for an already silenced channel.""" subtests = ( - (False, MockTextChannel(), PermissionOverwrite(send_messages=False, add_reactions=False)), - (True, MockTextChannel(), PermissionOverwrite(send_messages=True, add_reactions=True)), - (True, MockTextChannel(), PermissionOverwrite(send_messages=False, add_reactions=False)), + ( + False, + MockTextChannel(), + PermissionOverwrite( + send_messages=False, + add_reactions=False, + create_private_threads=False, + create_public_threads=False, + send_messages_in_threads=False + ) + ), + ( + True, + MockTextChannel(), + PermissionOverwrite( + send_messages=True, + add_reactions=True, + create_private_threads=True, + create_public_threads=True, + send_messages_in_threads=True + ) + ), + ( + True, + MockTextChannel(), + PermissionOverwrite( + send_messages=False, + add_reactions=False, + create_private_threads=False, + create_public_threads=False, + send_messages_in_threads=False + ) + ), (False, MockVoiceChannel(), PermissionOverwrite(connect=False, speak=False)), (True, MockVoiceChannel(), PermissionOverwrite(connect=True, speak=True)), (True, MockVoiceChannel(), PermissionOverwrite(connect=False, speak=False)), @@ -559,11 +588,16 @@ class SilenceTests(unittest.IsolatedAsyncioTestCase): await self.cog._set_silence_overwrites(self.text_channel) new_overwrite_dict = dict(self.text_overwrite) - # Remove 'send_messages' & 'add_reactions' keys because they were changed by the method. - del prev_overwrite_dict['send_messages'] - del prev_overwrite_dict['add_reactions'] - del new_overwrite_dict['send_messages'] - del new_overwrite_dict['add_reactions'] + # Remove related permission keys because they were changed by the method. + for perm_name in ( + "send_messages", + "add_reactions", + "create_private_threads", + "create_public_threads", + "send_messages_in_threads" + ): + del prev_overwrite_dict[perm_name] + del new_overwrite_dict[perm_name] self.assertDictEqual(prev_overwrite_dict, new_overwrite_dict) @@ -601,7 +635,10 @@ class SilenceTests(unittest.IsolatedAsyncioTestCase): async def test_cached_previous_overwrites(self): """Channel's previous overwrites were cached.""" - overwrite_json = '{"send_messages": true, "add_reactions": false}' + overwrite_json = ( + '{"send_messages": true, "add_reactions": false, "create_private_threads": true, ' + '"create_public_threads": false, "send_messages_in_threads": true}' + ) await self.cog._set_silence_overwrites(self.text_channel) self.cog.previous_overwrites.set.assert_awaited_once_with(self.text_channel.id, overwrite_json) diff --git a/tests/bot/exts/utils/test_snekbox.py b/tests/bot/exts/utils/test_snekbox.py index 321a92445..8bdeedd27 100644 --- a/tests/bot/exts/utils/test_snekbox.py +++ b/tests/bot/exts/utils/test_snekbox.py @@ -2,6 +2,7 @@ import asyncio import unittest from unittest.mock import AsyncMock, MagicMock, Mock, call, create_autospec, patch +from discord import AllowedMentions from discord.ext import commands from bot import constants @@ -201,7 +202,7 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): ctx = MockContext() ctx.message = MockMessage() ctx.send = AsyncMock() - ctx.author.mention = '@LemonLemonishBeard#0042' + ctx.author = MockUser(mention='@LemonLemonishBeard#0042') self.cog.post_eval = AsyncMock(return_value={'stdout': '', 'returncode': 0}) self.cog.get_results_message = MagicMock(return_value=('Return code 0', '')) @@ -213,9 +214,16 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): self.bot.get_cog.return_value = mocked_filter_cog await self.cog.send_eval(ctx, 'MyAwesomeCode') - ctx.send.assert_called_once_with( + + ctx.send.assert_called_once() + self.assertEqual( + ctx.send.call_args.args[0], '@LemonLemonishBeard#0042 :yay!: Return code 0.\n\n```\n[No output]\n```' ) + allowed_mentions = ctx.send.call_args.kwargs['allowed_mentions'] + expected_allowed_mentions = AllowedMentions(everyone=False, roles=False, users=[ctx.author]) + self.assertEqual(allowed_mentions.to_dict(), expected_allowed_mentions.to_dict()) + self.cog.post_eval.assert_called_once_with('MyAwesomeCode') self.cog.get_status_emoji.assert_called_once_with({'stdout': '', 'returncode': 0}) self.cog.get_results_message.assert_called_once_with({'stdout': '', 'returncode': 0}) @@ -238,10 +246,14 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): self.bot.get_cog.return_value = mocked_filter_cog await self.cog.send_eval(ctx, 'MyAwesomeCode') - ctx.send.assert_called_once_with( + + ctx.send.assert_called_once() + self.assertEqual( + ctx.send.call_args.args[0], '@LemonLemonishBeard#0042 :yay!: Return code 0.' '\n\n```\nWay too long beard\n```\nFull output: lookatmybeard.com' ) + self.cog.post_eval.assert_called_once_with('MyAwesomeCode') self.cog.get_status_emoji.assert_called_once_with({'stdout': 'Way too long beard', 'returncode': 0}) self.cog.get_results_message.assert_called_once_with({'stdout': 'Way too long beard', 'returncode': 0}) @@ -263,9 +275,13 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): self.bot.get_cog.return_value = mocked_filter_cog await self.cog.send_eval(ctx, 'MyAwesomeCode') - ctx.send.assert_called_once_with( + + ctx.send.assert_called_once() + self.assertEqual( + ctx.send.call_args.args[0], '@LemonLemonishBeard#0042 :nope!: Return code 127.\n\n```\nBeard got stuck in the eval\n```' ) + self.cog.post_eval.assert_called_once_with('MyAwesomeCode') self.cog.get_status_emoji.assert_called_once_with({'stdout': 'ERROR', 'returncode': 127}) self.cog.get_results_message.assert_called_once_with({'stdout': 'ERROR', 'returncode': 127}) diff --git a/tests/bot/rules/test_mentions.py b/tests/bot/rules/test_mentions.py index 6444532f2..f8805ac48 100644 --- a/tests/bot/rules/test_mentions.py +++ b/tests/bot/rules/test_mentions.py @@ -2,12 +2,14 @@ from typing import Iterable from bot.rules import mentions from tests.bot.rules import DisallowedCase, RuleTest -from tests.helpers import MockMessage +from tests.helpers import MockMember, MockMessage -def make_msg(author: str, total_mentions: int) -> MockMessage: +def make_msg(author: str, total_user_mentions: int, total_bot_mentions: int = 0) -> MockMessage: """Makes a message with `total_mentions` mentions.""" - return MockMessage(author=author, mentions=list(range(total_mentions))) + user_mentions = [MockMember() for _ in range(total_user_mentions)] + bot_mentions = [MockMember(bot=True) for _ in range(total_bot_mentions)] + return MockMessage(author=author, mentions=user_mentions+bot_mentions) class TestMentions(RuleTest): @@ -48,11 +50,27 @@ class TestMentions(RuleTest): [make_msg("bob", 2), make_msg("alice", 3), make_msg("bob", 2)], ("bob",), 4, - ) + ), + DisallowedCase( + [make_msg("bob", 3, 1)], + ("bob",), + 3, + ), ) await self.run_disallowed(cases) + async def test_ignore_bot_mentions(self): + """Messages with an allowed amount of mentions, also containing bot mentions.""" + cases = ( + [make_msg("bob", 0, 3)], + [make_msg("bob", 2, 1)], + [make_msg("bob", 1, 2), make_msg("bob", 1, 2)], + [make_msg("bob", 1, 5), make_msg("alice", 2, 5)] + ) + + await self.run_allowed(cases) + def relevant_messages(self, case: DisallowedCase) -> Iterable[MockMessage]: last_message = case.recent_messages[0] return tuple( diff --git a/tests/bot/test_converters.py b/tests/bot/test_converters.py index 2a1c4e543..1bb678db2 100644 --- a/tests/bot/test_converters.py +++ b/tests/bot/test_converters.py @@ -1,19 +1,12 @@ -import datetime import re import unittest +from datetime import MAXYEAR, datetime, timezone from unittest.mock import MagicMock, patch from dateutil.relativedelta import relativedelta from discord.ext.commands import BadArgument -from bot.converters import ( - Duration, - HushDurationConverter, - ISODateTime, - PackageName, - TagContentConverter, - TagNameConverter, -) +from bot.converters import Duration, HushDurationConverter, ISODateTime, PackageName class ConverterTests(unittest.IsolatedAsyncioTestCase): @@ -24,59 +17,7 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): cls.context = MagicMock cls.context.author = 'bob' - cls.fixed_utc_now = datetime.datetime.fromisoformat('2019-01-01T00:00:00') - - async def test_tag_content_converter_for_valid(self): - """TagContentConverter should return correct values for valid input.""" - test_values = ( - ('hello', 'hello'), - (' h ello ', 'h ello'), - ) - - for content, expected_conversion in test_values: - with self.subTest(content=content, expected_conversion=expected_conversion): - conversion = await TagContentConverter.convert(self.context, content) - self.assertEqual(conversion, expected_conversion) - - async def test_tag_content_converter_for_invalid(self): - """TagContentConverter should raise the proper exception for invalid input.""" - test_values = ( - ('', "Tag contents should not be empty, or filled with whitespace."), - (' ', "Tag contents should not be empty, or filled with whitespace."), - ) - - for value, exception_message in test_values: - with self.subTest(tag_content=value, exception_message=exception_message): - with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): - await TagContentConverter.convert(self.context, value) - - async def test_tag_name_converter_for_valid(self): - """TagNameConverter should return the correct values for valid tag names.""" - test_values = ( - ('tracebacks', 'tracebacks'), - ('Tracebacks', 'tracebacks'), - (' Tracebacks ', 'tracebacks'), - ) - - for name, expected_conversion in test_values: - with self.subTest(name=name, expected_conversion=expected_conversion): - conversion = await TagNameConverter.convert(self.context, name) - self.assertEqual(conversion, expected_conversion) - - async def test_tag_name_converter_for_invalid(self): - """TagNameConverter should raise the correct exception for invalid tag names.""" - test_values = ( - ('👋', "Don't be ridiculous, you can't use that character!"), - ('', "Tag names should not be empty, or filled with whitespace."), - (' ', "Tag names should not be empty, or filled with whitespace."), - ('42', "Tag names must contain at least one letter."), - ('x' * 128, "Are you insane? That's way too long!"), - ) - - for invalid_name, exception_message in test_values: - with self.subTest(invalid_name=invalid_name, exception_message=exception_message): - with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): - await TagNameConverter.convert(self.context, invalid_name) + cls.fixed_utc_now = datetime.fromisoformat('2019-01-01T00:00:00+00:00') async def test_package_name_for_valid(self): """PackageName returns valid package names unchanged.""" @@ -155,7 +96,7 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): expected_datetime = self.fixed_utc_now + relativedelta(**duration_dict) with patch('bot.converters.datetime') as mock_datetime: - mock_datetime.utcnow.return_value = self.fixed_utc_now + mock_datetime.now.return_value = self.fixed_utc_now with self.subTest(duration=duration, duration_dict=duration_dict): converted_datetime = await converter.convert(self.context, duration) @@ -201,52 +142,53 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): async def test_duration_converter_out_of_range(self, mock_datetime): """Duration converter should raise BadArgument if datetime raises a ValueError.""" mock_datetime.__add__.side_effect = ValueError - mock_datetime.utcnow.return_value = mock_datetime + mock_datetime.now.return_value = mock_datetime - duration = f"{datetime.MAXYEAR}y" + duration = f"{MAXYEAR}y" exception_message = f"`{duration}` results in a datetime outside the supported range." with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): await Duration().convert(self.context, duration) async def test_isodatetime_converter_for_valid(self): """ISODateTime converter returns correct datetime for valid datetime string.""" + utc = timezone.utc test_values = ( # `YYYY-mm-ddTHH:MM:SSZ` | `YYYY-mm-dd HH:MM:SSZ` - ('2019-09-02T02:03:05Z', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02 02:03:05Z', datetime.datetime(2019, 9, 2, 2, 3, 5)), + ('2019-09-02T02:03:05Z', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02 02:03:05Z', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), # `YYYY-mm-ddTHH:MM:SS±HH:MM` | `YYYY-mm-dd HH:MM:SS±HH:MM` - ('2019-09-02T03:18:05+01:15', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02 03:18:05+01:15', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02T00:48:05-01:15', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02 00:48:05-01:15', datetime.datetime(2019, 9, 2, 2, 3, 5)), + ('2019-09-02T03:18:05+01:15', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02 03:18:05+01:15', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02T00:48:05-01:15', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02 00:48:05-01:15', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), # `YYYY-mm-ddTHH:MM:SS±HHMM` | `YYYY-mm-dd HH:MM:SS±HHMM` - ('2019-09-02T03:18:05+0115', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02 03:18:05+0115', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02T00:48:05-0115', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02 00:48:05-0115', datetime.datetime(2019, 9, 2, 2, 3, 5)), + ('2019-09-02T03:18:05+0115', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02 03:18:05+0115', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02T00:48:05-0115', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02 00:48:05-0115', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), # `YYYY-mm-ddTHH:MM:SS±HH` | `YYYY-mm-dd HH:MM:SS±HH` - ('2019-09-02 03:03:05+01', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02T01:03:05-01', datetime.datetime(2019, 9, 2, 2, 3, 5)), + ('2019-09-02 03:03:05+01', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02T01:03:05-01', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), # `YYYY-mm-ddTHH:MM:SS` | `YYYY-mm-dd HH:MM:SS` - ('2019-09-02T02:03:05', datetime.datetime(2019, 9, 2, 2, 3, 5)), - ('2019-09-02 02:03:05', datetime.datetime(2019, 9, 2, 2, 3, 5)), + ('2019-09-02T02:03:05', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), + ('2019-09-02 02:03:05', datetime(2019, 9, 2, 2, 3, 5, tzinfo=utc)), # `YYYY-mm-ddTHH:MM` | `YYYY-mm-dd HH:MM` - ('2019-11-12T09:15', datetime.datetime(2019, 11, 12, 9, 15)), - ('2019-11-12 09:15', datetime.datetime(2019, 11, 12, 9, 15)), + ('2019-11-12T09:15', datetime(2019, 11, 12, 9, 15, tzinfo=utc)), + ('2019-11-12 09:15', datetime(2019, 11, 12, 9, 15, tzinfo=utc)), # `YYYY-mm-dd` - ('2019-04-01', datetime.datetime(2019, 4, 1)), + ('2019-04-01', datetime(2019, 4, 1, tzinfo=utc)), # `YYYY-mm` - ('2019-02-01', datetime.datetime(2019, 2, 1)), + ('2019-02-01', datetime(2019, 2, 1, tzinfo=utc)), # `YYYY` - ('2025', datetime.datetime(2025, 1, 1)), + ('2025', datetime(2025, 1, 1, tzinfo=utc)), ) converter = ISODateTime() @@ -254,7 +196,6 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): for datetime_string, expected_dt in test_values: with self.subTest(datetime_string=datetime_string, expected_dt=expected_dt): converted_dt = await converter.convert(self.context, datetime_string) - self.assertIsNone(converted_dt.tzinfo) self.assertEqual(converted_dt, expected_dt) async def test_isodatetime_converter_for_invalid(self): diff --git a/tests/bot/utils/test_checks.py b/tests/bot/utils/test_checks.py index 883465e0b..4ae11d5d3 100644 --- a/tests/bot/utils/test_checks.py +++ b/tests/bot/utils/test_checks.py @@ -32,6 +32,7 @@ class ChecksTests(unittest.IsolatedAsyncioTestCase): async def test_has_no_roles_check_without_guild(self): """`has_no_roles_check` should return `False` when `Context.guild` is None.""" self.ctx.channel = MagicMock(DMChannel) + self.ctx.guild = None self.assertFalse(await checks.has_no_roles_check(self.ctx)) async def test_has_no_roles_check_returns_false_with_unwanted_role(self): diff --git a/tests/bot/utils/test_message_cache.py b/tests/bot/utils/test_message_cache.py new file mode 100644 index 000000000..04bfd28d1 --- /dev/null +++ b/tests/bot/utils/test_message_cache.py @@ -0,0 +1,214 @@ +import unittest + +from bot.utils.message_cache import MessageCache +from tests.helpers import MockMessage + + +# noinspection SpellCheckingInspection +class TestMessageCache(unittest.TestCase): + """Tests for the MessageCache class in the `bot.utils.caching` module.""" + + def test_first_append_sets_the_first_value(self): + """Test if the first append adds the message to the first cell.""" + cache = MessageCache(maxlen=10) + message = MockMessage() + + cache.append(message) + + self.assertEqual(cache[0], message) + + def test_append_adds_in_the_right_order(self): + """Test if two appends are added in the same order if newest_first is False, or in reverse order otherwise.""" + messages = [MockMessage(), MockMessage()] + + cache = MessageCache(maxlen=10, newest_first=False) + for msg in messages: + cache.append(msg) + self.assertListEqual(messages, list(cache)) + + cache = MessageCache(maxlen=10, newest_first=True) + for msg in messages: + cache.append(msg) + self.assertListEqual(messages[::-1], list(cache)) + + def test_appending_over_maxlen_removes_oldest(self): + """Test if three appends to a 2-cell cache leave the two newest messages.""" + cache = MessageCache(maxlen=2) + messages = [MockMessage() for _ in range(3)] + + for msg in messages: + cache.append(msg) + + self.assertListEqual(messages[1:], list(cache)) + + def test_appending_over_maxlen_with_newest_first_removes_oldest(self): + """Test if three appends to a 2-cell cache leave the two newest messages if newest_first is True.""" + cache = MessageCache(maxlen=2, newest_first=True) + messages = [MockMessage() for _ in range(3)] + + for msg in messages: + cache.append(msg) + + self.assertListEqual(messages[:0:-1], list(cache)) + + def test_pop_removes_from_the_end(self): + """Test if a pop removes the right-most message.""" + cache = MessageCache(maxlen=3) + messages = [MockMessage() for _ in range(3)] + + for msg in messages: + cache.append(msg) + msg = cache.pop() + + self.assertEqual(msg, messages[-1]) + self.assertListEqual(messages[:-1], list(cache)) + + def test_popleft_removes_from_the_beginning(self): + """Test if a popleft removes the left-most message.""" + cache = MessageCache(maxlen=3) + messages = [MockMessage() for _ in range(3)] + + for msg in messages: + cache.append(msg) + msg = cache.popleft() + + self.assertEqual(msg, messages[0]) + self.assertListEqual(messages[1:], list(cache)) + + def test_clear(self): + """Test if a clear makes the cache empty.""" + cache = MessageCache(maxlen=5) + messages = [MockMessage() for _ in range(3)] + + for msg in messages: + cache.append(msg) + cache.clear() + + self.assertListEqual(list(cache), []) + self.assertEqual(len(cache), 0) + + def test_get_message_returns_the_message(self): + """Test if get_message returns the cached message.""" + cache = MessageCache(maxlen=5) + message = MockMessage(id=1234) + + cache.append(message) + + self.assertEqual(cache.get_message(1234), message) + + def test_get_message_returns_none(self): + """Test if get_message returns None for an ID of a non-cached message.""" + cache = MessageCache(maxlen=5) + message = MockMessage(id=1234) + + cache.append(message) + + self.assertIsNone(cache.get_message(4321)) + + def test_update_replaces_old_element(self): + """Test if an update replaced the old message with the same ID.""" + cache = MessageCache(maxlen=5) + message = MockMessage(id=1234) + + cache.append(message) + message = MockMessage(id=1234) + cache.update(message) + + self.assertIs(cache.get_message(1234), message) + self.assertEqual(len(cache), 1) + + def test_contains_returns_true_for_cached_message(self): + """Test if contains returns True for an ID of a cached message.""" + cache = MessageCache(maxlen=5) + message = MockMessage(id=1234) + + cache.append(message) + + self.assertIn(1234, cache) + + def test_contains_returns_false_for_non_cached_message(self): + """Test if contains returns False for an ID of a non-cached message.""" + cache = MessageCache(maxlen=5) + message = MockMessage(id=1234) + + cache.append(message) + + self.assertNotIn(4321, cache) + + def test_indexing(self): + """Test if the cache returns the correct messages by index.""" + cache = MessageCache(maxlen=5) + messages = [MockMessage() for _ in range(5)] + + for msg in messages: + cache.append(msg) + + for current_loop in range(-5, 5): + with self.subTest(current_loop=current_loop): + self.assertEqual(cache[current_loop], messages[current_loop]) + + def test_bad_index_raises_index_error(self): + """Test if the cache raises IndexError for invalid indices.""" + cache = MessageCache(maxlen=5) + messages = [MockMessage() for _ in range(3)] + test_cases = (-10, -4, 3, 4, 5) + + for msg in messages: + cache.append(msg) + + for current_loop in test_cases: + with self.subTest(current_loop=current_loop): + with self.assertRaises(IndexError): + cache[current_loop] + + def test_slicing_with_unfilled_cache(self): + """Test if slicing returns the correct messages if the cache is not yet fully filled.""" + sizes = (5, 10, 55, 101) + + slices = ( + slice(None), slice(2, None), slice(None, 2), slice(None, None, 2), slice(None, None, 3), slice(-1, 2), + slice(-1, 3000), slice(-3, -1), slice(-10, 3), slice(-10, 4, 2), slice(None, None, -1), slice(None, 3, -2), + slice(None, None, -3), slice(-1, -10, -2), slice(-3, -7, -1) + ) + + for size in sizes: + cache = MessageCache(maxlen=size) + messages = [MockMessage() for _ in range(size // 3 * 2)] + + for msg in messages: + cache.append(msg) + + for slice_ in slices: + with self.subTest(current_loop=(size, slice_)): + self.assertListEqual(cache[slice_], messages[slice_]) + + def test_slicing_with_overfilled_cache(self): + """Test if slicing returns the correct messages if the cache was appended with more messages it can contain.""" + sizes = (5, 10, 55, 101) + + slices = ( + slice(None), slice(2, None), slice(None, 2), slice(None, None, 2), slice(None, None, 3), slice(-1, 2), + slice(-1, 3000), slice(-3, -1), slice(-10, 3), slice(-10, 4, 2), slice(None, None, -1), slice(None, 3, -2), + slice(None, None, -3), slice(-1, -10, -2), slice(-3, -7, -1) + ) + + for size in sizes: + cache = MessageCache(maxlen=size) + messages = [MockMessage() for _ in range(size * 3 // 2)] + + for msg in messages: + cache.append(msg) + messages = messages[size // 2:] + + for slice_ in slices: + with self.subTest(current_loop=(size, slice_)): + self.assertListEqual(cache[slice_], messages[slice_]) + + def test_length(self): + """Test if len returns the correct number of items in the cache.""" + cache = MessageCache(maxlen=5) + + for current_loop in range(10): + with self.subTest(current_loop=current_loop): + self.assertEqual(len(cache), min(current_loop, 5)) + cache.append(MockMessage()) diff --git a/tests/bot/utils/test_time.py b/tests/bot/utils/test_time.py index 115ddfb0d..120d65176 100644 --- a/tests/bot/utils/test_time.py +++ b/tests/bot/utils/test_time.py @@ -13,13 +13,15 @@ class TimeTests(unittest.TestCase): """humanize_delta should be able to handle unknown units, and will not abort.""" # Does not abort for unknown units, as the unit name is checked # against the attribute of the relativedelta instance. - self.assertEqual(time.humanize_delta(relativedelta(days=2, hours=2), 'elephants', 2), '2 days and 2 hours') + actual = time.humanize_delta(relativedelta(days=2, hours=2), precision='elephants', max_units=2) + self.assertEqual(actual, '2 days and 2 hours') def test_humanize_delta_handle_high_units(self): """humanize_delta should be able to handle very high units.""" # Very high maximum units, but it only ever iterates over # each value the relativedelta might have. - self.assertEqual(time.humanize_delta(relativedelta(days=2, hours=2), 'hours', 20), '2 days and 2 hours') + actual = time.humanize_delta(relativedelta(days=2, hours=2), precision='hours', max_units=20) + self.assertEqual(actual, '2 days and 2 hours') def test_humanize_delta_should_normal_usage(self): """Testing humanize delta.""" @@ -32,7 +34,8 @@ class TimeTests(unittest.TestCase): for delta, precision, max_units, expected in test_cases: with self.subTest(delta=delta, precision=precision, max_units=max_units, expected=expected): - self.assertEqual(time.humanize_delta(delta, precision, max_units), expected) + actual = time.humanize_delta(delta, precision=precision, max_units=max_units) + self.assertEqual(actual, expected) def test_humanize_delta_raises_for_invalid_max_units(self): """humanize_delta should raises ValueError('max_units must be positive') for invalid max_units.""" @@ -40,22 +43,11 @@ class TimeTests(unittest.TestCase): for max_units in test_cases: with self.subTest(max_units=max_units), self.assertRaises(ValueError) as error: - time.humanize_delta(relativedelta(days=2, hours=2), 'hours', max_units) - self.assertEqual(str(error.exception), 'max_units must be positive') - - def test_parse_rfc1123(self): - """Testing parse_rfc1123.""" - self.assertEqual( - time.parse_rfc1123('Sun, 15 Sep 2019 12:00:00 GMT'), - datetime(2019, 9, 15, 12, 0, 0, tzinfo=timezone.utc) - ) - - def test_format_infraction(self): - """Testing format_infraction.""" - self.assertEqual(time.format_infraction('2019-12-12T00:01:00Z'), '2019-12-12 00:01') + time.humanize_delta(relativedelta(days=2, hours=2), precision='hours', max_units=max_units) + self.assertEqual(str(error.exception), 'max_units must be positive.') - def test_format_infraction_with_duration_none_expiry(self): - """format_infraction_with_duration should work for None expiry.""" + def test_format_with_duration_none_expiry(self): + """format_with_duration should work for None expiry.""" test_cases = ( (None, None, None, None), @@ -67,82 +59,71 @@ class TimeTests(unittest.TestCase): for expiry, date_from, max_units, expected in test_cases: with self.subTest(expiry=expiry, date_from=date_from, max_units=max_units, expected=expected): - self.assertEqual(time.format_infraction_with_duration(expiry, date_from, max_units), expected) + self.assertEqual(time.format_with_duration(expiry, date_from, max_units), expected) - def test_format_infraction_with_duration_custom_units(self): - """format_infraction_with_duration should work for custom max_units.""" + def test_format_with_duration_custom_units(self): + """format_with_duration should work for custom max_units.""" test_cases = ( - ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 5, 5), 6, - '2019-12-12 00:01 (11 hours, 55 minutes and 55 seconds)'), - ('2019-11-23T20:09:00Z', datetime(2019, 4, 25, 20, 15), 20, - '2019-11-23 20:09 (6 months, 28 days, 23 hours and 54 minutes)') + ('3000-12-12T00:01:00Z', datetime(3000, 12, 11, 12, 5, 5, tzinfo=timezone.utc), 6, + '<t:32533488060:f> (11 hours, 55 minutes and 55 seconds)'), + ('3000-11-23T20:09:00Z', datetime(3000, 4, 25, 20, 15, tzinfo=timezone.utc), 20, + '<t:32531918940:f> (6 months, 28 days, 23 hours and 54 minutes)') ) for expiry, date_from, max_units, expected in test_cases: with self.subTest(expiry=expiry, date_from=date_from, max_units=max_units, expected=expected): - self.assertEqual(time.format_infraction_with_duration(expiry, date_from, max_units), expected) + self.assertEqual(time.format_with_duration(expiry, date_from, max_units), expected) - def test_format_infraction_with_duration_normal_usage(self): - """format_infraction_with_duration should work for normal usage, across various durations.""" + def test_format_with_duration_normal_usage(self): + """format_with_duration should work for normal usage, across various durations.""" + utc = timezone.utc test_cases = ( - ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 0, 5), 2, '2019-12-12 00:01 (12 hours and 55 seconds)'), - ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 0, 5), 1, '2019-12-12 00:01 (12 hours)'), - ('2019-12-12T00:00:00Z', datetime(2019, 12, 11, 23, 59), 2, '2019-12-12 00:00 (1 minute)'), - ('2019-11-23T20:09:00Z', datetime(2019, 11, 15, 20, 15), 2, '2019-11-23 20:09 (7 days and 23 hours)'), - ('2019-11-23T20:09:00Z', datetime(2019, 4, 25, 20, 15), 2, '2019-11-23 20:09 (6 months and 28 days)'), - ('2019-11-23T20:58:00Z', datetime(2019, 11, 23, 20, 53), 2, '2019-11-23 20:58 (5 minutes)'), - ('2019-11-24T00:00:00Z', datetime(2019, 11, 23, 23, 59, 0), 2, '2019-11-24 00:00 (1 minute)'), - ('2019-11-23T23:59:00Z', datetime(2017, 7, 21, 23, 0), 2, '2019-11-23 23:59 (2 years and 4 months)'), - ('2019-11-23T23:59:00Z', datetime(2019, 11, 23, 23, 49, 5), 2, - '2019-11-23 23:59 (9 minutes and 55 seconds)'), + ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 0, 5, tzinfo=utc), 2, + '<t:1576108860:f> (12 hours and 55 seconds)'), + ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 0, 5, tzinfo=utc), 1, '<t:1576108860:f> (12 hours)'), + ('2019-12-12T00:00:00Z', datetime(2019, 12, 11, 23, 59, tzinfo=utc), 2, '<t:1576108800:f> (1 minute)'), + ('2019-11-23T20:09:00Z', datetime(2019, 11, 15, 20, 15, tzinfo=utc), 2, + '<t:1574539740:f> (7 days and 23 hours)'), + ('2019-11-23T20:09:00Z', datetime(2019, 4, 25, 20, 15, tzinfo=utc), 2, + '<t:1574539740:f> (6 months and 28 days)'), + ('2019-11-23T20:58:00Z', datetime(2019, 11, 23, 20, 53, tzinfo=utc), 2, '<t:1574542680:f> (5 minutes)'), + ('2019-11-24T00:00:00Z', datetime(2019, 11, 23, 23, 59, 0, tzinfo=utc), 2, '<t:1574553600:f> (1 minute)'), + ('2019-11-23T23:59:00Z', datetime(2017, 7, 21, 23, 0, tzinfo=utc), 2, + '<t:1574553540:f> (2 years and 4 months)'), + ('2019-11-23T23:59:00Z', datetime(2019, 11, 23, 23, 49, 5, tzinfo=utc), 2, + '<t:1574553540:f> (9 minutes and 55 seconds)'), (None, datetime(2019, 11, 23, 23, 49, 5), 2, None), ) for expiry, date_from, max_units, expected in test_cases: with self.subTest(expiry=expiry, date_from=date_from, max_units=max_units, expected=expected): - self.assertEqual(time.format_infraction_with_duration(expiry, date_from, max_units), expected) + self.assertEqual(time.format_with_duration(expiry, date_from, max_units), expected) def test_until_expiration_with_duration_none_expiry(self): - """until_expiration should work for None expiry.""" - test_cases = ( - (None, None, None, None), - - # To make sure that now and max_units are not touched - (None, 'Why hello there!', None, None), - (None, None, float('inf'), None), - (None, 'Why hello there!', float('inf'), None), - ) - - for expiry, now, max_units, expected in test_cases: - with self.subTest(expiry=expiry, now=now, max_units=max_units, expected=expected): - self.assertEqual(time.until_expiration(expiry, now, max_units), expected) + """until_expiration should return "Permanent" is expiry is None.""" + self.assertEqual(time.until_expiration(None), "Permanent") def test_until_expiration_with_duration_custom_units(self): """until_expiration should work for custom max_units.""" test_cases = ( - ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 5, 5), 6, '11 hours, 55 minutes and 55 seconds'), - ('2019-11-23T20:09:00Z', datetime(2019, 4, 25, 20, 15), 20, '6 months, 28 days, 23 hours and 54 minutes') + ('3000-12-12T00:01:00Z', '<t:32533488060:R>'), + ('3000-11-23T20:09:00Z', '<t:32531918940:R>') ) - for expiry, now, max_units, expected in test_cases: - with self.subTest(expiry=expiry, now=now, max_units=max_units, expected=expected): - self.assertEqual(time.until_expiration(expiry, now, max_units), expected) + for expiry, expected in test_cases: + with self.subTest(expiry=expiry, expected=expected): + self.assertEqual(time.until_expiration(expiry,), expected) def test_until_expiration_normal_usage(self): """until_expiration should work for normal usage, across various durations.""" test_cases = ( - ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 0, 5), 2, '12 hours and 55 seconds'), - ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 0, 5), 1, '12 hours'), - ('2019-12-12T00:00:00Z', datetime(2019, 12, 11, 23, 59), 2, '1 minute'), - ('2019-11-23T20:09:00Z', datetime(2019, 11, 15, 20, 15), 2, '7 days and 23 hours'), - ('2019-11-23T20:09:00Z', datetime(2019, 4, 25, 20, 15), 2, '6 months and 28 days'), - ('2019-11-23T20:58:00Z', datetime(2019, 11, 23, 20, 53), 2, '5 minutes'), - ('2019-11-24T00:00:00Z', datetime(2019, 11, 23, 23, 59, 0), 2, '1 minute'), - ('2019-11-23T23:59:00Z', datetime(2017, 7, 21, 23, 0), 2, '2 years and 4 months'), - ('2019-11-23T23:59:00Z', datetime(2019, 11, 23, 23, 49, 5), 2, '9 minutes and 55 seconds'), - (None, datetime(2019, 11, 23, 23, 49, 5), 2, None), + ('3000-12-12T00:01:00Z', '<t:32533488060:R>'), + ('3000-12-12T00:01:00Z', '<t:32533488060:R>'), + ('3000-12-12T00:00:00Z', '<t:32533488000:R>'), + ('3000-11-23T20:09:00Z', '<t:32531918940:R>'), + ('3000-11-23T20:09:00Z', '<t:32531918940:R>'), ) - for expiry, now, max_units, expected in test_cases: - with self.subTest(expiry=expiry, now=now, max_units=max_units, expected=expected): - self.assertEqual(time.until_expiration(expiry, now, max_units), expected) + for expiry, expected in test_cases: + with self.subTest(expiry=expiry, expected=expected): + self.assertEqual(time.until_expiration(expiry), expected) diff --git a/tests/helpers.py b/tests/helpers.py index 3978076ed..9d4988d23 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -39,7 +39,7 @@ class HashableMixin(discord.mixins.EqualityComparable): class ColourMixin: - """A mixin for Mocks that provides the aliasing of color->colour like discord.py does.""" + """A mixin for Mocks that provides the aliasing of (accent_)color->(accent_)colour like discord.py does.""" @property def color(self) -> discord.Colour: @@ -49,6 +49,14 @@ class ColourMixin: def color(self, color: discord.Colour) -> None: self.colour = color + @property + def accent_color(self) -> discord.Colour: + return self.accent_colour + + @accent_color.setter + def accent_color(self, color: discord.Colour) -> None: + self.accent_colour = color + class CustomMockMixin: """ @@ -235,13 +243,20 @@ class MockMember(CustomMockMixin, unittest.mock.Mock, ColourMixin, HashableMixin self.roles = [MockRole(name="@everyone", position=1, id=0)] if roles: self.roles.extend(roles) + self.top_role = max(self.roles) if 'mention' not in kwargs: self.mention = f"@{self.name}" # Create a User instance to get a realistic Mock of `discord.User` -user_instance = discord.User(data=unittest.mock.MagicMock(), state=unittest.mock.MagicMock()) +_user_data_mock = collections.defaultdict(unittest.mock.MagicMock, { + "accent_color": 0 +}) +user_instance = discord.User( + data=unittest.mock.MagicMock(get=unittest.mock.Mock(side_effect=_user_data_mock.get)), + state=unittest.mock.MagicMock() +) class MockUser(CustomMockMixin, unittest.mock.Mock, ColourMixin, HashableMixin): @@ -278,7 +293,10 @@ def _get_mock_loop() -> unittest.mock.Mock: # Since calling `create_task` on our MockBot does not actually schedule the coroutine object # as a task in the asyncio loop, this `side_effect` calls `close()` on the coroutine object # to prevent "has not been awaited"-warnings. - loop.create_task.side_effect = lambda coroutine: coroutine.close() + def mock_create_task(coroutine, **kwargs): + coroutine.close() + return unittest.mock.Mock() + loop.create_task.side_effect = mock_create_task return loop @@ -424,7 +442,12 @@ message_instance = discord.Message(state=state, channel=channel, data=message_da # Create a Context instance to get a realistic MagicMock of `discord.ext.commands.Context` -context_instance = Context(message=unittest.mock.MagicMock(), prefix=unittest.mock.MagicMock()) +context_instance = Context( + message=unittest.mock.MagicMock(), + prefix="$", + bot=MockBot(), + view=None +) context_instance.invoked_from_error_handler = None @@ -439,6 +462,7 @@ class MockContext(CustomMockMixin, unittest.mock.MagicMock): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) + self.me = kwargs.get('me', MockMember()) self.bot = kwargs.get('bot', MockBot()) self.guild = kwargs.get('guild', MockGuild()) self.author = kwargs.get('author', MockMember()) @@ -532,7 +556,7 @@ class MockReaction(CustomMockMixin, unittest.mock.MagicMock): self.__str__.return_value = str(self.emoji) -webhook_instance = discord.Webhook(data=unittest.mock.MagicMock(), adapter=unittest.mock.MagicMock()) +webhook_instance = discord.Webhook(data=unittest.mock.MagicMock(), session=unittest.mock.MagicMock()) class MockAsyncWebhook(CustomMockMixin, unittest.mock.MagicMock): diff --git a/tests/test_base.py b/tests/test_base.py index a7db4bf3e..365805a71 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -1,8 +1,7 @@ import logging -import unittest import unittest.mock - +from bot.log import get_logger from tests.base import LoggingTestsMixin, _CaptureLogHandler @@ -15,7 +14,7 @@ class LoggingTestCaseTests(unittest.TestCase): @classmethod def setUpClass(cls): - cls.log = logging.getLogger(__name__) + cls.log = get_logger(__name__) def test_assert_not_logs_does_not_raise_with_no_logs(self): """Test if LoggingTestCase.assertNotLogs does not raise when no logs were emitted.""" @@ -56,15 +55,15 @@ class LoggingTestCaseTests(unittest.TestCase): def test_logging_test_case_works_with_logger_instance(self): """Test if the LoggingTestCase captures logging for provided logger.""" - log = logging.getLogger("new_logger") + log = get_logger("new_logger") with self.assertRaises(AssertionError): with LoggingTestCase.assertNotLogs(self, logger=log): log.info("Hello, this should raise an AssertionError") def test_logging_test_case_respects_alternative_logger(self): """Test if LoggingTestCase only checks the provided logger.""" - log_one = logging.getLogger("log one") - log_two = logging.getLogger("log two") + log_one = get_logger("log one") + log_two = get_logger("log two") with LoggingTestCase.assertNotLogs(self, logger=log_one): log_two.info("Hello, this should not raise an AssertionError") @@ -5,7 +5,7 @@ import-order-style=pycharm application_import_names=bot,tests exclude=.cache,.venv,.git,constants.py ignore= - B311,W503,E226,S311,T000 + B311,W503,E226,S311,T000,E731 # Missing Docstrings D100,D104,D105,D107, # Docstring Whitespace |