diff options
32 files changed, 1314 insertions, 323 deletions
@@ -14,7 +14,7 @@ beautifulsoup4 = "~=4.9" colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} coloredlogs = "~=14.0" deepdiff = "~=4.0" -"discord.py" = {git = "https://github.com/Rapptz/discord.py.git", ref = "94f76e63947b102e5de6dae9a2cd687b308033"} +"discord.py" = "~=1.6.0" feedparser = "~=5.2" fuzzywuzzy = "~=0.17" lxml = "~=4.4" @@ -24,6 +24,7 @@ python-dateutil = "~=2.8" pyyaml = "~=5.1" sentry-sdk = "~=0.19" statsd = "~=3.3" +arrow = "~=0.17" emoji = "~=0.6" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 7fe041806..5aff33383 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f5286278b549c5f297710bedf1b85a54b89e6fbd29e5be9ad9d5583ab2b88818" + "sha256": "33874d325a918682da3ae4d833748263695836d0cda4c1b0627ce5a5f29746e5" }, "pipfile-spec": 6, "requires": { @@ -99,6 +99,14 @@ "markers": "python_version >= '3.6'", "version": "==3.3.1" }, + "arrow": { + "hashes": [ + "sha256:e098abbd9af3665aea81bdd6c869e93af4feb078e98468dd351c383af187aac5", + "sha256:ff08d10cda1d36c68657d6ad20d74fbea493d980f8b2d45344e00d6ed2bf6ed4" + ], + "index": "pypi", + "version": "==0.17.0" + }, "async-rediscache": { "extras": [ "fakeredis" @@ -216,13 +224,13 @@ "index": "pypi", "version": "==4.3.2" }, - "discord-py": { - "git": "https://github.com/Rapptz/discord.py.git", - "ref": "94f76e63947b102e5de6dae9a2cd687b308033dd" - }, "discord.py": { - "git": "https://github.com/Rapptz/discord.py.git", - "ref": "94f76e63947b102e5de6dae9a2cd687b308033" + "hashes": [ + "sha256:3df148daf6fbcc7ab5b11042368a3cd5f7b730b62f09fb5d3cbceff59bcfbb12", + "sha256:ba8be99ff1b8c616f7b6dcb700460d0222b29d4c11048e74366954c465fdd05f" + ], + "index": "pypi", + "version": "==1.6.0" }, "emoji": { "hashes": [ @@ -368,11 +376,11 @@ }, "markdownify": { "hashes": [ - "sha256:31d7c13ac2ada8bfc7535a25fee6622ca720e1b5f2d4a9cbc429d167c21f886d", - "sha256:7489fd5c601536996a376c4afbcd1dd034db7690af807120681461e82fbc0acc" + "sha256:2147197d9c45cdd24d57302b94e01cac44988862960ac42eba730345a31aebbc", + "sha256:3de08764db001e7119cb06481de4ec0b2ea0338fd26cf49bdf16c4475ef44b81" ], "index": "pypi", - "version": "==0.6.1" + "version": "==0.6.3" }, "more-itertools": { "hashes": [ @@ -500,22 +508,30 @@ }, "pyyaml": { "hashes": [ - "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", - "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", - "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", - "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e", - "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", - "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", - "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", - "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", - "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", - "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a", - "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", - "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", - "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" - ], - "index": "pypi", - "version": "==5.3.1" + "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", + "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696", + "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393", + "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77", + "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922", + "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5", + "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8", + "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10", + "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc", + "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018", + "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e", + "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253", + "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183", + "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb", + "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185", + "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db", + "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46", + "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b", + "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63", + "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df", + "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc" + ], + "index": "pypi", + "version": "==5.4.1" }, "redis": { "hashes": [ @@ -817,11 +833,11 @@ }, "identify": { "hashes": [ - "sha256:18994e850ba50c37bcaed4832be8b354d6a06c8fb31f54e0e7ece76d32f69bc8", - "sha256:892473bf12e655884132a3a32aca737a3cbefaa34a850ff52d501773a45837bc" + "sha256:70b638cf4743f33042bebb3b51e25261a0a10e80f978739f17e7fd4837664a66", + "sha256:9dfb63a2e871b807e3ba62f029813552a24b5289504f5b071dea9b041aee9fe4" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.5.12" + "version": "==1.5.13" }, "idna": { "hashes": [ @@ -887,22 +903,30 @@ }, "pyyaml": { "hashes": [ - "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", - "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", - "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", - "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e", - "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", - "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", - "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", - "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", - "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", - "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a", - "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", - "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", - "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" - ], - "index": "pypi", - "version": "==5.3.1" + "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", + "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696", + "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393", + "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77", + "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922", + "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5", + "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8", + "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10", + "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc", + "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018", + "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e", + "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253", + "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183", + "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb", + "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185", + "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db", + "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46", + "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b", + "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63", + "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df", + "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc" + ], + "index": "pypi", + "version": "==5.4.1" }, "requests": { "hashes": [ @@ -922,10 +946,10 @@ }, "snowballstemmer": { "hashes": [ - "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0", - "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52" + "sha256:b51b447bea85f9968c13b650126a888aabd4cb4463fca868ec596826325dedc2", + "sha256:e997baa4f2e9139951b6f4c631bad912dfd3c792467e2f03d7239464af90e914" ], - "version": "==2.0.0" + "version": "==2.1.0" }, "toml": { "hashes": [ @@ -945,11 +969,11 @@ }, "virtualenv": { "hashes": [ - "sha256:205a7577275dd0d9223c730dd498e21a8910600085c3dee97412b041fc4b853b", - "sha256:7992b8de87e544a4ab55afc2240bf8388c4e3b5765d03784dad384bfdf9097ee" + "sha256:219ee956e38b08e32d5639289aaa5bd190cfbe7dafcb8fa65407fca08e808f9c", + "sha256:227a8fed626f2f20a6cdb0870054989f82dd27b2560a911935ba905a2a5e0034" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.3.0" + "version": "==20.4.0" } } } @@ -5,8 +5,7 @@ [![Build][3]][4] [![Deploy][5]][6] [](https://coveralls.io/github/python-discord/bot) -[](LICENSE) -[](https://pythondiscord.com) +[](LICENSE) This project is a Discord bot specifically for use with the Python Discord server. It provides numerous utilities and other tools to help keep the server running like a well-oiled machine. @@ -19,5 +18,5 @@ Read the [Contributing Guide](https://pythondiscord.com/pages/contributing/bot/) [4]: https://github.com/python-discord/bot/actions?query=workflow%3ABuild+branch%3Amaster [5]: https://github.com/python-discord/bot/workflows/Deploy/badge.svg?branch=master [6]: https://github.com/python-discord/bot/actions?query=workflow%3ADeploy+branch%3Amaster -[7]: https://img.shields.io/static/v1?label=Python%20Discord&logo=discord&message=%3E100k%20members&color=%237289DA&logoColor=white -[8]: https://discord.gg/2B963hn +[7]: https://raw.githubusercontent.com/python-discord/branding/master/logos/badge/badge_github.svg +[8]: https://discord.gg/python diff --git a/bot/constants.py b/bot/constants.py index d813046ab..2f5cf0e8a 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -13,7 +13,7 @@ their default values from `config-default.yml`. import logging import os from collections.abc import Mapping -from enum import Enum +from enum import Enum, IntEnum from pathlib import Path from typing import Dict, List, Optional @@ -249,6 +249,9 @@ class Colours(metaclass=YAMLGetter): soft_green: int soft_orange: int bright_green: int + orange: int + pink: int + purple: int class DuckPond(metaclass=YAMLGetter): @@ -299,6 +302,8 @@ class Emojis(metaclass=YAMLGetter): comments: str user: str + ok_hand: str + class Icons(metaclass=YAMLGetter): section = "style" @@ -403,6 +408,7 @@ class Channels(metaclass=YAMLGetter): code_help_voice_2: int cooldown: int defcon: int + discord_py: int dev_contrib: int dev_core: int dev_log: int @@ -424,7 +430,7 @@ class Channels(metaclass=YAMLGetter): off_topic_1: int off_topic_2: int organisation: int - python_discussion: int + python_general: int python_events: int python_news: int reddit: int @@ -601,6 +607,12 @@ class VoiceGate(metaclass=YAMLGetter): voice_ping_delete_delay: int +class Branding(metaclass=YAMLGetter): + section = "branding" + + cycle_frequency: int + + class Event(Enum): """ Event names. This does not include every event (for example, raw diff --git a/bot/exts/backend/branding/__init__.py b/bot/exts/backend/branding/__init__.py new file mode 100644 index 000000000..81ea3bf49 --- /dev/null +++ b/bot/exts/backend/branding/__init__.py @@ -0,0 +1,7 @@ +from bot.bot import Bot +from bot.exts.backend.branding._cog import BrandingManager + + +def setup(bot: Bot) -> None: + """Loads BrandingManager cog.""" + bot.add_cog(BrandingManager(bot)) diff --git a/bot/exts/backend/branding/_cog.py b/bot/exts/backend/branding/_cog.py new file mode 100644 index 000000000..20df83a89 --- /dev/null +++ b/bot/exts/backend/branding/_cog.py @@ -0,0 +1,566 @@ +import asyncio +import itertools +import logging +import random +import typing as t +from datetime import datetime, time, timedelta + +import arrow +import async_timeout +import discord +from async_rediscache import RedisCache +from discord.ext import commands + +from bot.bot import Bot +from bot.constants import Branding, Colours, Emojis, Guild, MODERATION_ROLES +from bot.exts.backend.branding import _constants, _decorators, _errors, _seasons + +log = logging.getLogger(__name__) + + +class GitHubFile(t.NamedTuple): + """ + Represents a remote file on GitHub. + + The `sha` hash is kept so that we can determine that a file has changed, + despite its filename remaining unchanged. + """ + + download_url: str + path: str + sha: str + + +def pretty_files(files: t.Iterable[GitHubFile]) -> str: + """Provide a human-friendly representation of `files`.""" + return "\n".join(file.path for file in files) + + +def time_until_midnight() -> timedelta: + """ + Determine amount of time until the next-up UTC midnight. + + The exact `midnight` moment is actually delayed to 5 seconds after, in order + to avoid potential problems due to imprecise sleep. + """ + now = datetime.utcnow() + tomorrow = now + timedelta(days=1) + midnight = datetime.combine(tomorrow, time(second=5)) + + return midnight - now + + +class BrandingManager(commands.Cog): + """ + Manages the guild's branding. + + The purpose of this cog is to help automate the synchronization of the branding + repository with the guild. It is capable of discovering assets in the repository + via GitHub's API, resolving download urls for them, and delegating + to the `bot` instance to upload them to the guild. + + BrandingManager is designed to be entirely autonomous. Its `daemon` background task awakens + once a day (see `time_until_midnight`) to detect new seasons, or to cycle icons within a single + season. The daemon can be turned on and off via the `daemon` cmd group. The value set via + its `start` and `stop` commands is persisted across sessions. If turned on, the daemon will + automatically start on the next bot start-up. Otherwise, it will wait to be started manually. + + All supported operations, e.g. setting seasons, applying the branding, or cycling icons, can + also be invoked manually, via the following API: + + branding list + - Show all available seasons + + branding set <season_name> + - Set the cog's internal state to represent `season_name`, if it exists + - If no `season_name` is given, set chronologically current season + - This will not automatically apply the season's branding to the guild, + the cog's state can be detached from the guild + - Seasons can therefore be 'previewed' using this command + + branding info + - View detailed information about resolved assets for current season + + branding refresh + - Refresh internal state, i.e. synchronize with branding repository + + branding apply + - Apply the current internal state to the guild, i.e. upload the assets + + branding cycle + - If there are multiple available icons for current season, randomly pick + and apply the next one + + The daemon calls these methods autonomously as appropriate. The use of this cog + is locked to moderation roles. As it performs media asset uploads, it is prone to + rate-limits - the `apply` command should be used with caution. The `set` command can, + however, be used freely to 'preview' seasonal branding and check whether paths have been + resolved as appropriate. + + While the bot is in debug mode, it will 'mock' asset uploads by logging the passed + download urls and pretending that the upload was successful. Make use of this + to test this cog's behaviour. + """ + + current_season: t.Type[_seasons.SeasonBase] + + banner: t.Optional[GitHubFile] + + available_icons: t.List[GitHubFile] + remaining_icons: t.List[GitHubFile] + + days_since_cycle: t.Iterator + + daemon: t.Optional[asyncio.Task] + + # Branding configuration + branding_configuration = RedisCache() + + def __init__(self, bot: Bot) -> None: + """ + Assign safe default values on init. + + At this point, we don't have information about currently available branding. + Most of these attributes will be overwritten once the daemon connects, or once + the `refresh` command is used. + """ + self.bot = bot + self.current_season = _seasons.get_current_season() + + self.banner = None + + self.available_icons = [] + self.remaining_icons = [] + + self.days_since_cycle = itertools.cycle([None]) + + self.daemon = None + self._startup_task = self.bot.loop.create_task(self._initial_start_daemon()) + + async def _initial_start_daemon(self) -> None: + """Checks is daemon active and when is, start it at cog load.""" + if await self.branding_configuration.get("daemon_active"): + self.daemon = self.bot.loop.create_task(self._daemon_func()) + + @property + def _daemon_running(self) -> bool: + """True if the daemon is currently active, False otherwise.""" + return self.daemon is not None and not self.daemon.done() + + async def _daemon_func(self) -> None: + """ + Manage all automated behaviour of the BrandingManager cog. + + Once a day, the daemon will perform the following tasks: + - Update `current_season` + - Poll GitHub API to see if the available branding for `current_season` has changed + - Update assets if changes are detected (banner, guild icon, bot avatar, bot nickname) + - Check whether it's time to cycle guild icons + + The internal loop runs once when activated, then periodically at the time + given by `time_until_midnight`. + + All method calls in the internal loop are considered safe, i.e. no errors propagate + to the daemon's loop. The daemon itself does not perform any error handling on its own. + """ + await self.bot.wait_until_guild_available() + + while True: + self.current_season = _seasons.get_current_season() + branding_changed = await self.refresh() + + if branding_changed: + await self.apply() + + elif next(self.days_since_cycle) == Branding.cycle_frequency: + await self.cycle() + + until_midnight = time_until_midnight() + await asyncio.sleep(until_midnight.total_seconds()) + + async def _info_embed(self) -> discord.Embed: + """Make an informative embed representing current season.""" + info_embed = discord.Embed(description=self.current_season.description, colour=self.current_season.colour) + + # If we're in a non-evergreen season, also show active months + if self.current_season is not _seasons.SeasonBase: + title = f"{self.current_season.season_name} ({', '.join(str(m) for m in self.current_season.months)})" + else: + title = self.current_season.season_name + + # Use the author field to show the season's name and avatar if available + info_embed.set_author(name=title) + + banner = self.banner.path if self.banner is not None else "Unavailable" + info_embed.add_field(name="Banner", value=banner, inline=False) + + icons = pretty_files(self.available_icons) or "Unavailable" + info_embed.add_field(name="Available icons", value=icons, inline=False) + + # Only display cycle frequency if we're actually cycling + if len(self.available_icons) > 1 and Branding.cycle_frequency: + info_embed.set_footer(text=f"Icon cycle frequency: {Branding.cycle_frequency}") + + return info_embed + + async def _reset_remaining_icons(self) -> None: + """Set `remaining_icons` to a shuffled copy of `available_icons`.""" + self.remaining_icons = random.sample(self.available_icons, k=len(self.available_icons)) + + async def _reset_days_since_cycle(self) -> None: + """ + Reset the `days_since_cycle` iterator based on configured frequency. + + If the current season only has 1 icon, or if `Branding.cycle_frequency` is falsey, + the iterator will always yield None. This signals that the icon shouldn't be cycled. + + Otherwise, it will yield ints in range [1, `Branding.cycle_frequency`] indefinitely. + When the iterator yields a value equal to `Branding.cycle_frequency`, it is time to cycle. + """ + if len(self.available_icons) > 1 and Branding.cycle_frequency: + sequence = range(1, Branding.cycle_frequency + 1) + else: + sequence = [None] + + self.days_since_cycle = itertools.cycle(sequence) + + async def _get_files(self, path: str, include_dirs: bool = False) -> t.Dict[str, GitHubFile]: + """ + Get files at `path` in the branding repository. + + If `include_dirs` is False (default), only returns files at `path`. + Otherwise, will return both files and directories. Never returns symlinks. + + Return dict mapping from filename to corresponding `GitHubFile` instance. + This may return an empty dict if the response status is non-200, + or if the target directory is empty. + """ + url = f"{_constants.BRANDING_URL}/{path}" + async with self.bot.http_session.get( + url, headers=_constants.HEADERS, params=_constants.PARAMS + ) as resp: + # Short-circuit if we get non-200 response + if resp.status != _constants.STATUS_OK: + log.error(f"GitHub API returned non-200 response: {resp}") + return {} + directory = await resp.json() # Directory at `path` + + allowed_types = {"file", "dir"} if include_dirs else {"file"} + return { + file["name"]: GitHubFile(file["download_url"], file["path"], file["sha"]) + for file in directory + if file["type"] in allowed_types + } + + async def refresh(self) -> bool: + """ + Synchronize available assets with branding repository. + + If the current season is not the evergreen, and lacks at least one asset, + we use the evergreen seasonal dir as fallback for missing assets. + + Finally, if neither the seasonal nor fallback branding directories contain + an asset, it will simply be ignored. + + Return True if the branding has changed. This will be the case when we enter + a new season, or when something changes in the current seasons's directory + in the branding repository. + """ + old_branding = (self.banner, self.available_icons) + seasonal_dir = await self._get_files(self.current_season.branding_path, include_dirs=True) + + # Only make a call to the fallback directory if there is something to be gained + branding_incomplete = any( + asset not in seasonal_dir + for asset in (_constants.FILE_BANNER, _constants.FILE_AVATAR, _constants.SERVER_ICONS) + ) + if branding_incomplete and self.current_season is not _seasons.SeasonBase: + fallback_dir = await self._get_files( + _seasons.SeasonBase.branding_path, include_dirs=True + ) + else: + fallback_dir = {} + + # Resolve assets in this directory, None is a safe value + self.banner = ( + seasonal_dir.get(_constants.FILE_BANNER) + or fallback_dir.get(_constants.FILE_BANNER) + ) + + # Now resolve server icons by making a call to the proper sub-directory + if _constants.SERVER_ICONS in seasonal_dir: + icons_dir = await self._get_files( + f"{self.current_season.branding_path}/{_constants.SERVER_ICONS}" + ) + self.available_icons = list(icons_dir.values()) + + elif _constants.SERVER_ICONS in fallback_dir: + icons_dir = await self._get_files( + f"{_seasons.SeasonBase.branding_path}/{_constants.SERVER_ICONS}" + ) + self.available_icons = list(icons_dir.values()) + + else: + self.available_icons = [] # This should never be the case, but an empty list is a safe value + + # GitHubFile instances carry a `sha` attr so this will pick up if a file changes + branding_changed = old_branding != (self.banner, self.available_icons) + + if branding_changed: + log.info(f"New branding detected (season: {self.current_season.season_name})") + await self._reset_remaining_icons() + await self._reset_days_since_cycle() + + return branding_changed + + async def cycle(self) -> bool: + """ + Apply the next-up server icon. + + Returns True if an icon is available and successfully gets applied, False otherwise. + """ + if not self.available_icons: + log.info("Cannot cycle: no icons for this season") + return False + + if not self.remaining_icons: + log.info("Reset & shuffle remaining icons") + await self._reset_remaining_icons() + + next_up = self.remaining_icons.pop(0) + success = await self.set_icon(next_up.download_url) + + return success + + async def apply(self) -> t.List[str]: + """ + Apply current branding to the guild and bot. + + This delegates to the bot instance to do all the work. We only provide download urls + for available assets. Assets unavailable in the branding repo will be ignored. + + Returns a list of names of all failed assets. An asset is considered failed + if it isn't found in the branding repo, or if something goes wrong while the + bot is trying to apply it. + + An empty list denotes that all assets have been applied successfully. + """ + report = {asset: False for asset in ("banner", "icon")} + + if self.banner is not None: + report["banner"] = await self.set_banner(self.banner.download_url) + + report["icon"] = await self.cycle() + + failed_assets = [asset for asset, succeeded in report.items() if not succeeded] + return failed_assets + + @commands.has_any_role(*MODERATION_ROLES) + @commands.group(name="branding") + async def branding_cmds(self, ctx: commands.Context) -> None: + """Manual branding control.""" + if not ctx.invoked_subcommand: + await ctx.send_help(ctx.command) + + @branding_cmds.command(name="list", aliases=["ls"]) + async def branding_list(self, ctx: commands.Context) -> None: + """List all available seasons and branding sources.""" + embed = discord.Embed(title="Available seasons", colour=Colours.soft_green) + + for season in _seasons.get_all_seasons(): + if season is _seasons.SeasonBase: + active_when = "always" + else: + active_when = f"in {', '.join(str(m) for m in season.months)}" + + description = ( + f"Active {active_when}\n" + f"Branding: {season.branding_path}" + ) + embed.add_field(name=season.season_name, value=description, inline=False) + + await ctx.send(embed=embed) + + @branding_cmds.command(name="set") + async def branding_set(self, ctx: commands.Context, *, season_name: t.Optional[str] = None) -> None: + """ + Manually set season, or reset to current if none given. + + Season search is a case-less comparison against both seasonal class name, + and its `season_name` attr. + + This only pre-loads the cog's internal state to the chosen season, but does not + automatically apply the branding. As that is an expensive operation, the `apply` + command must be called explicitly after this command finishes. + + This means that this command can be used to 'preview' a season gathering info + about its available assets, without applying them to the guild. + + If the daemon is running, it will automatically reset the season to current when + it wakes up. The season set via this command can therefore remain 'detached' from + what it should be - the daemon will make sure that it's set back properly. + """ + if season_name is None: + new_season = _seasons.get_current_season() + else: + new_season = _seasons.get_season(season_name) + if new_season is None: + raise _errors.BrandingError("No such season exists") + + if self.current_season is new_season: + raise _errors.BrandingError(f"Season {self.current_season.season_name} already active") + + self.current_season = new_season + await self.branding_refresh(ctx) + + @branding_cmds.command(name="info", aliases=["status"]) + async def branding_info(self, ctx: commands.Context) -> None: + """ + Show available assets for current season. + + This can be used to confirm that assets have been resolved properly. + When `apply` is used, it attempts to upload exactly the assets listed here. + """ + await ctx.send(embed=await self._info_embed()) + + @branding_cmds.command(name="refresh") + async def branding_refresh(self, ctx: commands.Context) -> None: + """Sync currently available assets with branding repository.""" + async with ctx.typing(): + await self.refresh() + await self.branding_info(ctx) + + @branding_cmds.command(name="apply") + async def branding_apply(self, ctx: commands.Context) -> None: + """ + Apply current season's branding to the guild. + + Use `info` to check which assets will be applied. Shows which assets have + failed to be applied, if any. + """ + async with ctx.typing(): + failed_assets = await self.apply() + if failed_assets: + raise _errors.BrandingError( + f"Failed to apply following assets: {', '.join(failed_assets)}" + ) + + response = discord.Embed(description=f"All assets applied {Emojis.ok_hand}", colour=Colours.soft_green) + await ctx.send(embed=response) + + @branding_cmds.command(name="cycle") + async def branding_cycle(self, ctx: commands.Context) -> None: + """ + Apply the next-up guild icon, if multiple are available. + + The order is random. + """ + async with ctx.typing(): + success = await self.cycle() + if not success: + raise _errors.BrandingError("Failed to cycle icon") + + response = discord.Embed(description=f"Success {Emojis.ok_hand}", colour=Colours.soft_green) + await ctx.send(embed=response) + + @branding_cmds.group(name="daemon", aliases=["d", "task"]) + async def daemon_group(self, ctx: commands.Context) -> None: + """Control the background daemon.""" + if not ctx.invoked_subcommand: + await ctx.send_help(ctx.command) + + @daemon_group.command(name="status") + async def daemon_status(self, ctx: commands.Context) -> None: + """Check whether daemon is currently active.""" + if self._daemon_running: + remaining_time = (arrow.utcnow() + time_until_midnight()).humanize() + response = discord.Embed(description=f"Daemon running {Emojis.ok_hand}", colour=Colours.soft_green) + response.set_footer(text=f"Next refresh {remaining_time}") + else: + response = discord.Embed(description="Daemon not running", colour=Colours.soft_red) + + await ctx.send(embed=response) + + @daemon_group.command(name="start") + async def daemon_start(self, ctx: commands.Context) -> None: + """If the daemon isn't running, start it.""" + if self._daemon_running: + raise _errors.BrandingError("Daemon already running!") + + self.daemon = self.bot.loop.create_task(self._daemon_func()) + await self.branding_configuration.set("daemon_active", True) + + response = discord.Embed(description=f"Daemon started {Emojis.ok_hand}", colour=Colours.soft_green) + await ctx.send(embed=response) + + @daemon_group.command(name="stop") + async def daemon_stop(self, ctx: commands.Context) -> None: + """If the daemon is running, stop it.""" + if not self._daemon_running: + raise _errors.BrandingError("Daemon not running!") + + self.daemon.cancel() + await self.branding_configuration.set("daemon_active", False) + + response = discord.Embed(description=f"Daemon stopped {Emojis.ok_hand}", colour=Colours.soft_green) + await ctx.send(embed=response) + + async def _fetch_image(self, url: str) -> bytes: + """Retrieve and read image from `url`.""" + log.debug(f"Getting image from: {url}") + async with self.bot.http_session.get(url) as resp: + return await resp.read() + + async def _apply_asset(self, target: discord.Guild, asset: _constants.AssetType, url: str) -> bool: + """ + Internal method for applying media assets to the guild. + + This shouldn't be called directly. The purpose of this method is mainly generic + error handling to reduce needless code repetition. + + Return True if upload was successful, False otherwise. + """ + log.info(f"Attempting to set {asset.name}: {url}") + + kwargs = {asset.value: await self._fetch_image(url)} + try: + async with async_timeout.timeout(5): + await target.edit(**kwargs) + + except asyncio.TimeoutError: + log.info("Asset upload timed out") + return False + + except discord.HTTPException as discord_error: + log.exception("Asset upload failed", exc_info=discord_error) + return False + + else: + log.info("Asset successfully applied") + return True + + @_decorators.mock_in_debug(return_value=True) + async def set_banner(self, url: str) -> bool: + """Set the guild's banner to image at `url`.""" + guild = self.bot.get_guild(Guild.id) + if guild is None: + log.info("Failed to get guild instance, aborting asset upload") + return False + + return await self._apply_asset(guild, _constants.AssetType.BANNER, url) + + @_decorators.mock_in_debug(return_value=True) + async def set_icon(self, url: str) -> bool: + """Sets the guild's icon to image at `url`.""" + guild = self.bot.get_guild(Guild.id) + if guild is None: + log.info("Failed to get guild instance, aborting asset upload") + return False + + return await self._apply_asset(guild, _constants.AssetType.SERVER_ICON, url) + + def cog_unload(self) -> None: + """Cancels startup and daemon task.""" + self._startup_task.cancel() + if self.daemon is not None: + self.daemon.cancel() diff --git a/bot/exts/backend/branding/_constants.py b/bot/exts/backend/branding/_constants.py new file mode 100644 index 000000000..dbc7615f2 --- /dev/null +++ b/bot/exts/backend/branding/_constants.py @@ -0,0 +1,51 @@ +from enum import Enum, IntEnum + +from bot.constants import Keys + + +class Month(IntEnum): + """All month constants for seasons.""" + + JANUARY = 1 + FEBRUARY = 2 + MARCH = 3 + APRIL = 4 + MAY = 5 + JUNE = 6 + JULY = 7 + AUGUST = 8 + SEPTEMBER = 9 + OCTOBER = 10 + NOVEMBER = 11 + DECEMBER = 12 + + def __str__(self) -> str: + return self.name.title() + + +class AssetType(Enum): + """ + Discord media assets. + + The values match exactly the kwarg keys that can be passed to `Guild.edit`. + """ + + BANNER = "banner" + SERVER_ICON = "icon" + + +STATUS_OK = 200 # HTTP status code + +FILE_BANNER = "banner.png" +FILE_AVATAR = "avatar.png" +SERVER_ICONS = "server_icons" + +BRANDING_URL = "https://api.github.com/repos/python-discord/branding/contents" + +PARAMS = {"ref": "master"} # Target branch +HEADERS = {"Accept": "application/vnd.github.v3+json"} # Ensure we use API v3 + +# A GitHub token is not necessary for the cog to operate, +# unauthorized requests are however limited to 60 per hour +if Keys.github: + HEADERS["Authorization"] = f"token {Keys.github}" diff --git a/bot/exts/backend/branding/_decorators.py b/bot/exts/backend/branding/_decorators.py new file mode 100644 index 000000000..6a1e7e869 --- /dev/null +++ b/bot/exts/backend/branding/_decorators.py @@ -0,0 +1,27 @@ +import functools +import logging +import typing as t + +from bot.constants import DEBUG_MODE + +log = logging.getLogger(__name__) + + +def mock_in_debug(return_value: t.Any) -> t.Callable: + """ + Short-circuit function execution if in debug mode and return `return_value`. + + The original function name, and the incoming args and kwargs are DEBUG level logged + upon each call. This is useful for expensive operations, i.e. media asset uploads + that are prone to rate-limits but need to be tested extensively. + """ + def decorator(func: t.Callable) -> t.Callable: + @functools.wraps(func) + async def wrapped(*args, **kwargs) -> t.Any: + """Short-circuit and log if in debug mode.""" + if DEBUG_MODE: + log.debug(f"Function {func.__name__} called with args: {args}, kwargs: {kwargs}") + return return_value + return await func(*args, **kwargs) + return wrapped + return decorator diff --git a/bot/exts/backend/branding/_errors.py b/bot/exts/backend/branding/_errors.py new file mode 100644 index 000000000..7cd271af3 --- /dev/null +++ b/bot/exts/backend/branding/_errors.py @@ -0,0 +1,2 @@ +class BrandingError(Exception): + """Exception raised by the BrandingManager cog.""" diff --git a/bot/exts/backend/branding/_seasons.py b/bot/exts/backend/branding/_seasons.py new file mode 100644 index 000000000..5f6256b30 --- /dev/null +++ b/bot/exts/backend/branding/_seasons.py @@ -0,0 +1,175 @@ +import logging +import typing as t +from datetime import datetime + +from bot.constants import Colours +from bot.exts.backend.branding._constants import Month +from bot.exts.backend.branding._errors import BrandingError + +log = logging.getLogger(__name__) + + +class SeasonBase: + """ + Base for Seasonal classes. + + This serves as the off-season fallback for when no specific + seasons are active. + + Seasons are 'registered' simply by inheriting from `SeasonBase`. + We discover them by calling `__subclasses__`. + """ + + season_name: str = "Evergreen" + + colour: str = Colours.soft_green + description: str = "The default season!" + + branding_path: str = "seasonal/evergreen" + + months: t.Set[Month] = set(Month) + + +class Christmas(SeasonBase): + """Branding for December.""" + + season_name = "Festive season" + + colour = Colours.soft_red + description = ( + "The time is here to get into the festive spirit! No matter who you are, where you are, " + "or what beliefs you may follow, we hope every one of you enjoy this festive season!" + ) + + branding_path = "seasonal/christmas" + + months = {Month.DECEMBER} + + +class Easter(SeasonBase): + """Branding for April.""" + + season_name = "Easter" + + colour = Colours.bright_green + description = ( + "Bunny here, bunny there, bunny everywhere! Here at Python Discord, we celebrate " + "our version of Easter during the entire month of April." + ) + + branding_path = "seasonal/easter" + + months = {Month.APRIL} + + +class Halloween(SeasonBase): + """Branding for October.""" + + season_name = "Halloween" + + colour = Colours.orange + description = "Trick or treat?!" + + branding_path = "seasonal/halloween" + + months = {Month.OCTOBER} + + +class Pride(SeasonBase): + """Branding for June.""" + + season_name = "Pride" + + colour = Colours.pink + description = ( + "The month of June is a special month for us at Python Discord. It is very important to us " + "that everyone feels welcome here, no matter their origin, identity or sexuality. During the " + "month of June, while some of you are participating in Pride festivals across the world, " + "we will be celebrating individuality and commemorating the history and challenges " + "of the LGBTQ+ community with a Pride event of our own!" + ) + + branding_path = "seasonal/pride" + + months = {Month.JUNE} + + +class Valentines(SeasonBase): + """Branding for February.""" + + season_name = "Valentines" + + colour = Colours.pink + description = "Love is in the air!" + + branding_path = "seasonal/valentines" + + months = {Month.FEBRUARY} + + +class Wildcard(SeasonBase): + """Branding for August.""" + + season_name = "Wildcard" + + colour = Colours.purple + description = "A season full of surprises!" + + months = {Month.AUGUST} + + +def get_all_seasons() -> t.List[t.Type[SeasonBase]]: + """Give all available season classes.""" + return [SeasonBase] + SeasonBase.__subclasses__() + + +def get_current_season() -> t.Type[SeasonBase]: + """Give active season, based on current UTC month.""" + current_month = Month(datetime.utcnow().month) + + active_seasons = tuple( + season + for season in SeasonBase.__subclasses__() + if current_month in season.months + ) + + if not active_seasons: + return SeasonBase + + return active_seasons[0] + + +def get_season(name: str) -> t.Optional[t.Type[SeasonBase]]: + """ + Give season such that its class name or its `season_name` attr match `name` (caseless). + + If no such season exists, return None. + """ + name = name.casefold() + + for season in get_all_seasons(): + matches = (season.__name__.casefold(), season.season_name.casefold()) + + if name in matches: + return season + + +def _validate_season_overlap() -> None: + """ + Raise BrandingError if there are any colliding seasons. + + This serves as a local test to ensure that seasons haven't been misconfigured. + """ + month_to_season = {} + + for season in SeasonBase.__subclasses__(): + for month in season.months: + colliding_season = month_to_season.get(month) + + if colliding_season: + raise BrandingError(f"Season {season} collides with {colliding_season} in {month.name}") + else: + month_to_season[month] = season + + +_validate_season_overlap() diff --git a/bot/exts/backend/error_handler.py b/bot/exts/backend/error_handler.py index 5b5840858..b8bb3757f 100644 --- a/bot/exts/backend/error_handler.py +++ b/bot/exts/backend/error_handler.py @@ -1,5 +1,7 @@ import contextlib +import difflib import logging +import random import typing as t from discord import Embed @@ -8,9 +10,10 @@ from sentry_sdk import push_scope from bot.api import ResponseCodeError from bot.bot import Bot -from bot.constants import Colours +from bot.constants import Colours, ERROR_REPLIES, Icons, MODERATION_ROLES from bot.converters import TagNameConverter from bot.errors import LockedResourceError +from bot.exts.backend.branding._errors import BrandingError from bot.utils.checks import InWhitelistCheckFailure log = logging.getLogger(__name__) @@ -76,6 +79,9 @@ class ErrorHandler(Cog): await self.handle_api_error(ctx, e.original) elif isinstance(e.original, LockedResourceError): await ctx.send(f"{e.original} Please wait for it to finish and try again later.") + elif isinstance(e.original, BrandingError): + await ctx.send(embed=self._get_error_embed(random.choice(ERROR_REPLIES), str(e.original))) + return else: await self.handle_unexpected_error(ctx, e.original) return # Exit early to avoid logging. @@ -154,10 +160,46 @@ class ErrorHandler(Cog): ) else: with contextlib.suppress(ResponseCodeError): - await ctx.invoke(tags_get_command, tag_name=tag_name) + if await ctx.invoke(tags_get_command, tag_name=tag_name): + return + + if not any(role.id in MODERATION_ROLES for role in ctx.author.roles): + await self.send_command_suggestion(ctx, ctx.invoked_with) + # Return to not raise the exception return + async def send_command_suggestion(self, ctx: Context, command_name: str) -> None: + """Sends user similar commands if any can be found.""" + # No similar tag found, or tag on cooldown - + # searching for a similar command + raw_commands = [] + for cmd in self.bot.walk_commands(): + if not cmd.hidden: + raw_commands += (cmd.name, *cmd.aliases) + if similar_command_data := difflib.get_close_matches(command_name, raw_commands, 1): + similar_command_name = similar_command_data[0] + similar_command = self.bot.get_command(similar_command_name) + + if not similar_command: + return + + log_msg = "Cancelling attempt to suggest a command due to failed checks." + try: + if not await similar_command.can_run(ctx): + log.debug(log_msg) + return + except errors.CommandError as cmd_error: + log.debug(log_msg) + await self.on_command_error(ctx, cmd_error) + return + + misspelled_content = ctx.message.content + e = Embed() + e.set_author(name="Did you mean:", icon_url=Icons.questionmark) + e.description = f"{misspelled_content.replace(command_name, similar_command_name, 1)}" + await ctx.send(embed=e, delete_after=10.0) + async def handle_user_input_error(self, ctx: Context, e: errors.UserInputError) -> None: """ Send an error message in `ctx` for UserInputError, sometimes invoking the help command too. diff --git a/bot/exts/backend/sync/_syncers.py b/bot/exts/backend/sync/_syncers.py index 2eb9f9971..c9f2d2da8 100644 --- a/bot/exts/backend/sync/_syncers.py +++ b/bot/exts/backend/sync/_syncers.py @@ -5,12 +5,15 @@ from collections import namedtuple from discord import Guild from discord.ext.commands import Context +from more_itertools import chunked import bot from bot.api import ResponseCodeError log = logging.getLogger(__name__) +CHUNK_SIZE = 1000 + # These objects are declared as namedtuples because tuples are hashable, # something that we make use of when diffing site roles against guild roles. _Role = namedtuple('Role', ('id', 'name', 'colour', 'permissions', 'position')) @@ -207,10 +210,13 @@ class UserSyncer(Syncer): @staticmethod async def _sync(diff: _Diff) -> None: """Synchronise the database with the user cache of `guild`.""" + # Using asyncio.gather would still consume too many resources on the site. log.trace("Syncing created users...") if diff.created: - await bot.instance.api_client.post("bot/users", json=diff.created) + for chunk in chunked(diff.created, CHUNK_SIZE): + await bot.instance.api_client.post("bot/users", json=chunk) log.trace("Syncing updated users...") if diff.updated: - await bot.instance.api_client.patch("bot/users/bulk_patch", json=diff.updated) + for chunk in chunked(diff.updated, CHUNK_SIZE): + await bot.instance.api_client.patch("bot/users/bulk_patch", json=chunk) diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py index 208fc9e1f..3527bf8bb 100644 --- a/bot/exts/filters/filtering.py +++ b/bot/exts/filters/filtering.py @@ -48,7 +48,6 @@ class Stats(NamedTuple): message_content: str additional_embeds: Optional[List[discord.Embed]] - additional_embeds_msg: Optional[str] class Filtering(Cog): @@ -358,7 +357,6 @@ class Filtering(Cog): channel_id=Channels.mod_alerts, ping_everyone=ping_everyone, additional_embeds=stats.additional_embeds, - additional_embeds_msg=stats.additional_embeds_msg ) def _add_stats(self, name: str, match: FilterMatch, content: str) -> Stats: @@ -375,7 +373,6 @@ class Filtering(Cog): message_content = content additional_embeds = None - additional_embeds_msg = None self.bot.stats.incr(f"filters.{name}") @@ -392,13 +389,11 @@ class Filtering(Cog): embed.set_thumbnail(url=data["icon"]) embed.set_footer(text=f"Guild ID: {data['id']}") additional_embeds.append(embed) - additional_embeds_msg = "For the following guild(s):" elif name == "watch_rich_embeds": additional_embeds = match - additional_embeds_msg = "With the following embed(s):" - return Stats(message_content, additional_embeds, additional_embeds_msg) + return Stats(message_content, additional_embeds) @staticmethod def _check_filter(msg: Message) -> bool: diff --git a/bot/exts/help_channels/_channel.py b/bot/exts/help_channels/_channel.py index e717d7af8..224214b00 100644 --- a/bot/exts/help_channels/_channel.py +++ b/bot/exts/help_channels/_channel.py @@ -4,8 +4,10 @@ from datetime import datetime, timedelta import discord +import bot from bot import constants from bot.exts.help_channels import _caches, _message +from bot.utils.channel import try_get_channel log = logging.getLogger(__name__) @@ -55,3 +57,43 @@ async def get_in_use_time(channel_id: int) -> t.Optional[timedelta]: def is_excluded_channel(channel: discord.abc.GuildChannel) -> bool: """Check if a channel should be excluded from the help channel system.""" return not isinstance(channel, discord.TextChannel) or channel.id in EXCLUDED_CHANNELS + + +async def move_to_bottom(channel: discord.TextChannel, category_id: int, **options) -> None: + """ + Move the `channel` to the bottom position of `category` and edit channel attributes. + + To ensure "stable sorting", we use the `bulk_channel_update` endpoint and provide the current + positions of the other channels in the category as-is. This should make sure that the channel + really ends up at the bottom of the category. + + If `options` are provided, the channel will be edited after the move is completed. This is the + same order of operations that `discord.TextChannel.edit` uses. For information on available + options, see the documentation on `discord.TextChannel.edit`. While possible, position-related + options should be avoided, as it may interfere with the category move we perform. + """ + # Get a fresh copy of the category from the bot to avoid the cache mismatch issue we had. + category = await try_get_channel(category_id) + + payload = [{"id": c.id, "position": c.position} for c in category.channels] + + # Calculate the bottom position based on the current highest position in the category. If the + # category is currently empty, we simply use the current position of the channel to avoid making + # unnecessary changes to positions in the guild. + bottom_position = payload[-1]["position"] + 1 if payload else channel.position + + payload.append( + { + "id": channel.id, + "position": bottom_position, + "parent_id": category.id, + "lock_permissions": True, + } + ) + + # We use d.py's method to ensure our request is processed by d.py's rate limit manager + await bot.instance.http.bulk_channel_update(category.guild.id, payload) + + # Now that the channel is moved, we can edit the other attributes + if options: + await channel.edit(**options) diff --git a/bot/exts/help_channels/_cog.py b/bot/exts/help_channels/_cog.py index 983c5d183..0995c8a79 100644 --- a/bot/exts/help_channels/_cog.py +++ b/bot/exts/help_channels/_cog.py @@ -3,6 +3,7 @@ import logging import random import typing as t from datetime import datetime, timezone +from operator import attrgetter import discord import discord.abc @@ -10,12 +11,12 @@ from discord.ext import commands from bot import constants from bot.bot import Bot -from bot.exts.help_channels import _caches, _channel, _cooldown, _message, _name -from bot.utils import channel as channel_utils -from bot.utils.scheduling import Scheduler +from bot.exts.help_channels import _caches, _channel, _cooldown, _message, _name, _stats +from bot.utils import channel as channel_utils, lock, scheduling log = logging.getLogger(__name__) +NAMESPACE = "help" HELP_CHANNEL_TOPIC = """ This is a Python help channel. You can claim your own help channel in the Python Help: Available category. """ @@ -58,7 +59,7 @@ class HelpChannels(commands.Cog): def __init__(self, bot: Bot): self.bot = bot - self.scheduler = Scheduler(self.__class__.__name__) + self.scheduler = scheduling.Scheduler(self.__class__.__name__) # Categories self.available_category: discord.CategoryChannel = None @@ -73,7 +74,6 @@ class HelpChannels(commands.Cog): # Asyncio stuff self.queue_tasks: t.List[asyncio.Task] = [] - self.on_message_lock = asyncio.Lock() self.init_task = self.bot.loop.create_task(self.init_cog()) def cog_unload(self) -> None: @@ -87,6 +87,36 @@ class HelpChannels(commands.Cog): self.scheduler.cancel_all() + @lock.lock_arg(NAMESPACE, "message", attrgetter("channel.id")) + @lock.lock_arg(NAMESPACE, "message", attrgetter("author.id")) + @lock.lock_arg(f"{NAMESPACE}.unclaim", "message", attrgetter("author.id"), wait=True) + async def claim_channel(self, message: discord.Message) -> None: + """ + Claim the channel in which the question `message` was sent. + + Move the channel to the In Use category and pin the `message`. Add a cooldown to the + claimant to prevent them from asking another question. Lastly, make a new channel available. + """ + log.info(f"Channel #{message.channel} was claimed by `{message.author.id}`.") + await self.move_to_in_use(message.channel) + await _cooldown.revoke_send_permissions(message.author, self.scheduler) + + await _message.pin(message) + + # Add user with channel for dormant check. + await _caches.claimants.set(message.channel.id, message.author.id) + + self.bot.stats.incr("help.claimed") + + # Must use a timezone-aware datetime to ensure a correct POSIX timestamp. + timestamp = datetime.now(timezone.utc).timestamp() + await _caches.claim_times.set(message.channel.id, timestamp) + + await _caches.unanswered.set(message.channel.id, True) + + # Not awaited because it may indefinitely hold the lock while waiting for a channel. + scheduling.create_task(self.move_to_available(), name=f"help_claim_{message.id}") + def create_channel_queue(self) -> asyncio.Queue: """ Return a queue of dormant channels to use for getting the next available channel. @@ -124,8 +154,12 @@ class HelpChannels(commands.Cog): log.debug(f"Creating a new dormant channel named {name}.") return await self.dormant_category.create_text_channel(name, topic=HELP_CHANNEL_TOPIC) - async def dormant_check(self, ctx: commands.Context) -> bool: - """Return True if the user is the help channel claimant or passes the role check.""" + async def close_check(self, ctx: commands.Context) -> bool: + """Return True if the channel is in use and the user is the claimant or has a whitelisted role.""" + if ctx.channel.category != self.in_use_category: + log.debug(f"{ctx.author} invoked command 'close' outside an in-use help channel") + return False + if await _caches.claimants.get(ctx.channel.id) == ctx.author.id: log.trace(f"{ctx.author} is the help channel claimant, passing the check for dormant.") self.bot.stats.incr("help.dormant_invoke.claimant") @@ -144,18 +178,12 @@ class HelpChannels(commands.Cog): """ Make the current in-use help channel dormant. - Make the channel dormant if the user passes the `dormant_check`, - delete the message that invoked this. + May only be invoked by the channel's claimant or by staff. """ - log.trace("close command invoked; checking if the channel is in-use.") - - if ctx.channel.category != self.in_use_category: - log.debug(f"{ctx.author} invoked command 'dormant' outside an in-use help channel") - return - - if await self.dormant_check(ctx): - await self.move_to_dormant(ctx.channel, "command") - self.scheduler.cancel(ctx.channel.id) + # Don't use a discord.py check because the check needs to fail silently. + if await self.close_check(ctx): + log.info(f"Close command invoked by {ctx.author} in #{ctx.channel}.") + await self.unclaim_channel(ctx.channel, is_auto=False) async def get_available_candidate(self) -> discord.TextChannel: """ @@ -201,7 +229,7 @@ class HelpChannels(commands.Cog): elif missing < 0: log.trace(f"Moving {abs(missing)} superfluous available channels over to the Dormant category.") for channel in channels[:abs(missing)]: - await self.move_to_dormant(channel, "auto") + await self.unclaim_channel(channel) async def init_categories(self) -> None: """Get the help category objects. Remove the cog if retrieval fails.""" @@ -248,20 +276,10 @@ class HelpChannels(commands.Cog): self.close_command.enabled = True await self.init_available() - self.report_stats() + _stats.report_counts() log.info("Cog is ready!") - def report_stats(self) -> None: - """Report the channel count stats.""" - total_in_use = sum(1 for _ in _channel.get_category_channels(self.in_use_category)) - total_available = sum(1 for _ in _channel.get_category_channels(self.available_category)) - total_dormant = sum(1 for _ in _channel.get_category_channels(self.dormant_category)) - - self.bot.stats.gauge("help.total.in_use", total_in_use) - self.bot.stats.gauge("help.total.available", total_available) - self.bot.stats.gauge("help.total.dormant", total_dormant) - async def move_idle_channel(self, channel: discord.TextChannel, has_task: bool = True) -> None: """ Make the `channel` dormant if idle or schedule the move if still active. @@ -284,7 +302,7 @@ class HelpChannels(commands.Cog): f"and will be made dormant." ) - await self.move_to_dormant(channel, "auto") + await self.unclaim_channel(channel) else: # Cancel the existing task, if any. if has_task: @@ -298,45 +316,6 @@ class HelpChannels(commands.Cog): self.scheduler.schedule_later(delay, channel.id, self.move_idle_channel(channel)) - async def move_to_bottom_position(self, channel: discord.TextChannel, category_id: int, **options) -> None: - """ - Move the `channel` to the bottom position of `category` and edit channel attributes. - - To ensure "stable sorting", we use the `bulk_channel_update` endpoint and provide the current - positions of the other channels in the category as-is. This should make sure that the channel - really ends up at the bottom of the category. - - If `options` are provided, the channel will be edited after the move is completed. This is the - same order of operations that `discord.TextChannel.edit` uses. For information on available - options, see the documentation on `discord.TextChannel.edit`. While possible, position-related - options should be avoided, as it may interfere with the category move we perform. - """ - # Get a fresh copy of the category from the bot to avoid the cache mismatch issue we had. - category = await channel_utils.try_get_channel(category_id) - - payload = [{"id": c.id, "position": c.position} for c in category.channels] - - # Calculate the bottom position based on the current highest position in the category. If the - # category is currently empty, we simply use the current position of the channel to avoid making - # unnecessary changes to positions in the guild. - bottom_position = payload[-1]["position"] + 1 if payload else channel.position - - payload.append( - { - "id": channel.id, - "position": bottom_position, - "parent_id": category.id, - "lock_permissions": True, - } - ) - - # We use d.py's method to ensure our request is processed by d.py's rate limit manager - await self.bot.http.bulk_channel_update(category.guild.id, payload) - - # Now that the channel is moved, we can edit the other attributes - if options: - await channel.edit(**options) - async def move_to_available(self) -> None: """Make a channel available.""" log.trace("Making a channel available.") @@ -348,78 +327,81 @@ class HelpChannels(commands.Cog): log.trace(f"Moving #{channel} ({channel.id}) to the Available category.") - await self.move_to_bottom_position( + await _channel.move_to_bottom( channel=channel, category_id=constants.Categories.help_available, ) - self.report_stats() - - async def move_to_dormant(self, channel: discord.TextChannel, caller: str) -> None: - """ - Make the `channel` dormant. + _stats.report_counts() - A caller argument is provided for metrics. - """ + async def move_to_dormant(self, channel: discord.TextChannel) -> None: + """Make the `channel` dormant.""" log.info(f"Moving #{channel} ({channel.id}) to the Dormant category.") - - await self.move_to_bottom_position( + await _channel.move_to_bottom( channel=channel, category_id=constants.Categories.help_dormant, ) - await self.unclaim_channel(channel) - - self.bot.stats.incr(f"help.dormant_calls.{caller}") - - in_use_time = await _channel.get_in_use_time(channel.id) - if in_use_time: - self.bot.stats.timing("help.in_use_time", in_use_time) - - unanswered = await _caches.unanswered.get(channel.id) - if unanswered: - self.bot.stats.incr("help.sessions.unanswered") - elif unanswered is not None: - self.bot.stats.incr("help.sessions.answered") - - log.trace(f"Position of #{channel} ({channel.id}) is actually {channel.position}.") log.trace(f"Sending dormant message for #{channel} ({channel.id}).") embed = discord.Embed(description=_message.DORMANT_MSG) await channel.send(embed=embed) - await _message.unpin(channel) - log.trace(f"Pushing #{channel} ({channel.id}) into the channel queue.") self.channel_queue.put_nowait(channel) - self.report_stats() - async def unclaim_channel(self, channel: discord.TextChannel) -> None: + _stats.report_counts() + + @lock.lock_arg(f"{NAMESPACE}.unclaim", "channel") + async def unclaim_channel(self, channel: discord.TextChannel, *, is_auto: bool = True) -> None: """ - Mark the channel as unclaimed and remove the cooldown role from the claimant if needed. + Unclaim an in-use help `channel` to make it dormant. + + Unpin the claimant's question message and move the channel to the Dormant category. + Remove the cooldown role from the channel claimant if they have no other channels claimed. + Cancel the scheduled cooldown role removal task. - The role is only removed if they have no claimed channels left once the current one is unclaimed. - This method also handles canceling the automatic removal of the cooldown role. + Set `is_auto` to True if the channel was automatically closed or False if manually closed. """ - claimant_id = await _caches.claimants.pop(channel.id) + claimant_id = await _caches.claimants.get(channel.id) + _unclaim_channel = self._unclaim_channel + + # It could be possible that there is no claimant cached. In such case, it'd be useless and + # possibly incorrect to lock on None. Therefore, the lock is applied conditionally. + if claimant_id is not None: + decorator = lock.lock_arg(f"{NAMESPACE}.unclaim", "claimant_id", wait=True) + _unclaim_channel = decorator(_unclaim_channel) + + return await _unclaim_channel(channel, claimant_id, is_auto) - # Ignore missing task when cooldown has passed but the channel still isn't dormant. + async def _unclaim_channel(self, channel: discord.TextChannel, claimant_id: int, is_auto: bool) -> None: + """Actual implementation of `unclaim_channel`. See that for full documentation.""" + await _caches.claimants.delete(channel.id) + + # Ignore missing tasks because a channel may still be dormant after the cooldown expires. if claimant_id in self.scheduler: self.scheduler.cancel(claimant_id) claimant = self.bot.get_guild(constants.Guild.id).get_member(claimant_id) if claimant is None: log.info(f"{claimant_id} left the guild during their help session; the cooldown role won't be removed") - return - - # Remove the cooldown role if the claimant has no other channels left - if not any(claimant.id == user_id for _, user_id in await _caches.claimants.items()): + elif not any(claimant.id == user_id for _, user_id in await _caches.claimants.items()): + # Remove the cooldown role if the claimant has no other channels left await _cooldown.remove_cooldown_role(claimant) + await _message.unpin(channel) + await _stats.report_complete_session(channel.id, is_auto) + await self.move_to_dormant(channel) + + # Cancel the task that makes the channel dormant only if called by the close command. + # In other cases, the task is either already done or not-existent. + if not is_auto: + self.scheduler.cancel(channel.id) + async def move_to_in_use(self, channel: discord.TextChannel) -> None: """Make a channel in-use and schedule it to be made dormant.""" log.info(f"Moving #{channel} ({channel.id}) to the In Use category.") - await self.move_to_bottom_position( + await _channel.move_to_bottom( channel=channel, category_id=constants.Categories.help_in_use, ) @@ -428,7 +410,7 @@ class HelpChannels(commands.Cog): log.trace(f"Scheduling #{channel} ({channel.id}) to become dormant in {timeout} sec.") self.scheduler.schedule_later(timeout, channel.id, self.move_idle_channel(channel)) - self.report_stats() + _stats.report_counts() @commands.Cog.listener() async def on_message(self, message: discord.Message) -> None: @@ -436,51 +418,13 @@ class HelpChannels(commands.Cog): if message.author.bot: return # Ignore messages sent by bots. - channel = message.channel - - await _message.check_for_answer(message) - - is_available = channel_utils.is_in_category(channel, constants.Categories.help_available) - if not is_available or _channel.is_excluded_channel(channel): - return # Ignore messages outside the Available category or in excluded channels. - - log.trace("Waiting for the cog to be ready before processing messages.") await self.init_task - log.trace("Acquiring lock to prevent a channel from being processed twice...") - async with self.on_message_lock: - log.trace(f"on_message lock acquired for {message.id}.") - - if not channel_utils.is_in_category(channel, constants.Categories.help_available): - log.debug( - f"Message {message.id} will not make #{channel} ({channel.id}) in-use " - f"because another message in the channel already triggered that." - ) - return - - log.info(f"Channel #{channel} was claimed by `{message.author.id}`.") - await self.move_to_in_use(channel) - await _cooldown.revoke_send_permissions(message.author, self.scheduler) - - await _message.pin(message) - - # Add user with channel for dormant check. - await _caches.claimants.set(channel.id, message.author.id) - - self.bot.stats.incr("help.claimed") - - # Must use a timezone-aware datetime to ensure a correct POSIX timestamp. - timestamp = datetime.now(timezone.utc).timestamp() - await _caches.claim_times.set(channel.id, timestamp) - - await _caches.unanswered.set(channel.id, True) - - log.trace(f"Releasing on_message lock for {message.id}.") - - # Move a dormant channel to the Available category to fill in the gap. - # This is done last and outside the lock because it may wait indefinitely for a channel to - # be put in the queue. - await self.move_to_available() + if channel_utils.is_in_category(message.channel, constants.Categories.help_available): + if not _channel.is_excluded_channel(message.channel): + await self.claim_channel(message) + else: + await _message.check_for_answer(message) @commands.Cog.listener() async def on_message_delete(self, msg: discord.Message) -> None: @@ -489,15 +433,14 @@ class HelpChannels(commands.Cog): The new time for the dormant task is configured with `HelpChannels.deleted_idle_minutes`. """ + await self.init_task + if not channel_utils.is_in_category(msg.channel, constants.Categories.help_in_use): return if not await _message.is_empty(msg.channel): return - log.trace("Waiting for the cog to be ready before processing deleted messages.") - await self.init_task - log.info(f"Claimant of #{msg.channel} ({msg.author}) deleted message, channel is empty now. Rescheduling task.") # Cancel existing dormant task before scheduling new. diff --git a/bot/exts/help_channels/_stats.py b/bot/exts/help_channels/_stats.py new file mode 100644 index 000000000..b8778e7d9 --- /dev/null +++ b/bot/exts/help_channels/_stats.py @@ -0,0 +1,42 @@ +import logging + +from more_itertools import ilen + +import bot +from bot import constants +from bot.exts.help_channels import _caches, _channel + +log = logging.getLogger(__name__) + + +def report_counts() -> None: + """Report channel count stats of each help category.""" + for name in ("in_use", "available", "dormant"): + id_ = getattr(constants.Categories, f"help_{name}") + category = bot.instance.get_channel(id_) + + if category: + total = ilen(_channel.get_category_channels(category)) + bot.instance.stats.gauge(f"help.total.{name}", total) + else: + log.warning(f"Couldn't find category {name!r} to track channel count stats.") + + +async def report_complete_session(channel_id: int, is_auto: bool) -> None: + """ + Report stats for a completed help session channel `channel_id`. + + Set `is_auto` to True if the channel was automatically closed or False if manually closed. + """ + caller = "auto" if is_auto else "command" + bot.instance.stats.incr(f"help.dormant_calls.{caller}") + + in_use_time = await _channel.get_in_use_time(channel_id) + if in_use_time: + bot.instance.stats.timing("help.in_use_time", in_use_time) + + unanswered = await _caches.unanswered.get(channel_id) + if unanswered: + bot.instance.stats.incr("help.sessions.unanswered") + elif unanswered is not None: + bot.instance.stats.incr("help.sessions.answered") diff --git a/bot/exts/info/help.py b/bot/exts/info/help.py index 461ff82fd..3a05b2c8a 100644 --- a/bot/exts/info/help.py +++ b/bot/exts/info/help.py @@ -5,7 +5,7 @@ from contextlib import suppress from typing import List, Union from discord import Colour, Embed -from discord.ext.commands import Bot, Cog, Command, Context, Group, HelpCommand +from discord.ext.commands import Bot, Cog, Command, CommandError, Context, DisabledCommand, Group, HelpCommand from fuzzywuzzy import fuzz, process from fuzzywuzzy.utils import full_process @@ -20,6 +20,8 @@ log = logging.getLogger(__name__) COMMANDS_PER_PAGE = 8 PREFIX = constants.Bot.prefix +NOT_ALLOWED_TO_RUN_MESSAGE = "***You cannot run this command.***\n\n" + Category = namedtuple("Category", ["name", "description", "cogs"]) @@ -173,9 +175,16 @@ class CustomHelpCommand(HelpCommand): if aliases: command_details += f"**Can also use:** {aliases}\n\n" - # check if the user is allowed to run this command - if not await command.can_run(self.context): - command_details += "***You cannot run this command.***\n\n" + # when command is disabled, show message about it, + # when other CommandError or user is not allowed to run command, + # add this to help message. + try: + if not await command.can_run(self.context): + command_details += NOT_ALLOWED_TO_RUN_MESSAGE + except DisabledCommand: + command_details += "***This command is disabled.***\n\n" + except CommandError: + command_details += NOT_ALLOWED_TO_RUN_MESSAGE command_details += f"*{command.help or 'No details provided.'}*\n" embed.description = command_details diff --git a/bot/exts/info/information.py b/bot/exts/info/information.py index 38e760ee3..9fb875925 100644 --- a/bot/exts/info/information.py +++ b/bot/exts/info/information.py @@ -6,7 +6,8 @@ from collections import Counter, defaultdict from string import Template from typing import Any, Mapping, Optional, Tuple, Union -from discord import ChannelType, Colour, Embed, Guild, Message, Role, Status, utils +import fuzzywuzzy +from discord import ChannelType, Colour, Embed, Guild, Message, Role, Status from discord.abc import GuildChannel from discord.ext.commands import BucketType, Cog, Context, Paginator, command, group, has_any_role @@ -106,22 +107,28 @@ class Information(Cog): To specify multiple roles just add to the arguments, delimit roles with spaces in them using quotation marks. """ - parsed_roles = [] - failed_roles = [] + parsed_roles = set() + failed_roles = set() + all_roles = {role.id: role.name for role in ctx.guild.roles} for role_name in roles: if isinstance(role_name, Role): # Role conversion has already succeeded - parsed_roles.append(role_name) + parsed_roles.add(role_name) continue - role = utils.find(lambda r: r.name.lower() == role_name.lower(), ctx.guild.roles) + match = fuzzywuzzy.process.extractOne( + role_name, all_roles, score_cutoff=80, + scorer=fuzzywuzzy.fuzz.ratio + ) - if not role: - failed_roles.append(role_name) + if not match: + failed_roles.add(role_name) continue - parsed_roles.append(role) + # `match` is a (role name, score, role id) tuple + role = ctx.guild.get_role(match[2]) + parsed_roles.add(role) if failed_roles: await ctx.send(f":x: Could not retrieve the following roles: {', '.join(failed_roles)}") diff --git a/bot/exts/info/tags.py b/bot/exts/info/tags.py index da4154316..00b4d1a78 100644 --- a/bot/exts/info/tags.py +++ b/bot/exts/info/tags.py @@ -182,10 +182,15 @@ class Tags(Cog): matching_tags = self._get_tags_via_content(any, keywords or 'any', ctx.author) await self._send_matching_tags(ctx, keywords, matching_tags) - @tags_group.command(name='get', aliases=('show', 'g')) - async def get_command(self, ctx: Context, *, tag_name: TagNameConverter = None) -> None: - """Get a specified tag, or a list of all tags if no tag is specified.""" + async def display_tag(self, ctx: Context, tag_name: str = None) -> bool: + """ + If a tag is not found, display similar tag names as suggestions. + If a tag is not specified, display a paginated embed of all tags. + + Tags are on cooldowns on a per-tag, per-channel basis. If a tag is on cooldown, display + nothing and return False. + """ def _command_on_cooldown(tag_name: str) -> bool: """ Check if the command is currently on cooldown, on a per-tag, per-channel basis. @@ -212,7 +217,7 @@ class Tags(Cog): f"{ctx.author} tried to get the '{tag_name}' tag, but the tag is on cooldown. " f"Cooldown ends in {time_left:.1f} seconds." ) - return + return False if tag_name is not None: temp_founds = self._get_tag(tag_name) @@ -237,6 +242,7 @@ class Tags(Cog): await ctx.send(embed=Embed.from_dict(tag['embed'])), [ctx.author.id], ) + return True elif founds and len(tag_name) >= 3: await wait_for_deletion( await ctx.send( @@ -247,6 +253,7 @@ class Tags(Cog): ), [ctx.author.id], ) + return True else: tags = self._cache.values() @@ -255,6 +262,7 @@ class Tags(Cog): description="**There are no tags in the database!**", colour=Colour.red() )) + return True else: embed: Embed = Embed(title="**Current tags**") await LinePaginator.paginate( @@ -268,6 +276,18 @@ class Tags(Cog): empty=False, max_lines=15 ) + return True + + return False + + @tags_group.command(name='get', aliases=('show', 'g')) + async def get_command(self, ctx: Context, *, tag_name: TagNameConverter = None) -> bool: + """ + Get a specified tag, or a list of all tags if no tag is specified. + + Returns False if a tag is on cooldown, or if no matches are found. + """ + return await self.display_tag(ctx, tag_name) def setup(bot: Bot) -> None: diff --git a/bot/exts/moderation/dm_relay.py b/bot/exts/moderation/dm_relay.py index 4d5142b55..6d081741c 100644 --- a/bot/exts/moderation/dm_relay.py +++ b/bot/exts/moderation/dm_relay.py @@ -52,6 +52,10 @@ class DMRelay(Cog): await ctx.message.add_reaction("❌") return + if member.id == self.bot.user.id: + log.debug("Not sending message to bot user") + return await ctx.send("🚫 I can't send messages to myself!") + try: await member.send(message) except discord.errors.Forbidden: diff --git a/bot/exts/moderation/infraction/infractions.py b/bot/exts/moderation/infraction/infractions.py index 18e937e87..b3d069b34 100644 --- a/bot/exts/moderation/infraction/infractions.py +++ b/bot/exts/moderation/infraction/infractions.py @@ -198,7 +198,7 @@ class Infractions(InfractionScheduler, commands.Cog): # endregion # region: Temporary shadow infractions - @command(hidden=True, aliases=["shadowtempban, stempban"]) + @command(hidden=True, aliases=["shadowtempban", "stempban"]) async def shadow_tempban( self, ctx: Context, diff --git a/bot/exts/moderation/modlog.py b/bot/exts/moderation/modlog.py index b01de0ee3..e4b119f41 100644 --- a/bot/exts/moderation/modlog.py +++ b/bot/exts/moderation/modlog.py @@ -92,7 +92,6 @@ class ModLog(Cog, name="ModLog"): files: t.Optional[t.List[discord.File]] = None, content: t.Optional[str] = None, additional_embeds: t.Optional[t.List[discord.Embed]] = None, - additional_embeds_msg: t.Optional[str] = None, timestamp_override: t.Optional[datetime] = None, footer: t.Optional[str] = None, ) -> Context: @@ -133,8 +132,6 @@ class ModLog(Cog, name="ModLog"): ) if additional_embeds: - if additional_embeds_msg: - await channel.send(additional_embeds_msg) for additional_embed in additional_embeds: await channel.send(embed=additional_embed) diff --git a/bot/exts/moderation/slowmode.py b/bot/exts/moderation/slowmode.py index efd862aa5..c449752e1 100644 --- a/bot/exts/moderation/slowmode.py +++ b/bot/exts/moderation/slowmode.py @@ -7,7 +7,7 @@ from discord import TextChannel from discord.ext.commands import Cog, Context, group, has_any_role from bot.bot import Bot -from bot.constants import Emojis, MODERATION_ROLES +from bot.constants import Channels, Emojis, MODERATION_ROLES from bot.converters import DurationDelta from bot.utils import time @@ -15,6 +15,12 @@ log = logging.getLogger(__name__) SLOWMODE_MAX_DELAY = 21600 # seconds +COMMONLY_SLOWMODED_CHANNELS = { + Channels.python_general: "python_general", + Channels.discord_py: "discordpy", + Channels.off_topic_0: "ot0", +} + class Slowmode(Cog): """Commands for getting and setting slowmode delays of text channels.""" @@ -58,6 +64,10 @@ class Slowmode(Cog): log.info(f'{ctx.author} set the slowmode delay for #{channel} to {humanized_delay}.') await channel.edit(slowmode_delay=slowmode_delay) + if channel.id in COMMONLY_SLOWMODED_CHANNELS: + log.info(f'Recording slowmode change in stats for {channel.name}.') + self.bot.stats.gauge(f"slowmode.{COMMONLY_SLOWMODED_CHANNELS[channel.id]}", slowmode_delay) + await ctx.send( f'{Emojis.check_mark} The slowmode delay for {channel.mention} is now {humanized_delay}.' ) @@ -75,16 +85,7 @@ class Slowmode(Cog): @slowmode_group.command(name='reset', aliases=['r']) async def reset_slowmode(self, ctx: Context, channel: Optional[TextChannel]) -> None: """Reset the slowmode delay for a text channel to 0 seconds.""" - # Use the channel this command was invoked in if one was not given - if channel is None: - channel = ctx.channel - - log.info(f'{ctx.author} reset the slowmode delay for #{channel} to 0 seconds.') - - await channel.edit(slowmode_delay=0) - await ctx.send( - f'{Emojis.check_mark} The slowmode delay for {channel.mention} has been reset to 0 seconds.' - ) + await self.set_slowmode(ctx, channel, relativedelta(seconds=0)) async def cog_check(self, ctx: Context) -> bool: """Only allow moderators to invoke the commands in this cog.""" diff --git a/bot/exts/moderation/verification.py b/bot/exts/moderation/verification.py index 2a24c8ec6..bfe9b74b4 100644 --- a/bot/exts/moderation/verification.py +++ b/bot/exts/moderation/verification.py @@ -81,13 +81,11 @@ class Verification(Cog): if member.guild.id != constants.Guild.id: return # Only listen for PyDis events - raw_member = await self.bot.http.get_member(member.guild.id, member.id) - # If the user has the pending flag set, they will be using the alternate # gate and will not need a welcome DM with verification instructions. # We will send them an alternate DM once they verify with the welcome # video when they pass the gate. - if raw_member.get("pending"): + if member.pending: return log.trace(f"Sending on join message to new member: {member.id}") diff --git a/bot/exts/moderation/watchchannels/talentpool.py b/bot/exts/moderation/watchchannels/talentpool.py index df2ce586e..dd3349c3a 100644 --- a/bot/exts/moderation/watchchannels/talentpool.py +++ b/bot/exts/moderation/watchchannels/talentpool.py @@ -122,8 +122,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): if history: total = f"({len(history)} previous nominations in total)" start_reason = f"Watched: {textwrap.shorten(history[0]['reason'], width=500, placeholder='...')}" - end_reason = f"Unwatched: {textwrap.shorten(history[0]['end_reason'], width=500, placeholder='...')}" - msg += f"\n\nUser's previous watch reasons {total}:```{start_reason}\n\n{end_reason}```" + msg += f"\n\nUser's previous watch reasons {total}:```{start_reason}```" await ctx.send(msg) diff --git a/bot/log.py b/bot/log.py index 0935666d1..e92233a33 100644 --- a/bot/log.py +++ b/bot/log.py @@ -54,6 +54,9 @@ def setup() -> None: logging.getLogger("chardet").setLevel(logging.WARNING) logging.getLogger("async_rediscache").setLevel(logging.WARNING) + # Set back to the default of INFO even if asyncio's debug mode is enabled. + logging.getLogger("asyncio").setLevel(logging.INFO) + def setup_sentry() -> None: """Set up the Sentry logging integrations.""" diff --git a/bot/utils/lock.py b/bot/utils/lock.py index b4bb0ebc7..b4c93f063 100644 --- a/bot/utils/lock.py +++ b/bot/utils/lock.py @@ -48,39 +48,21 @@ class SharedEvent: await self._event.wait() -class LockGuard: - """ - A context manager which acquires and releases a lock (mutex). - - Raise RuntimeError if trying to acquire a locked lock. - """ - - def __init__(self): - self._locked = False - - @property - def locked(self) -> bool: - """Return True if currently locked or False if unlocked.""" - return self._locked - - def __enter__(self): - if self._locked: - raise RuntimeError("Cannot acquire a locked lock.") - - self._locked = True - - def __exit__(self, _exc_type, _exc_value, _traceback): # noqa: ANN001 - self._locked = False - return False # Indicate any raised exception shouldn't be suppressed. - - -def lock(namespace: Hashable, resource_id: ResourceId, *, raise_error: bool = False) -> Callable: +def lock( + namespace: Hashable, + resource_id: ResourceId, + *, + raise_error: bool = False, + wait: bool = False, +) -> Callable: """ Turn the decorated coroutine function into a mutually exclusive operation on a `resource_id`. - If any other mutually exclusive function currently holds the lock for a resource, do not run the - decorated function and return None. If `raise_error` is True, raise `LockedResourceError` if - the lock cannot be acquired. + If `wait` is True, wait until the lock becomes available. Otherwise, if any other mutually + exclusive function currently holds the lock for a resource, do not run the decorated function + and return None. + + If `raise_error` is True, raise `LockedResourceError` if the lock cannot be acquired. `namespace` is an identifier used to prevent collisions among resource IDs. @@ -110,15 +92,19 @@ def lock(namespace: Hashable, resource_id: ResourceId, *, raise_error: bool = Fa else: id_ = resource_id - log.trace(f"{name}: getting lock for resource {id_!r} under namespace {namespace!r}") + log.trace(f"{name}: getting the lock object for resource {namespace!r}:{id_!r}") # Get the lock for the ID. Create a lock if one doesn't exist yet. locks = __lock_dicts[namespace] - lock_guard = locks.setdefault(id_, LockGuard()) - - if not lock_guard.locked: - log.debug(f"{name}: resource {namespace!r}:{id_!r} is free; acquiring it...") - with lock_guard: + lock_ = locks.setdefault(id_, asyncio.Lock()) + + # It's safe to check an asyncio.Lock is free before acquiring it because: + # 1. Synchronous code like `if not lock_.locked()` does not yield execution + # 2. `asyncio.Lock.acquire()` does not internally await anything if the lock is free + # 3. awaits only yield execution to the event loop at actual I/O boundaries + if wait or not lock_.locked(): + log.debug(f"{name}: acquiring lock for resource {namespace!r}:{id_!r}...") + async with lock_: return await func(*args, **kwargs) else: log.info(f"{name}: aborted because resource {namespace!r}:{id_!r} is locked") @@ -135,6 +121,7 @@ def lock_arg( func: Callable[[Any], _IdCallableReturn] = None, *, raise_error: bool = False, + wait: bool = False, ) -> Callable: """ Apply the `lock` decorator using the value of the arg at the given name/position as the ID. @@ -142,5 +129,5 @@ def lock_arg( `func` is an optional callable or awaitable which will return the ID given the argument value. See `lock` docs for more information. """ - decorator_func = partial(lock, namespace, raise_error=raise_error) + decorator_func = partial(lock, namespace, raise_error=raise_error, wait=wait) return function.get_arg_value_wrapper(decorator_func, name_or_pos, func) diff --git a/bot/utils/scheduling.py b/bot/utils/scheduling.py index 03f31d78f..4dd036e4f 100644 --- a/bot/utils/scheduling.py +++ b/bot/utils/scheduling.py @@ -155,3 +155,20 @@ class Scheduler: # Log the exception if one exists. if exception: self._log.error(f"Error in task #{task_id} {id(done_task)}!", exc_info=exception) + + +def create_task(*args, **kwargs) -> asyncio.Task: + """Wrapper for `asyncio.create_task` which logs exceptions raised in the task.""" + task = asyncio.create_task(*args, **kwargs) + task.add_done_callback(_log_task_exception) + return task + + +def _log_task_exception(task: asyncio.Task) -> None: + """Retrieve and log the exception raised in `task` if one exists.""" + with contextlib.suppress(asyncio.CancelledError): + exception = task.exception() + # Log the exception if one exists. + if exception: + log = logging.getLogger(__name__) + log.error(f"Error in task {task.get_name()} {id(task)}!", exc_info=exception) diff --git a/config-default.yml b/config-default.yml index 175460a31..6695cffed 100644 --- a/config-default.yml +++ b/config-default.yml @@ -28,6 +28,9 @@ style: soft_green: 0x68c290 soft_orange: 0xf9cb54 bright_green: 0x01d277 + orange: 0xe67e22 + pink: 0xcf84e0 + purple: 0xb734eb emojis: defcon_disabled: "<:defcondisabled:470326273952972810>" @@ -68,6 +71,8 @@ style: comments: "<:reddit_comments:755845255001014384>" user: "<:reddit_users:755845303822974997>" + ok_hand: ":ok_hand:" + icons: crown_blurple: "https://cdn.discordapp.com/emojis/469964153289965568.png" crown_green: "https://cdn.discordapp.com/emojis/469964154719961088.png" @@ -152,11 +157,14 @@ guild: # Discussion meta: 429409067623251969 - python_discussion: &PY_DISCUSSION 267624335836053506 + python_general: &PY_GENERAL 267624335836053506 # Python Help: Available cooldown: 720603994149486673 + # Topical + discord_py: 343944376055103488 + # Logs attachment_log: &ATTACH_LOG 649243850006855680 message_log: &MESSAGE_LOG 467752170159079424 @@ -185,6 +193,8 @@ guild: mods: &MODS 305126844661760000 mod_alerts: 473092532147060736 mod_spam: &MOD_SPAM 620607373828030464 + mod_tools: &MOD_TOOLS 775413915391098921 + mod_meta: &MOD_META 775412552795947058 organisation: &ORGANISATION 551789653284356126 staff_lounge: &STAFF_LOUNGE 464905259261755392 duck_pond: &DUCK_POND 637820308341915648 @@ -218,6 +228,8 @@ guild: moderation_channels: - *ADMINS - *ADMIN_SPAM + - *MOD_META + - *MOD_TOOLS - *MODS - *MOD_SPAM @@ -421,7 +433,7 @@ code_block: # The channels which will be affected by a cooldown. These channels are also whitelisted. cooldown_channels: - - *PY_DISCUSSION + - *PY_GENERAL # Sending instructions triggers a cooldown on a per-channel basis. # More instruction messages will not be sent in the same channel until the cooldown has elapsed. @@ -519,5 +531,9 @@ voice_gate: voice_ping_delete_delay: 60 # Seconds before deleting the bot's ping to user in Voice Gate +branding: + cycle_frequency: 3 # How many days bot wait before refreshing server icon + + config: required_keys: ['bot.token'] diff --git a/tests/bot/exts/backend/sync/test_users.py b/tests/bot/exts/backend/sync/test_users.py index 61673e1bb..27932be95 100644 --- a/tests/bot/exts/backend/sync/test_users.py +++ b/tests/bot/exts/backend/sync/test_users.py @@ -188,30 +188,37 @@ class UserSyncerSyncTests(unittest.IsolatedAsyncioTestCase): """Tests for the API requests that sync users.""" def setUp(self): - patcher = mock.patch("bot.instance", new=helpers.MockBot()) - self.bot = patcher.start() - self.addCleanup(patcher.stop) + bot_patcher = mock.patch("bot.instance", new=helpers.MockBot()) + self.bot = bot_patcher.start() + self.addCleanup(bot_patcher.stop) + + chunk_patcher = mock.patch("bot.exts.backend.sync._syncers.CHUNK_SIZE", 2) + self.chunk_size = chunk_patcher.start() + self.addCleanup(chunk_patcher.stop) + + self.chunk_count = 2 + self.users = [fake_user(id=i) for i in range(self.chunk_size * self.chunk_count)] async def test_sync_created_users(self): """Only POST requests should be made with the correct payload.""" - users = [fake_user(id=111), fake_user(id=222)] - - diff = _Diff(users, [], None) + diff = _Diff(self.users, [], None) await UserSyncer._sync(diff) - self.bot.api_client.post.assert_called_once_with("bot/users", json=diff.created) + self.bot.api_client.post.assert_any_call("bot/users", json=diff.created[:self.chunk_size]) + self.bot.api_client.post.assert_any_call("bot/users", json=diff.created[self.chunk_size:]) + self.assertEqual(self.bot.api_client.post.call_count, self.chunk_count) self.bot.api_client.put.assert_not_called() self.bot.api_client.delete.assert_not_called() async def test_sync_updated_users(self): """Only PUT requests should be made with the correct payload.""" - users = [fake_user(id=111), fake_user(id=222)] - - diff = _Diff([], users, None) + diff = _Diff([], self.users, None) await UserSyncer._sync(diff) - self.bot.api_client.patch.assert_called_once_with("bot/users/bulk_patch", json=diff.updated) + self.bot.api_client.patch.assert_any_call("bot/users/bulk_patch", json=diff.updated[:self.chunk_size]) + self.bot.api_client.patch.assert_any_call("bot/users/bulk_patch", json=diff.updated[self.chunk_size:]) + self.assertEqual(self.bot.api_client.patch.call_count, self.chunk_count) self.bot.api_client.post.assert_not_called() self.bot.api_client.delete.assert_not_called() diff --git a/tests/bot/exts/info/test_information.py b/tests/bot/exts/info/test_information.py index d077be960..80731c9f0 100644 --- a/tests/bot/exts/info/test_information.py +++ b/tests/bot/exts/info/test_information.py @@ -65,7 +65,7 @@ class InformationCogTests(unittest.IsolatedAsyncioTestCase): permissions=discord.Permissions(0), ) - self.ctx.guild.roles.append([dummy_role, admin_role]) + self.ctx.guild.roles.extend([dummy_role, admin_role]) self.cog.role_info.can_run = unittest.mock.AsyncMock() self.cog.role_info.can_run.return_value = True diff --git a/tests/bot/exts/moderation/test_slowmode.py b/tests/bot/exts/moderation/test_slowmode.py index dad751e0d..5483b7a64 100644 --- a/tests/bot/exts/moderation/test_slowmode.py +++ b/tests/bot/exts/moderation/test_slowmode.py @@ -85,22 +85,14 @@ class SlowmodeTests(unittest.IsolatedAsyncioTestCase): self.ctx.reset_mock() - async def test_reset_slowmode_no_channel(self) -> None: - """Reset slowmode without a given channel.""" - self.ctx.channel = MockTextChannel(name='careers', slowmode_delay=6) - - await self.cog.reset_slowmode(self.cog, self.ctx, None) - self.ctx.send.assert_called_once_with( - f'{Emojis.check_mark} The slowmode delay for #careers has been reset to 0 seconds.' - ) - - async def test_reset_slowmode_with_channel(self) -> None: + async def test_reset_slowmode_sets_delay_to_zero(self) -> None: """Reset slowmode with a given channel.""" text_channel = MockTextChannel(name='meta', slowmode_delay=1) + self.cog.set_slowmode = mock.AsyncMock() await self.cog.reset_slowmode(self.cog, self.ctx, text_channel) - self.ctx.send.assert_called_once_with( - f'{Emojis.check_mark} The slowmode delay for #meta has been reset to 0 seconds.' + self.cog.set_slowmode.assert_awaited_once_with( + self.ctx, text_channel, relativedelta(seconds=0) ) @mock.patch("bot.exts.moderation.slowmode.has_any_role") |