diff options
Diffstat (limited to 'bot/exts/backend')
| -rw-r--r-- | bot/exts/backend/branding/__init__.py | 4 | ||||
| -rw-r--r-- | bot/exts/backend/branding/_cog.py | 137 | ||||
| -rw-r--r-- | bot/exts/backend/branding/_repository.py | 11 | ||||
| -rw-r--r-- | bot/exts/backend/config_verifier.py | 8 | ||||
| -rw-r--r-- | bot/exts/backend/error_handler.py | 53 | ||||
| -rw-r--r-- | bot/exts/backend/logging.py | 6 | ||||
| -rw-r--r-- | bot/exts/backend/sync/__init__.py | 4 | ||||
| -rw-r--r-- | bot/exts/backend/sync/_cog.py | 24 | ||||
| -rw-r--r-- | bot/exts/backend/sync/_syncers.py | 15 |
9 files changed, 153 insertions, 109 deletions
diff --git a/bot/exts/backend/branding/__init__.py b/bot/exts/backend/branding/__init__.py index 20a747b7f..8460465cb 100644 --- a/bot/exts/backend/branding/__init__.py +++ b/bot/exts/backend/branding/__init__.py @@ -2,6 +2,6 @@ from bot.bot import Bot from bot.exts.backend.branding._cog import Branding -def setup(bot: Bot) -> None: +async def setup(bot: Bot) -> None: """Load Branding cog.""" - bot.add_cog(Branding(bot)) + await bot.add_cog(Branding(bot)) diff --git a/bot/exts/backend/branding/_cog.py b/bot/exts/backend/branding/_cog.py index 0c5839a7a..ff2704835 100644 --- a/bot/exts/backend/branding/_cog.py +++ b/bot/exts/backend/branding/_cog.py @@ -1,6 +1,7 @@ import asyncio import contextlib import random +import types import typing as t from datetime import timedelta from enum import Enum @@ -17,7 +18,6 @@ from bot.constants import Branding as BrandingConfig, Channels, Colours, Guild, from bot.decorators import mock_in_debug from bot.exts.backend.branding._repository import BrandingRepository, Event, RemoteObject from bot.log import get_logger -from bot.utils import scheduling log = get_logger(__name__) @@ -105,19 +105,24 @@ class Branding(commands.Cog): """ # RedisCache[ - # "daemon_active": bool | If True, daemon starts on start-up. Controlled via commands. - # "event_path": str | Current event's path in the branding repo. - # "event_description": str | Current event's Markdown description. - # "event_duration": str | Current event's human-readable date range. - # "banner_hash": str | SHA of the currently applied banner. - # "icons_hash": str | Compound SHA of all icons in current rotation. - # "last_rotation_timestamp": float | POSIX UTC timestamp. + # "daemon_active": bool | If True, daemon starts on start-up. Controlled via commands. + # "event_path": str | Current event's path in the branding repo. + # "event_description": str | Current event's Markdown description. + # "event_duration": str | Current event's human-readable date range. + # "banners_hash": str | Compound SHA of all banners in the current rotation. + # "icons_hash": str | Compound SHA of all icons in current rotation. + # "last_icon_rotation_timestamp": float | POSIX UTC timestamp. + # "last_banner_rotation_timestamp": float | POSIX UTC timestamp. # ] cache_information = RedisCache() - # Icons in current rotation. Keys (str) are download URLs, values (int) track the amount of times each - # icon has been used in the current rotation. - cache_icons = RedisCache() + # Icons and banners in current rotation. + # Keys (str) are download URLs, values (int) track the amount of times each + # asset has been used in the current rotation. + asset_caches = types.MappingProxyType({ + AssetType.ICON: RedisCache(namespace="Branding.icon_cache"), + AssetType.BANNER: RedisCache(namespace="Branding.banner_cache") + }) # All available event names & durations. Cached by the daemon nightly; read by the calendar command. cache_events = RedisCache() @@ -127,7 +132,9 @@ class Branding(commands.Cog): self.bot = bot self.repository = BrandingRepository(bot) - scheduling.create_task(self.maybe_start_daemon(), event_loop=self.bot.loop) # Start depending on cache. + async def cog_load(self) -> None: + """Carry out cog asynchronous initialisation.""" + await self.maybe_start_daemon() # Start depending on cache. # region: Internal logic & state management @@ -163,107 +170,92 @@ class Branding(commands.Cog): log.trace("Asset uploaded successfully.") return True - async def apply_banner(self, banner: RemoteObject) -> bool: + async def rotate_assets(self, asset_type: AssetType) -> bool: """ - Apply `banner` to the guild and cache its hash if successful. + Choose and apply the next-up asset in rotation. - Banners should always be applied via this method to ensure that the last hash is cached. - - Return a boolean indicating whether the application was successful. - """ - success = await self.apply_asset(AssetType.BANNER, banner.download_url) - - if success: - await self.cache_information.set("banner_hash", banner.sha) - - return success - - async def rotate_icons(self) -> bool: - """ - Choose and apply the next-up icon in rotation. - - We keep track of the amount of times each icon has been used. The values in `cache_icons` can be understood - to be iteration IDs. When an icon is chosen & applied, we bump its count, pushing it into the next iteration. + We keep track of the amount of times each asset has been used. The values in the cache can be understood + to be iteration IDs. When an asset is chosen & applied, we bump its count, pushing it into the next iteration. Once the current iteration (lowest count in the cache) depletes, we move onto the next iteration. - In the case that there is only 1 icon in the rotation and has already been applied, do nothing. + In the case that there is only 1 asset in the rotation and has already been applied, do nothing. - Return a boolean indicating whether a new icon was applied successfully. + Return a boolean indicating whether a new asset was applied successfully. """ - log.debug("Rotating icons.") + log.debug(f"Rotating {asset_type.value}s.") - state = await self.cache_icons.to_dict() - log.trace(f"Total icons in rotation: {len(state)}.") + state = await self.asset_caches[asset_type].to_dict() + log.trace(f"Total {asset_type.value}s in rotation: {len(state)}.") if not state: # This would only happen if rotation not initiated, but we can handle gracefully. - log.warning("Attempted icon rotation with an empty icon cache. This indicates wrong logic.") + log.warning(f"Attempted {asset_type.value} rotation with an empty cache. This indicates wrong logic.") return False if len(state) == 1 and 1 in state.values(): - log.debug("Aborting icon rotation: only 1 icon is available and has already been applied.") + log.debug(f"Aborting {asset_type.value} rotation: only 1 asset is available and has already been applied.") return False current_iteration = min(state.values()) # Choose iteration to draw from. options = [download_url for download_url, times_used in state.items() if times_used == current_iteration] - log.trace(f"Choosing from {len(options)} icons in iteration {current_iteration}.") - next_icon = random.choice(options) + log.trace(f"Choosing from {len(options)} {asset_type.value}s in iteration {current_iteration}.") + next_asset = random.choice(options) - success = await self.apply_asset(AssetType.ICON, next_icon) + success = await self.apply_asset(asset_type, next_asset) if success: - await self.cache_icons.increment(next_icon) # Push the icon into the next iteration. + await self.asset_caches[asset_type].increment(next_asset) # Push the asset into the next iteration. timestamp = Arrow.utcnow().timestamp() - await self.cache_information.set("last_rotation_timestamp", timestamp) + await self.cache_information.set(f"last_{asset_type.value}_rotation_timestamp", timestamp) return success - async def maybe_rotate_icons(self) -> None: + async def maybe_rotate_assets(self, asset_type: AssetType) -> None: """ - Call `rotate_icons` if the configured amount of time has passed since last rotation. + Call `rotate_assets` if the configured amount of time has passed since last rotation. We offset the calculated time difference into the future to avoid off-by-a-little-bit errors. Because there is work to be done before the timestamp is read and written, the next read will likely commence slightly under 24 hours after the last write. """ - log.debug("Checking whether it's time for icons to rotate.") + log.debug(f"Checking whether it's time for {asset_type.value}s to rotate.") - last_rotation_timestamp = await self.cache_information.get("last_rotation_timestamp") + last_rotation_timestamp = await self.cache_information.get(f"last_{asset_type.value}_rotation_timestamp") if last_rotation_timestamp is None: # Maiden case ~ never rotated. - await self.rotate_icons() + await self.rotate_assets(asset_type) return last_rotation = Arrow.utcfromtimestamp(last_rotation_timestamp) difference = (Arrow.utcnow() - last_rotation) + timedelta(minutes=5) - log.trace(f"Icons last rotated at {last_rotation} (difference: {difference}).") + log.trace(f"{asset_type.value.title()}s last rotated at {last_rotation} (difference: {difference}).") if difference.days >= BrandingConfig.cycle_frequency: - await self.rotate_icons() + await self.rotate_assets(asset_type) - async def initiate_icon_rotation(self, available_icons: t.List[RemoteObject]) -> None: + async def initiate_rotation(self, asset_type: AssetType, available_assets: list[RemoteObject]) -> None: """ - Set up a new icon rotation. + Set up a new asset rotation. - This function should be called whenever available icons change. This is generally the case when we enter + This function should be called whenever available asset groups change. This is generally the case when we enter a new event, but potentially also when the assets of an on-going event change. In such cases, a reset - of `cache_icons` is necessary, because it contains download URLs which may have gotten stale. + of the cache is necessary, because it contains download URLs which may have gotten stale. - This function does not upload a new icon! + This function does not upload a new asset! """ - log.debug("Initiating new icon rotation.") + log.debug(f"Initiating new {asset_type.value} rotation.") - await self.cache_icons.clear() + await self.asset_caches[asset_type].clear() - new_state = {icon.download_url: 0 for icon in available_icons} - await self.cache_icons.update(new_state) + new_state = {asset.download_url: 0 for asset in available_assets} + await self.asset_caches[asset_type].update(new_state) - log.trace(f"Icon rotation initiated for {len(new_state)} icons.") + log.trace(f"{asset_type.value.title()} rotation initiated for {len(new_state)} assets.") - await self.cache_information.set("icons_hash", compound_hash(available_icons)) + await self.cache_information.set(f"{asset_type.value}s_hash", compound_hash(available_assets)) async def send_info_embed(self, channel_id: int, *, is_notification: bool) -> None: """ @@ -315,10 +307,12 @@ class Branding(commands.Cog): """ log.info(f"Entering event: '{event.path}'.") - banner_success = await self.apply_banner(event.banner) # Only one asset ~ apply directly. + # Prepare and apply new icon and banner rotations + await self.initiate_rotation(AssetType.ICON, event.icons) + await self.initiate_rotation(AssetType.BANNER, event.banners) - await self.initiate_icon_rotation(event.icons) # Prepare a new rotation. - icon_success = await self.rotate_icons() # Apply an icon from the new rotation. + icon_success = await self.rotate_assets(AssetType.ICON) + banner_success = await self.rotate_assets(AssetType.BANNER) # This will only be False in the case of a manual same-event re-synchronisation. event_changed = event.path != await self.cache_information.get("event_path") @@ -413,7 +407,7 @@ class Branding(commands.Cog): if should_begin: self.daemon_loop.start() - def cog_unload(self) -> None: + async def cog_unload(self) -> None: """ Cancel the daemon in case of cog unload. @@ -453,16 +447,19 @@ class Branding(commands.Cog): log.trace("Daemon main: event has not changed, checking for change in assets.") - if new_event.banner.sha != await self.cache_information.get("banner_hash"): + if compound_hash(new_event.banners) != await self.cache_information.get("banners_hash"): log.debug("Daemon main: detected banner change.") - await self.apply_banner(new_event.banner) + await self.initiate_rotation(AssetType.BANNER, new_event.banners) + await self.rotate_assets(AssetType.BANNER) + else: + await self.maybe_rotate_assets(AssetType.BANNER) if compound_hash(new_event.icons) != await self.cache_information.get("icons_hash"): log.debug("Daemon main: detected icon change.") - await self.initiate_icon_rotation(new_event.icons) - await self.rotate_icons() + await self.initiate_rotation(AssetType.ICON, new_event.icons) + await self.rotate_assets(AssetType.ICON) else: - await self.maybe_rotate_icons() + await self.maybe_rotate_assets(AssetType.ICON) @tasks.loop(hours=24) async def daemon_loop(self) -> None: diff --git a/bot/exts/backend/branding/_repository.py b/bot/exts/backend/branding/_repository.py index d88ea67f3..e14f0a1ef 100644 --- a/bot/exts/backend/branding/_repository.py +++ b/bot/exts/backend/branding/_repository.py @@ -64,8 +64,8 @@ class Event(t.NamedTuple): path: str # Path from repo root where event lives. This is the event's identity. meta: MetaFile - banner: RemoteObject - icons: t.List[RemoteObject] + banners: list[RemoteObject] + icons: list[RemoteObject] def __str__(self) -> str: return f"<Event at '{self.path}'>" @@ -163,21 +163,24 @@ class BrandingRepository: """ contents = await self.fetch_directory(directory.path) - missing_assets = {"meta.md", "banner.png", "server_icons"} - contents.keys() + missing_assets = {"meta.md", "server_icons", "banners"} - contents.keys() if missing_assets: raise BrandingMisconfiguration(f"Directory is missing following assets: {missing_assets}") server_icons = await self.fetch_directory(contents["server_icons"].path, types=("file",)) + banners = await self.fetch_directory(contents["banners"].path, types=("file",)) if len(server_icons) == 0: raise BrandingMisconfiguration("Found no server icons!") + if len(banners) == 0: + raise BrandingMisconfiguration("Found no server banners!") meta_bytes = await self.fetch_file(contents["meta.md"].download_url) meta_file = self.parse_meta_file(meta_bytes) - return Event(directory.path, meta_file, contents["banner.png"], list(server_icons.values())) + return Event(directory.path, meta_file, list(banners.values()), list(server_icons.values())) async def get_events(self) -> t.List[Event]: """ diff --git a/bot/exts/backend/config_verifier.py b/bot/exts/backend/config_verifier.py index dc85a65a2..97c8869a1 100644 --- a/bot/exts/backend/config_verifier.py +++ b/bot/exts/backend/config_verifier.py @@ -3,7 +3,6 @@ from discord.ext.commands import Cog from bot import constants from bot.bot import Bot from bot.log import get_logger -from bot.utils import scheduling log = get_logger(__name__) @@ -13,9 +12,8 @@ class ConfigVerifier(Cog): def __init__(self, bot: Bot): self.bot = bot - self.channel_verify_task = scheduling.create_task(self.verify_channels(), event_loop=self.bot.loop) - async def verify_channels(self) -> None: + async def cog_load(self) -> None: """ Verify channels. @@ -34,6 +32,6 @@ class ConfigVerifier(Cog): log.warning(f"Configured channels do not exist in server: {', '.join(invalid_channels)}.") -def setup(bot: Bot) -> None: +async def setup(bot: Bot) -> None: """Load the ConfigVerifier cog.""" - bot.add_cog(ConfigVerifier(bot)) + await bot.add_cog(ConfigVerifier(bot)) diff --git a/bot/exts/backend/error_handler.py b/bot/exts/backend/error_handler.py index 5bef72808..761991488 100644 --- a/bot/exts/backend/error_handler.py +++ b/bot/exts/backend/error_handler.py @@ -1,13 +1,13 @@ +import copy import difflib +from botcore.site_api import ResponseCodeError from discord import Embed from discord.ext.commands import ChannelNotFound, Cog, Context, TextChannelConverter, VoiceChannelConverter, errors from sentry_sdk import push_scope -from bot.api import ResponseCodeError from bot.bot import Bot from bot.constants import Colours, Icons, MODERATION_ROLES -from bot.converters import TagNameConverter from bot.errors import InvalidInfractedUserError, LockedResourceError from bot.log import get_logger from bot.utils.checks import ContextCheckFailure @@ -66,6 +66,8 @@ class ErrorHandler(Cog): if isinstance(e, errors.CommandNotFound) and not getattr(ctx, "invoked_from_error_handler", False): if await self.try_silence(ctx): return + if await self.try_run_eval(ctx): + return await self.try_get_tag(ctx) # Try to look for a tag with the command's name elif isinstance(e, errors.UserInputError): log.debug(debug_message) @@ -174,20 +176,36 @@ class ErrorHandler(Cog): await self.on_command_error(ctx, tag_error) return - try: - tag_name = await TagNameConverter.convert(ctx, ctx.invoked_with) - except errors.BadArgument: - log.debug( - f"{ctx.author} tried to use an invalid command " - f"and the fallback tag failed validation in TagNameConverter." - ) - else: - if await ctx.invoke(tags_get_command, tag_name=tag_name): - return + if await ctx.invoke(tags_get_command, argument_string=ctx.message.content): + return if not any(role.id in MODERATION_ROLES for role in ctx.author.roles): await self.send_command_suggestion(ctx, ctx.invoked_with) + async def try_run_eval(self, ctx: Context) -> bool: + """ + Attempt to run eval command with backticks directly after command. + + For example: !eval```print("hi")``` + + Return True if command was invoked, else False + """ + msg = copy.copy(ctx.message) + + command, sep, end = msg.content.partition("```") + msg.content = command + " " + sep + end + new_ctx = await self.bot.get_context(msg) + + eval_command = self.bot.get_command("eval") + if eval_command is None or new_ctx.command != eval_command: + return False + + log.debug("Running fixed eval command.") + new_ctx.invoked_from_error_handler = True + await self.bot.invoke(new_ctx) + + return True + async def send_command_suggestion(self, ctx: Context, command_name: str) -> None: """Sends user similar commands if any can be found.""" # No similar tag found, or tag on cooldown - @@ -293,8 +311,11 @@ class ErrorHandler(Cog): await ctx.send("There does not seem to be anything matching your query.") ctx.bot.stats.incr("errors.api_error_404") elif e.status == 400: - content = await e.response.json() - log.debug(f"API responded with 400 for command {ctx.command}: %r.", content) + log.error( + "API responded with 400 for command %s: %r.", + ctx.command, + e.response_json or e.response_text, + ) await ctx.send("According to the API, your request is malformed.") ctx.bot.stats.incr("errors.api_error_400") elif 500 <= e.status < 600: @@ -337,6 +358,6 @@ class ErrorHandler(Cog): log.error(f"Error executing command invoked by {ctx.message.author}: {ctx.message.content}", exc_info=e) -def setup(bot: Bot) -> None: +async def setup(bot: Bot) -> None: """Load the ErrorHandler cog.""" - bot.add_cog(ErrorHandler(bot)) + await bot.add_cog(ErrorHandler(bot)) diff --git a/bot/exts/backend/logging.py b/bot/exts/backend/logging.py index 2d03cd580..b9504c2eb 100644 --- a/bot/exts/backend/logging.py +++ b/bot/exts/backend/logging.py @@ -1,10 +1,10 @@ +from botcore.utils import scheduling from discord import Embed from discord.ext.commands import Cog from bot.bot import Bot from bot.constants import Channels, DEBUG_MODE from bot.log import get_logger -from bot.utils import scheduling log = get_logger(__name__) @@ -36,6 +36,6 @@ class Logging(Cog): await self.bot.get_channel(Channels.dev_log).send(embed=embed) -def setup(bot: Bot) -> None: +async def setup(bot: Bot) -> None: """Load the Logging cog.""" - bot.add_cog(Logging(bot)) + await bot.add_cog(Logging(bot)) diff --git a/bot/exts/backend/sync/__init__.py b/bot/exts/backend/sync/__init__.py index 829098f79..1978917e6 100644 --- a/bot/exts/backend/sync/__init__.py +++ b/bot/exts/backend/sync/__init__.py @@ -1,8 +1,8 @@ from bot.bot import Bot -def setup(bot: Bot) -> None: +async def setup(bot: Bot) -> None: """Load the Sync cog.""" # Defer import to reduce side effects from importing the sync package. from bot.exts.backend.sync._cog import Sync - bot.add_cog(Sync(bot)) + await bot.add_cog(Sync(bot)) diff --git a/bot/exts/backend/sync/_cog.py b/bot/exts/backend/sync/_cog.py index 80f5750bc..433ff5024 100644 --- a/bot/exts/backend/sync/_cog.py +++ b/bot/exts/backend/sync/_cog.py @@ -1,17 +1,18 @@ +import asyncio from typing import Any, Dict +from botcore.site_api import ResponseCodeError from discord import Member, Role, User from discord.ext import commands from discord.ext.commands import Cog, Context from bot import constants -from bot.api import ResponseCodeError from bot.bot import Bot from bot.exts.backend.sync import _syncers from bot.log import get_logger -from bot.utils import scheduling log = get_logger(__name__) +MAX_ATTEMPTS = 3 class Sync(Cog): @@ -19,9 +20,8 @@ class Sync(Cog): def __init__(self, bot: Bot) -> None: self.bot = bot - scheduling.create_task(self.sync_guild(), event_loop=self.bot.loop) - async def sync_guild(self) -> None: + async def cog_load(self) -> None: """Syncs the roles/users of the guild with the database.""" await self.bot.wait_until_guild_available() @@ -29,6 +29,22 @@ class Sync(Cog): if guild is None: return + attempts = 0 + while True: + attempts += 1 + if guild.chunked: + log.info("Guild was found to be chunked after %d attempt(s).", attempts) + break + + if attempts == MAX_ATTEMPTS: + log.info("Guild not chunked after %d attempts, calling chunk manually.", MAX_ATTEMPTS) + await guild.chunk() + break + + log.info("Attempt %d/%d: Guild not yet chunked, checking again in 10s.", attempts, MAX_ATTEMPTS) + await asyncio.sleep(10) + + log.info("Starting syncers.") for syncer in (_syncers.RoleSyncer, _syncers.UserSyncer): await syncer.sync(guild) diff --git a/bot/exts/backend/sync/_syncers.py b/bot/exts/backend/sync/_syncers.py index 45301b098..799137cb9 100644 --- a/bot/exts/backend/sync/_syncers.py +++ b/bot/exts/backend/sync/_syncers.py @@ -2,14 +2,14 @@ import abc import typing as t from collections import namedtuple +import discord.errors +from botcore.site_api import ResponseCodeError from discord import Guild from discord.ext.commands import Context from more_itertools import chunked import bot -from bot.api import ResponseCodeError from bot.log import get_logger -from bot.utils.members import get_or_fetch_member log = get_logger(__name__) @@ -157,7 +157,16 @@ class UserSyncer(Syncer): if db_user[db_field] != guild_value: updated_fields[db_field] = guild_value - if guild_user := await get_or_fetch_member(guild, db_user["id"]): + guild_user = guild.get_member(db_user["id"]) + if not guild_user and db_user["in_guild"]: + # The member was in the guild during the last sync. + # We try to fetch them to verify cache integrity. + try: + guild_user = await guild.fetch_member(db_user["id"]) + except discord.errors.NotFound: + guild_user = None + + if guild_user: seen_guild_users.add(guild_user.id) maybe_update("name", guild_user.name) |