diff options
| author | 2019-07-16 15:02:43 -0700 | |
|---|---|---|
| committer | 2019-07-16 15:02:43 -0700 | |
| commit | 3232fd566d5b35f10ffff0d47129f9c052df1303 (patch) | |
| tree | 5c7eec255a7973afaba5b033e9feb7f3c4d6ad70 | |
| parent | Merge remote-tracking branch 'remotes/origin/master' into django (diff) | |
| parent | Merge pull request #376 from python-discord/django-watchchannels (diff) | |
Merge remote-tracking branch 'origin/django' into django-merge
Merge in RMQ removal, Snekbox changes, and watch channel rework, among
other minor things.
| -rw-r--r-- | bot/__init__.py | 2 | ||||
| -rw-r--r-- | bot/__main__.py | 14 | ||||
| -rw-r--r-- | bot/cogs/alias.py | 26 | ||||
| -rw-r--r-- | bot/cogs/bigbrother.py | 258 | ||||
| -rw-r--r-- | bot/cogs/nominations.py | 120 | ||||
| -rw-r--r-- | bot/cogs/rmq.py | 229 | ||||
| -rw-r--r-- | bot/cogs/snekbox.py | 264 | ||||
| -rw-r--r-- | bot/cogs/watchchannels/__init__.py | 15 | ||||
| -rw-r--r-- | bot/cogs/watchchannels/bigbrother.py | 100 | ||||
| -rw-r--r-- | bot/cogs/watchchannels/talentpool.py | 233 | ||||
| -rw-r--r-- | bot/cogs/watchchannels/watchchannel.py | 353 | ||||
| -rw-r--r-- | bot/constants.py | 20 | ||||
| -rw-r--r-- | bot/utils/messages.py | 28 | ||||
| -rw-r--r-- | bot/utils/moderation.py | 12 | ||||
| -rw-r--r-- | bot/utils/service_discovery.py | 22 | ||||
| -rw-r--r-- | config-default.yml | 16 |
16 files changed, 927 insertions, 785 deletions
diff --git a/bot/__init__.py b/bot/__init__.py index a088138a0..8efa5e53c 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -55,7 +55,7 @@ else: logging.basicConfig( - format="%(asctime)s pd.beardfist.com Bot: | %(name)30s | %(levelname)8s | %(message)s", + format="%(asctime)s pd.beardfist.com Bot: | %(name)33s | %(levelname)8s | %(message)s", datefmt="%b %d %H:%M:%S", level=logging.TRACE if DEBUG_MODE else logging.INFO, handlers=logging_handlers diff --git a/bot/__main__.py b/bot/__main__.py index ab212a8a1..b3f80ef55 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -8,7 +8,6 @@ from discord.ext.commands import Bot, when_mentioned_or from bot.api import APIClient from bot.constants import Bot as BotConfig, DEBUG_MODE -from bot.utils.service_discovery import wait_for_rmq log = logging.getLogger(__name__) @@ -31,14 +30,6 @@ bot.http_session = ClientSession( ) bot.api_client = APIClient(loop=asyncio.get_event_loop()) -log.info("Waiting for RabbitMQ...") -has_rmq = wait_for_rmq() - -if has_rmq: - log.info("RabbitMQ found") -else: - log.warning("Timed out while waiting for RabbitMQ") - # Internal/debug bot.load_extension("bot.cogs.filtering") bot.load_extension("bot.cogs.logging") @@ -47,7 +38,6 @@ bot.load_extension("bot.cogs.security") # Commands, etc bot.load_extension("bot.cogs.antispam") -bot.load_extension("bot.cogs.bigbrother") bot.load_extension("bot.cogs.bot") bot.load_extension("bot.cogs.clean") bot.load_extension("bot.cogs.cogs") @@ -78,11 +68,9 @@ bot.load_extension("bot.cogs.sync") bot.load_extension("bot.cogs.tags") bot.load_extension("bot.cogs.token_remover") bot.load_extension("bot.cogs.utils") +bot.load_extension("bot.cogs.watchchannels") bot.load_extension("bot.cogs.wolfram") -if has_rmq: - bot.load_extension("bot.cogs.rmq") - bot.run(BotConfig.token) bot.http_session.close() # Close the aiohttp session when the bot finishes running diff --git a/bot/cogs/alias.py b/bot/cogs/alias.py index 02f92a79d..85d101448 100644 --- a/bot/cogs/alias.py +++ b/bot/cogs/alias.py @@ -1,11 +1,13 @@ import inspect import logging +from typing import Union -from discord import Colour, Embed, User +from discord import Colour, Embed, Member, User from discord.ext.commands import ( Command, Context, clean_content, command, group ) +from bot.cogs.watchchannels.watchchannel import proxy_user from bot.converters import TagNameConverter from bot.pagination import LinePaginator @@ -70,9 +72,7 @@ class Alias: await self.invoke(ctx, "site resources") @command(name="watch", hidden=True) - async def bigbrother_watch_alias( - self, ctx: Context, user: User, *, reason: str - ): + async def bigbrother_watch_alias(self, ctx, user: Union[Member, User, proxy_user], *, reason: str): """ Alias for invoking <prefix>bigbrother watch [user] [reason]. """ @@ -80,7 +80,7 @@ class Alias: await self.invoke(ctx, "bigbrother watch", user, reason=reason) @command(name="unwatch", hidden=True) - async def bigbrother_unwatch_alias(self, ctx, user: User, *, reason: str): + async def bigbrother_unwatch_alias(self, ctx, user: Union[User, proxy_user], *, reason: str): """ Alias for invoking <prefix>bigbrother unwatch [user] [reason]. """ @@ -181,6 +181,22 @@ class Alias: await self.invoke(ctx, "docs get", symbol) + @command(name="nominate", hidden=True) + async def nomination_add_alias(self, ctx, user: Union[Member, User, proxy_user], *, reason: str): + """ + Alias for invoking <prefix>talentpool add [user] [reason]. + """ + + await self.invoke(ctx, "talentpool add", user, reason=reason) + + @command(name="unnominate", hidden=True) + async def nomination_end_alias(self, ctx, user: Union[User, proxy_user], *, reason: str): + """ + Alias for invoking <prefix>nomination end [user] [reason]. + """ + + await self.invoke(ctx, "nomination end", user, reason=reason) + def setup(bot): bot.add_cog(Alias(bot)) diff --git a/bot/cogs/bigbrother.py b/bot/cogs/bigbrother.py deleted file mode 100644 index df7a0b576..000000000 --- a/bot/cogs/bigbrother.py +++ /dev/null @@ -1,258 +0,0 @@ -import asyncio -import logging -import re -from collections import defaultdict, deque -from typing import List, Union - -from discord import Color, Embed, Guild, Member, Message, User -from discord.ext.commands import Bot, Context, group - -from bot.constants import ( - BigBrother as BigBrotherConfig, Channels, Emojis, Guild as GuildConfig, Roles -) -from bot.decorators import with_role -from bot.pagination import LinePaginator -from bot.utils import messages -from bot.utils.moderation import post_infraction - -log = logging.getLogger(__name__) - -URL_RE = re.compile(r"(https?://[^\s]+)") - - -class BigBrother: - """User monitoring to assist with moderation.""" - - def __init__(self, bot: Bot): - self.bot = bot - self.watched_users = set() # { user_id } - self.channel_queues = defaultdict(lambda: defaultdict(deque)) # { user_id: { channel_id: queue(messages) } - self.last_log = [None, None, 0] # [user_id, channel_id, message_count] - self.consuming = False - - def update_cache(self, api_response: List[dict]): - """ - Updates the internal cache of watched users from the given `api_response`. - This function will only add (or update) existing keys, it will not delete - keys that were not present in the API response. - A user is only added if the bot can find a channel - with the given `channel_id` in its channel cache. - """ - - for entry in api_response: - user_id = entry['user'] - self.watched_users.add(user_id) - - async def on_ready(self): - """Retrieves watched users from the API.""" - - self.channel = self.bot.get_channel(Channels.big_brother_logs) - if self.channel is None: - log.error("Cannot find Big Brother channel. Cannot watch users.") - else: - data = await self.bot.api_client.get( - 'bot/infractions', - params={ - 'active': 'true', - 'type': 'watch' - } - ) - self.update_cache(data) - - async def on_member_ban(self, guild: Guild, user: Union[User, Member]): - if guild.id == GuildConfig.id and user.id in self.watched_users: - [active_watch] = await self.bot.api_client.get( - 'bot/infractions', - params={ - 'active': 'true', - 'type': 'watch', - 'user__id': str(user.id) - } - ) - await self.bot.api_client.put( - 'bot/infractions/' + str(active_watch['id']), - json={'active': False} - ) - self.watched_users.remove(user.id) - del self.channel_queues[user.id] - await self.channel.send( - f"{Emojis.bb_message}:hammer: {user} got banned, so " - f"`BigBrother` will no longer relay their messages." - ) - - async def on_message(self, msg: Message): - """Queues up messages sent by watched users.""" - - if msg.author.id in self.watched_users: - if not self.consuming: - self.bot.loop.create_task(self.consume_messages()) - - log.trace(f"Received message: {msg.content} ({len(msg.attachments)} attachments)") - self.channel_queues[msg.author.id][msg.channel.id].append(msg) - - async def consume_messages(self): - """Consumes the message queues to log watched users' messages.""" - - if not self.consuming: - self.consuming = True - log.trace("Sleeping before consuming...") - await asyncio.sleep(BigBrotherConfig.log_delay) - - log.trace("Begin consuming messages.") - channel_queues = self.channel_queues.copy() - self.channel_queues.clear() - for _, queues in channel_queues.items(): - for queue in queues.values(): - while queue: - msg = queue.popleft() - log.trace(f"Consuming message: {msg.clean_content} ({len(msg.attachments)} attachments)") - - self.last_log[2] += 1 # Increment message count. - await self.send_header(msg) - await self.log_message(msg) - - if self.channel_queues: - log.trace("Queue not empty; continue consumption.") - self.bot.loop.create_task(self.consume_messages()) - else: - log.trace("Done consuming messages.") - self.consuming = False - - async def send_header(self, message: Message): - """ - Sends a log message header to the given channel. - - A header is only sent if the user or channel are different than the previous, or if the configured message - limit for a single header has been exceeded. - - :param message: the first message in the queue - """ - - last_user, last_channel, msg_count = self.last_log - limit = BigBrotherConfig.header_message_limit - - # Send header if user/channel are different or if message limit exceeded. - if message.author.id != last_user or message.channel.id != last_channel or msg_count > limit: - self.last_log = [message.author.id, message.channel.id, 0] - - embed = Embed(description=f"{message.author.mention} in [#{message.channel.name}]({message.jump_url})") - embed.set_author(name=message.author.nick or message.author.name, icon_url=message.author.avatar_url) - await self.channel.send(embed=embed) - - async def log_message(self, message: Message): - """ - Logs a watched user's message in the given channel. - - Attachments are also sent. All non-image or non-video URLs are put in inline code blocks to prevent preview - embeds from being automatically generated. - - :param message: the message to log - """ - - content = message.clean_content - if content: - # Put all non-media URLs in inline code blocks. - media_urls = {embed.url for embed in message.embeds if embed.type in ("image", "video")} - for url in URL_RE.findall(content): - if url not in media_urls: - content = content.replace(url, f"`{url}`") - - await self.channel.send(content) - - await messages.send_attachments(message, self.channel) - - @group(name='bigbrother', aliases=('bb',), invoke_without_command=True) - @with_role(Roles.owner, Roles.admin, Roles.moderator) - async def bigbrother_group(self, ctx: Context): - """Monitor users, NSA-style.""" - - await ctx.invoke(self.bot.get_command("help"), "bigbrother") - - @bigbrother_group.command(name='watched', aliases=('all',)) - @with_role(Roles.owner, Roles.admin, Roles.moderator) - async def watched_command(self, ctx: Context, from_cache: bool = True): - """ - Shows all users that are currently monitored and in which channel. - By default, the users are returned from the cache. - If this is not desired, `from_cache` can be given as a falsy value, e.g. e.g. 'no'. - """ - - if from_cache: - lines = tuple(f"• <@{user_id}>" for user_id in self.watched_users) - await LinePaginator.paginate( - lines or ("There's nothing here yet.",), - ctx, - Embed(title="Watched users (cached)", color=Color.blue()), - empty=False - ) - - else: - active_watches = await self.bot.api_client.get( - 'bot/infractions', - params={ - 'active': 'true', - 'type': 'watch' - } - ) - self.update_cache(active_watches) - lines = tuple( - f"• <@{entry['user']}>: {entry['reason'] or '*no reason provided*'}" - for entry in active_watches - ) - - await LinePaginator.paginate( - lines or ("There's nothing here yet.",), - ctx, - Embed(title="Watched users", color=Color.blue()), - empty=False - ) - - @bigbrother_group.command(name='watch', aliases=('w',)) - @with_role(Roles.owner, Roles.admin, Roles.moderator) - async def watch_command(self, ctx: Context, user: User, *, reason: str): - """ - Relay messages sent by the given `user` to the `#big-brother-logs` channel - - A `reason` for watching is required, which is added for the user to be watched as a - note (aka: shadow warning) - """ - - if user.id in self.watched_users: - return await ctx.send(":x: That user is already watched.") - - await post_infraction( - ctx, user, type='watch', reason=reason, hidden=True - ) - self.watched_users.add(user.id) - await ctx.send(f":ok_hand: will now relay messages sent by {user}") - - @bigbrother_group.command(name='unwatch', aliases=('uw',)) - @with_role(Roles.owner, Roles.admin, Roles.moderator) - async def unwatch_command(self, ctx: Context, user: User): - """Stop relaying messages by the given `user`.""" - - active_watches = await self.bot.api_client.get( - 'bot/infractions', - params={ - 'active': 'true', - 'type': 'watch', - 'user__id': str(user.id) - } - ) - if active_watches: - [infraction] = active_watches - await self.bot.api_client.patch( - 'bot/infractions/' + str(infraction['id']), - json={'active': False} - ) - await ctx.send(f":ok_hand: will no longer relay messages sent by {user}") - self.watched_users.remove(user.id) - if user.id in self.channel_queues: - del self.channel_queues[user.id] - else: - await ctx.send(":x: that user is currently not being watched") - - -def setup(bot: Bot): - bot.add_cog(BigBrother(bot)) - log.info("Cog loaded: BigBrother") diff --git a/bot/cogs/nominations.py b/bot/cogs/nominations.py deleted file mode 100644 index 93ee0d885..000000000 --- a/bot/cogs/nominations.py +++ /dev/null @@ -1,120 +0,0 @@ -import logging - -from discord import Color, Embed, User -from discord.ext.commands import Context, group - -from bot.cogs.bigbrother import BigBrother, Roles -from bot.constants import Channels -from bot.decorators import with_role -from bot.pagination import LinePaginator - - -log = logging.getLogger(__name__) - - -class Nominations(BigBrother): - """Monitor potential helpers, NSA-style.""" - - async def on_ready(self): - """Retrieve nominees from the API.""" - - self.channel = self.bot.get_channel(Channels.talent_pool) - if self.channel is None: - log.error("Cannot find talent pool channel. Cannot watch nominees.") - else: - nominations = await self.bot.api_client.get( - 'bot/nominations', - params={'active': 'true'} - ) - self.update_cache(nominations) - - async def on_member_ban(self, *_): - pass - - @group(name='nominations', aliases=('n',), invoke_without_command=True) - @with_role(Roles.owner, Roles.admin, Roles.moderator) - async def bigbrother_group(self, ctx: Context): - """Nominate helpers, NSA-style.""" - - await ctx.invoke(self.bot.get_command("help"), "nominations") - - @bigbrother_group.command(name='nominated', aliases=('nominees', 'all')) - @with_role(Roles.owner, Roles.admin, Roles.moderator) - async def watched_command(self, ctx: Context, from_cache: bool = True): - if from_cache: - lines = tuple(f"• <@{user_id}>" for user_id in self.watched_users) - - else: - active_nominations = await self.bot.api_client.get( - 'bot/nominations', - params={'active': 'true'} - ) - self.update_cache(active_nominations) - lines = tuple( - f"• <@{entry['user']}>: {entry['reason'] or '*no reason provided*'}" - for entry in active_nominations - ) - - await LinePaginator.paginate( - lines or ("There's nothing here yet.",), - ctx, - Embed( - title="Nominated users" + " (cached)" * from_cache, - color=Color.blue() - ), - empty=False - ) - - @bigbrother_group.command(name='nominate', aliases=('n',)) - @with_role(Roles.owner, Roles.admin, Roles.moderator) - async def watch_command(self, ctx: Context, user: User, *, reason: str): - """Talent pool the given `user`.""" - - active_nominations = await self.bot.api_client.get( - 'bot/nominations/' + str(user.id), - ) - if active_nominations: - active_nominations = await self.bot.api_client.put( - 'bot/nominations/' + str(user.id), - json={'active': True} - ) - await ctx.send(":ok_hand: user's watch was updated") - - else: - active_nominations = await self.bot.api_client.post( - 'bot/nominations/' + str(user.id), - json={ - 'active': True, - 'author': ctx.author.id, - 'reason': reason, - } - ) - self.watched_users.add(user.id) - await ctx.send(":ok_hand: user added to talent pool") - - @bigbrother_group.command(name='unnominate', aliases=('un',)) - @with_role(Roles.owner, Roles.admin, Roles.moderator) - async def unwatch_command(self, ctx: Context, user: User): - """Stop talent pooling the given `user`.""" - - nomination = await self.bot.api_client.get( - 'bot/nominations/' + str(user.id) - ) - - if not nomination['active']: - await ctx.send(":x: the nomination is already inactive") - - else: - await self.bot.api_client.put( - 'bot/nominations/' + str(user.id), - json={'active': False} - ) - self.watched_users.remove(user.id) - if user.id in self.channel_queues: - del self.channel_queues[user.id] - await ctx.send(f":ok_hand: {user} is no longer part of the talent pool") - - -def setup(bot): - bot.add_cog(Nominations(bot)) - log.info("Cog loaded: Nominations") diff --git a/bot/cogs/rmq.py b/bot/cogs/rmq.py deleted file mode 100644 index 585eacc25..000000000 --- a/bot/cogs/rmq.py +++ /dev/null @@ -1,229 +0,0 @@ -import asyncio -import datetime -import json -import logging -import pprint - -import aio_pika -from aio_pika import Message -from dateutil import parser as date_parser -from discord import Colour, Embed -from discord.ext.commands import Bot -from discord.utils import get - -from bot.constants import Channels, Guild, RabbitMQ - -log = logging.getLogger(__name__) - -LEVEL_COLOURS = { - "debug": Colour.blue(), - "info": Colour.green(), - "warning": Colour.gold(), - "error": Colour.red() -} - -DEFAULT_LEVEL_COLOUR = Colour.greyple() -EMBED_PARAMS = ( - "colour", "title", "url", "description", "timestamp" -) - -CONSUME_TIMEOUT = datetime.timedelta(seconds=10) - - -class RMQ: - """ - RabbitMQ event handling - """ - - rmq = None # type: aio_pika.Connection - channel = None # type: aio_pika.Channel - queue = None # type: aio_pika.Queue - - def __init__(self, bot: Bot): - self.bot = bot - - async def on_ready(self): - self.rmq = await aio_pika.connect_robust( - host=RabbitMQ.host, port=RabbitMQ.port, login=RabbitMQ.username, password=RabbitMQ.password - ) - - log.info("Connected to RabbitMQ") - - self.channel = await self.rmq.channel() - self.queue = await self.channel.declare_queue("bot_events", durable=True) - - log.debug("Channel opened, queue declared") - - async for message in self.queue: - with message.process(): - message.ack() - await self.handle_message(message, message.body.decode()) - - async def send_text(self, queue: str, data: str): - message = Message(data.encode("utf-8")) - await self.channel.default_exchange.publish(message, queue) - - async def send_json(self, queue: str, **data): - message = Message(json.dumps(data).encode("utf-8")) - await self.channel.default_exchange.publish(message, queue) - - async def consume(self, queue: str, **kwargs): - queue_obj = await self.channel.declare_queue(queue, **kwargs) - - result = None - start_time = datetime.datetime.now() - - while result is None: - if datetime.datetime.now() - start_time >= CONSUME_TIMEOUT: - result = "Timed out while waiting for a response." - else: - result = await queue_obj.get(timeout=5, fail=False) - await asyncio.sleep(0.5) - - if result: - result.ack() - - return result - - async def handle_message(self, message, data): - log.debug(f"Message: {message}") - log.debug(f"Data: {data}") - - try: - data = json.loads(data) - except Exception: - await self.do_mod_log("error", "Unable to parse event", data) - else: - event = data["event"] - event_data = data["data"] - - try: - func = getattr(self, f"do_{event}") - await func(**event_data) - except Exception as e: - await self.do_mod_log( - "error", f"Unable to handle event: {event}", - str(e) - ) - - async def do_mod_log(self, level: str, title: str, message: str): - colour = LEVEL_COLOURS.get(level, DEFAULT_LEVEL_COLOUR) - embed = Embed( - title=title, description=f"```\n{message}\n```", - colour=colour, timestamp=datetime.datetime.utcnow() - ) - - await self.bot.get_channel(Channels.devlog).send(embed=embed) - log.log(logging._nameToLevel[level.upper()], f"Modlog: {title} | {message}") - - async def do_send_message(self, target: int, message: str): - channel = self.bot.get_channel(target) - - if channel is None: - await self.do_mod_log( - "error", "Failed: Send Message", - f"Unable to find channel: {target}" - ) - else: - await channel.send(message) - - await self.do_mod_log( - "info", "Succeeded: Send Embed", - f"Message sent to channel {target}\n\n{message}" - ) - - async def do_send_embed(self, target: int, **embed_params): - for param, value in list(embed_params.items()): # To keep a full copy - if param not in EMBED_PARAMS: - await self.do_mod_log( - "warning", "Warning: Send Embed", - f"Unknown embed parameter: {param}" - ) - del embed_params[param] - - if param == "timestamp": - embed_params[param] = date_parser.parse(value) - elif param == "colour": - embed_params[param] = Colour(value) - - channel = self.bot.get_channel(target) - - if channel is None: - await self.do_mod_log( - "error", "Failed: Send Embed", - f"Unable to find channel: {target}" - ) - else: - await channel.send(embed=Embed(**embed_params)) - - await self.do_mod_log( - "info", "Succeeded: Send Embed", - f"Embed sent to channel {target}\n\n{pprint.pformat(embed_params, 4)}" - ) - - async def do_add_role(self, target: int, role_id: int, reason: str): - guild = self.bot.get_guild(Guild.id) - member = guild.get_member(int(target)) - - if member is None: - return await self.do_mod_log( - "error", "Failed: Add Role", - f"Unable to find member: {target}" - ) - - role = get(guild.roles, id=int(role_id)) - - if role is None: - return await self.do_mod_log( - "error", "Failed: Add Role", - f"Unable to find role: {role_id}" - ) - - try: - await member.add_roles(role, reason=reason) - except Exception as e: - await self.do_mod_log( - "error", "Failed: Add Role", - f"Error while adding role {role.name}: {e}" - ) - else: - await self.do_mod_log( - "info", "Succeeded: Add Role", - f"Role {role.name} added to member {target}" - ) - - async def do_remove_role(self, target: int, role_id: int, reason: str): - guild = self.bot.get_guild(Guild.id) - member = guild.get_member(int(target)) - - if member is None: - return await self.do_mod_log( - "error", "Failed: Remove Role", - f"Unable to find member: {target}" - ) - - role = get(guild.roles, id=int(role_id)) - - if role is None: - return await self.do_mod_log( - "error", "Failed: Remove Role", - f"Unable to find role: {role_id}" - ) - - try: - await member.remove_roles(role, reason=reason) - except Exception as e: - await self.do_mod_log( - "error", "Failed: Remove Role", - f"Error while adding role {role.name}: {e}" - ) - else: - await self.do_mod_log( - "info", "Succeeded: Remove Role", - f"Role {role.name} removed from member {target}" - ) - - -def setup(bot): - bot.add_cog(RMQ(bot)) - log.info("Cog loaded: RMQ") diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index cc18c0041..05834e421 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -3,13 +3,14 @@ import logging import random import re import textwrap +from signal import Signals +from typing import Optional, Tuple from discord import Colour, Embed from discord.ext.commands import ( Bot, CommandError, Context, NoPrivateMessage, command, guild_only ) -from bot.cogs.rmq import RMQ from bot.constants import Channels, ERROR_REPLIES, NEGATIVE_REPLIES, STAFF_ROLES, URLs from bot.decorators import InChannelCheckFailure, in_channel from bot.utils.messages import wait_for_deletion @@ -17,23 +18,6 @@ from bot.utils.messages import wait_for_deletion log = logging.getLogger(__name__) -RMQ_ARGS = { - "durable": False, - "arguments": {"x-message-ttl": 5000}, - "auto_delete": True -} - -CODE_TEMPLATE = """ -venv_file = "/snekbox/.venv/bin/activate_this.py" -exec(open(venv_file).read(), dict(__file__=venv_file)) - -try: -{CODE} -except: - import traceback - print(traceback.format_exc()) -""" - ESCAPE_REGEX = re.compile("[`\u202E\u200B]{3,}") FORMATTED_CODE_REGEX = re.compile( r"^\s*" # any leading whitespace from the beginning of the string @@ -53,42 +37,47 @@ RAW_CODE_REGEX = re.compile( re.DOTALL # "." also matches newlines ) +MAX_PASTE_LEN = 1000 + class Snekbox: """ - Safe evaluation using Snekbox + Safe evaluation of Python code using Snekbox """ def __init__(self, bot: Bot): self.bot = bot self.jobs = {} - @property - def rmq(self) -> RMQ: - return self.bot.get_cog("RMQ") - - @command(name='eval', aliases=('e',)) - @guild_only() - @in_channel(Channels.bot, bypass_roles=STAFF_ROLES) - async def eval_command(self, ctx: Context, *, code: str = None): - """ - Run some code. get the result back. We've done our best to make this safe, but do let us know if you - manage to find an issue with it! - - This command supports multiple lines of code, including code wrapped inside a formatted code block. - """ + async def post_eval(self, code: str) -> dict: + """Send a POST request to the Snekbox API to evaluate code and return the results.""" + url = URLs.snekbox_eval_api + data = {"input": code} + async with self.bot.http_session.post(url, json=data, raise_for_status=True) as resp: + return await resp.json() - if ctx.author.id in self.jobs: - await ctx.send(f"{ctx.author.mention} You've already got a job running - please wait for it to finish!") - return - - if not code: # None or empty string - return await ctx.invoke(self.bot.get_command("help"), "eval") + async def upload_output(self, output: str) -> Optional[str]: + """Upload the eval output to a paste service and return a URL to it if successful.""" + log.trace("Uploading full output to paste service...") - log.info(f"Received code from {ctx.author.name}#{ctx.author.discriminator} for evaluation:\n{code}") - self.jobs[ctx.author.id] = datetime.datetime.now() + if len(output) > MAX_PASTE_LEN: + log.info("Full output is too long to upload") + return "too long to upload" - # Strip whitespace and inline or block code markdown and extract the code and some formatting info + url = URLs.paste_service.format(key="documents") + try: + async with self.bot.http_session.post(url, data=output, raise_for_status=True) as resp: + data = await resp.json() + + if "key" in data: + return URLs.paste_service.format(key=data["key"]) + except Exception: + # 400 (Bad Request) means there are too many characters + log.exception("Failed to upload full output to paste service!") + + @staticmethod + def prepare_input(code: str) -> str: + """Extract code from the Markdown, format it, and insert it into the code template.""" match = FORMATTED_CODE_REGEX.fullmatch(code) if match: code, block, lang, delim = match.group("code", "block", "lang", "delim") @@ -100,87 +89,138 @@ class Snekbox: log.trace(f"Extracted {info} for evaluation:\n{code}") else: code = textwrap.dedent(RAW_CODE_REGEX.fullmatch(code).group("code")) - log.trace(f"Eval message contains not or badly formatted code, stripping whitespace only:\n{code}") + log.trace( + f"Eval message contains unformatted or badly formatted code, " + f"stripping whitespace only:\n{code}" + ) - try: - stripped_lines = [ln.strip() for ln in code.split('\n')] - if all(line.startswith('#') for line in stripped_lines): - return await ctx.send( - f"{ctx.author.mention} Your eval job has completed.\n\n```[No output]```" - ) + return code + + @staticmethod + def get_results_message(results: dict) -> Tuple[str, str]: + """Return a user-friendly message and error corresponding to the process's return code.""" + stdout, returncode = results["stdout"], results["returncode"] + msg = f"Your eval job has completed with return code {returncode}" + error = "" + + if returncode is None: + msg = "Your eval job has failed" + error = stdout.strip() + elif returncode == 128 + Signals.SIGKILL: + msg = "Your eval job timed out or ran out of memory" + elif returncode == 255: + msg = "Your eval job has failed" + error = "A fatal NsJail error occurred" + else: + # Try to append signal's name if one exists + try: + name = Signals(returncode - 128).name + msg = f"{msg} ({name})" + except ValueError: + pass + + return msg, error + + async def format_output(self, output: str) -> Tuple[str, Optional[str]]: + """ + Format the output and return a tuple of the formatted output and a URL to the full output. + + Prepend each line with a line number. Truncate if there are over 10 lines or 1000 characters + and upload the full output to a paste service. + """ + log.trace("Formatting output...") + + output = output.strip(" \n") + original_output = output # To be uploaded to a pasting service if needed + paste_link = None + + if "<@" in output: + output = output.replace("<@", "<@\u200B") # Zero-width space - code = textwrap.indent(code, " ") - code = CODE_TEMPLATE.replace("{CODE}", code) + if "<!@" in output: + output = output.replace("<!@", "<!@\u200B") # Zero-width space - await self.rmq.send_json( - "input", - snekid=str(ctx.author.id), message=code + if ESCAPE_REGEX.findall(output): + return "Code block escape attempt detected; will not output result", paste_link + + truncated = False + lines = output.count("\n") + + if lines > 0: + output = output.split("\n")[:10] # Only first 10 cause the rest is truncated anyway + output = (f"{i:03d} | {line}" for i, line in enumerate(output, 1)) + output = "\n".join(output) + + if lines > 10: + truncated = True + if len(output) >= 1000: + output = f"{output[:1000]}\n... (truncated - too long, too many lines)" + else: + output = f"{output}\n... (truncated - too many lines)" + elif len(output) >= 1000: + truncated = True + output = f"{output[:1000]}\n... (truncated - too long)" + + if truncated: + paste_link = await self.upload_output(original_output) + + output = output.strip() + if not output: + output = "[No output]" + + return output, paste_link + + @command(name="eval", aliases=("e",)) + @guild_only() + @in_channel(Channels.bot, bypass_roles=STAFF_ROLES) + async def eval_command(self, ctx: Context, *, code: str = None): + """ + Run Python code and get the results. + + This command supports multiple lines of code, including code wrapped inside a formatted code + block. We've done our best to make this safe, but do let us know if you manage to find an + issue with it! + """ + if ctx.author.id in self.jobs: + return await ctx.send( + f"{ctx.author.mention} You've already got a job running - " + f"please wait for it to finish!" ) - async with ctx.typing(): - message = await self.rmq.consume(str(ctx.author.id), **RMQ_ARGS) - paste_link = None + if not code: # None or empty string + return await ctx.invoke(self.bot.get_command("help"), "eval") - if isinstance(message, str): - output = str.strip(" \n") - else: - output = message.body.decode().strip(" \n") + log.info( + f"Received code from {ctx.author.name}#{ctx.author.discriminator} " + f"for evaluation:\n{code}" + ) - if "<@" in output: - output = output.replace("<@", "<@\u200B") # Zero-width space + self.jobs[ctx.author.id] = datetime.datetime.now() + code = self.prepare_input(code) - if "<!@" in output: - output = output.replace("<!@", "<!@\u200B") # Zero-width space + try: + async with ctx.typing(): + results = await self.post_eval(code) + msg, error = self.get_results_message(results) - if ESCAPE_REGEX.findall(output): - output = "Code block escape attempt detected; will not output result" + if error: + output, paste_link = error, None else: - # the original output, to send to a pasting service if needed - full_output = output - truncated = False - if output.count("\n") > 0: - output = [f"{i:03d} | {line}" for i, line in enumerate(output.split("\n"), start=1)] - output = "\n".join(output) - - if output.count("\n") > 10: - output = "\n".join(output.split("\n")[:10]) - - if len(output) >= 1000: - output = f"{output[:1000]}\n... (truncated - too long, too many lines)" - else: - output = f"{output}\n... (truncated - too many lines)" - truncated = True - - elif len(output) >= 1000: - output = f"{output[:1000]}\n... (truncated - too long)" - truncated = True - - if truncated: - try: - response = await self.bot.http_session.post( - URLs.paste_service.format(key="documents"), - data=full_output - ) - data = await response.json() - if "key" in data: - paste_link = URLs.paste_service.format(key=data["key"]) - except Exception: - log.exception("Failed to upload full output to paste service!") - - if output.strip(): - if paste_link: - msg = f"{ctx.author.mention} Your eval job has completed.\n\n```py\n{output}\n```" \ - f"\nFull output: {paste_link}" - else: - msg = f"{ctx.author.mention} Your eval job has completed.\n\n```py\n{output}\n```" - - response = await ctx.send(msg) - self.bot.loop.create_task(wait_for_deletion(response, user_ids=(ctx.author.id,), client=ctx.bot)) + output, paste_link = await self.format_output(results["stdout"]) - else: - await ctx.send( - f"{ctx.author.mention} Your eval job has completed.\n\n```[No output]```" - ) + msg = f"{ctx.author.mention} {msg}.\n\n```py\n{output}\n```" + if paste_link: + msg = f"{msg}\nFull output: {paste_link}" + + response = await ctx.send(msg) + self.bot.loop.create_task( + wait_for_deletion(response, user_ids=(ctx.author.id,), client=ctx.bot) + ) + + log.info( + f"{ctx.author.name}#{ctx.author.discriminator}'s job had a return code of " + f"{results['returncode']}" + ) finally: del self.jobs[ctx.author.id] diff --git a/bot/cogs/watchchannels/__init__.py b/bot/cogs/watchchannels/__init__.py new file mode 100644 index 000000000..ac7713803 --- /dev/null +++ b/bot/cogs/watchchannels/__init__.py @@ -0,0 +1,15 @@ +import logging + +from .bigbrother import BigBrother +from .talentpool import TalentPool + + +log = logging.getLogger(__name__) + + +def setup(bot): + bot.add_cog(BigBrother(bot)) + log.info("Cog loaded: BigBrother") + + bot.add_cog(TalentPool(bot)) + log.info("Cog loaded: TalentPool") diff --git a/bot/cogs/watchchannels/bigbrother.py b/bot/cogs/watchchannels/bigbrother.py new file mode 100644 index 000000000..e7b3d70bc --- /dev/null +++ b/bot/cogs/watchchannels/bigbrother.py @@ -0,0 +1,100 @@ +import logging +from collections import ChainMap +from typing import Union + +from discord import User +from discord.ext.commands import Context, group + +from bot.constants import Channels, Roles, Webhooks +from bot.decorators import with_role +from bot.utils.moderation import post_infraction +from .watchchannel import WatchChannel, proxy_user + +log = logging.getLogger(__name__) + + +class BigBrother(WatchChannel): + """Monitors users by relaying their messages to a watch channel to assist with moderation.""" + + def __init__(self, bot) -> None: + super().__init__( + bot, + destination=Channels.big_brother_logs, + webhook_id=Webhooks.big_brother, + api_endpoint='bot/infractions', + api_default_params={'active': 'true', 'type': 'watch', 'ordering': '-inserted_at'}, + logger=log + ) + + @group(name='bigbrother', aliases=('bb',), invoke_without_command=True) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def bigbrother_group(self, ctx: Context) -> None: + """Monitors users by relaying their messages to the Big Brother watch channel.""" + await ctx.invoke(self.bot.get_command("help"), "bigbrother") + + @bigbrother_group.command(name='watched', aliases=('all', 'list')) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def watched_command(self, ctx: Context, update_cache: bool = True) -> None: + """ + Shows the users that are currently being monitored by Big Brother. + + The optional kwarg `update_cache` can be used to update the user + cache using the API before listing the users. + """ + await self.list_watched_users(ctx, update_cache) + + @bigbrother_group.command(name='watch', aliases=('w',)) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def watch_command(self, ctx: Context, user: Union[User, proxy_user], *, reason: str) -> None: + """ + Relay messages sent by the given `user` to the `#big-brother` channel. + + A `reason` for adding the user to Big Brother is required and will be displayed + in the header when relaying messages of this user to the watchchannel. + """ + if user.bot: + await ctx.send(f":x: I'm sorry {ctx.author}, I'm afraid I can't do that. I only watch humans.") + return + + if not await self.fetch_user_cache(): + await ctx.send(f":x: Updating the user cache failed, can't watch user {user}") + return + + if user.id in self.watched_users: + await ctx.send(":x: The specified user is already being watched.") + return + + response = await post_infraction( + ctx, user, type='watch', reason=reason, hidden=True + ) + + if response is not None: + self.watched_users[user.id] = response + await ctx.send(f":white_check_mark: Messages sent by {user} will now be relayed to Big Brother.") + + @bigbrother_group.command(name='unwatch', aliases=('uw',)) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def unwatch_command(self, ctx: Context, user: Union[User, proxy_user], *, reason: str) -> None: + """Stop relaying messages by the given `user`.""" + active_watches = await self.bot.api_client.get( + self.api_endpoint, + params=ChainMap( + self.api_default_params, + {"user__id": str(user.id)} + ) + ) + if active_watches: + [infraction] = active_watches + + await self.bot.api_client.patch( + f"{self.api_endpoint}/{infraction['id']}", + json={'active': False} + ) + + await post_infraction(ctx, user, type='watch', reason=f"Unwatched: {reason}", hidden=True, active=False) + + await ctx.send(f":white_check_mark: Messages sent by {user} will no longer be relayed.") + + self._remove_user(user.id) + else: + await ctx.send(":x: The specified user is currently not being watched.") diff --git a/bot/cogs/watchchannels/talentpool.py b/bot/cogs/watchchannels/talentpool.py new file mode 100644 index 000000000..6fbe2bc03 --- /dev/null +++ b/bot/cogs/watchchannels/talentpool.py @@ -0,0 +1,233 @@ +import logging +import textwrap +from collections import ChainMap +from typing import Union + +from aiohttp.client_exceptions import ClientResponseError +from discord import Color, Embed, Member, User +from discord.ext.commands import Context, group + +from bot.constants import Channels, Guild, Roles, Webhooks +from bot.decorators import with_role +from bot.pagination import LinePaginator +from .watchchannel import WatchChannel, proxy_user + +log = logging.getLogger(__name__) +STAFF_ROLES = Roles.owner, Roles.admin, Roles.moderator, Roles.helpers # <- In constants after the merge? + + +class TalentPool(WatchChannel): + """Relays messages of helper candidates to a watch channel to observe them.""" + + def __init__(self, bot) -> None: + super().__init__( + bot, + destination=Channels.talent_pool, + webhook_id=Webhooks.talent_pool, + api_endpoint='bot/nominations', + api_default_params={'active': 'true', 'ordering': '-inserted_at'}, + logger=log, + ) + + @group(name='talentpool', aliases=('tp', 'talent', 'nomination', 'n'), invoke_without_command=True) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def nomination_group(self, ctx: Context) -> None: + """Highlights the activity of helper nominees by relaying their messages to the talent pool channel.""" + + await ctx.invoke(self.bot.get_command("help"), "talentpool") + + @nomination_group.command(name='watched', aliases=('all', 'list')) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def watched_command(self, ctx: Context, update_cache: bool = True) -> None: + """ + Shows the users that are currently being monitored in the talent pool. + + The optional kwarg `update_cache` can be used to update the user + cache using the API before listing the users. + """ + await self.list_watched_users(ctx, update_cache) + + @nomination_group.command(name='watch', aliases=('w', 'add', 'a')) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def watch_command(self, ctx: Context, user: Union[Member, User, proxy_user], *, reason: str) -> None: + """ + Relay messages sent by the given `user` to the `#talent-pool` channel. + + A `reason` for adding the user to the talent pool is required and will be displayed + in the header when relaying messages of this user to the channel. + """ + if user.bot: + await ctx.send(f":x: I'm sorry {ctx.author}, I'm afraid I can't do that. I only watch humans.") + return + + if isinstance(user, Member) and any(role.id in STAFF_ROLES for role in user.roles): + await ctx.send(f":x: Nominating staff members, eh? Here's a cookie :cookie:") + return + + if not await self.fetch_user_cache(): + await ctx.send(f":x: Failed to update the user cache; can't add {user}") + return + + if user.id in self.watched_users: + await ctx.send(":x: The specified user is already being watched in the talent pool") + return + + # Manual request with `raise_for_status` as False because we want the actual response + session = self.bot.api_client.session + url = self.bot.api_client._url_for(self.api_endpoint) + kwargs = { + 'json': { + 'actor': ctx.author.id, + 'reason': reason, + 'user': user.id + }, + 'raise_for_status': False, + } + async with session.post(url, **kwargs) as resp: + response_data = await resp.json() + + if resp.status == 400 and response_data.get('user', False): + await ctx.send(":x: The specified user can't be found in the database tables") + return + else: + resp.raise_for_status() + + self.watched_users[user.id] = response_data + await ctx.send(f":white_check_mark: Messages sent by {user} will now be relayed to the talent pool channel") + + @nomination_group.command(name='history', aliases=('info', 'search')) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def history_command(self, ctx: Context, user: Union[User, proxy_user]) -> None: + """Shows the specified user's nomination history.""" + result = await self.bot.api_client.get( + self.api_endpoint, + params={ + 'user__id': str(user.id), + 'ordering': "-active,-inserted_at" + } + ) + if not result: + await ctx.send(":warning: This user has never been nominated") + return + + embed = Embed( + title=f"Nominations for {user.display_name} `({user.id})`", + color=Color.blue() + ) + lines = [self._nomination_to_string(nomination) for nomination in result] + await LinePaginator.paginate( + lines, + ctx=ctx, + embed=embed, + empty=True, + max_lines=3, + max_size=1000 + ) + + @nomination_group.command(name='unwatch', aliases=('end', )) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def unwatch_command(self, ctx: Context, user: Union[User, proxy_user], *, reason: str) -> None: + """ + Ends the active nomination of the specified user with the given reason. + + Providing a `reason` is required. + """ + active_nomination = await self.bot.api_client.get( + self.api_endpoint, + params=ChainMap( + self.api_default_params, + {"user__id": str(user.id)} + ) + ) + + if not active_nomination: + await ctx.send(":x: The specified user does not have an active nomination") + return + + [nomination] = active_nomination + await self.bot.api_client.patch( + f"{self.api_endpoint}/{nomination['id']}", + json={'end_reason': reason, 'active': False} + ) + await ctx.send(f":white_check_mark: Messages sent by {user} will no longer be relayed") + self._remove_user(user.id) + + @nomination_group.group(name='edit', aliases=('e',), invoke_without_command=True) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def nomination_edit_group(self, ctx: Context) -> None: + """Commands to edit nominations.""" + + await ctx.invoke(self.bot.get_command("help"), "talentpool", "edit") + + @nomination_edit_group.command(name='reason') + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def edit_reason_command(self, ctx: Context, nomination_id: int, *, reason: str) -> None: + """ + Edits the reason/unnominate reason for the nomination with the given `id` depending on the status. + + If the nomination is active, the reason for nominating the user will be edited; + If the nomination is no longer active, the reason for ending the nomination will be edited instead. + """ + try: + nomination = await self.bot.api_client.get(f"{self.api_endpoint}/{nomination_id}") + except ClientResponseError as e: + if e.status == 404: + self.log.trace(f"Nomination API 404: Can't nomination with id {nomination_id}") + await ctx.send(f":x: Can't find a nomination with id `{nomination_id}`") + return + else: + raise + + field = "reason" if nomination["active"] else "end_reason" + + self.log.trace(f"Changing {field} for nomination with id {nomination_id} to {reason}") + + await self.bot.api_client.patch( + f"{self.api_endpoint}/{nomination_id}", + json={field: reason} + ) + + await ctx.send(f":white_check_mark: Updated the {field} of the nomination!") + + def _nomination_to_string(self, nomination_object: dict) -> str: + """Creates a string representation of a nomination.""" + guild = self.bot.get_guild(Guild.id) + + actor_id = nomination_object["actor"] + actor = guild.get_member(actor_id) + + active = nomination_object["active"] + log.debug(active) + log.debug(type(nomination_object["inserted_at"])) + + start_date = self._get_human_readable(nomination_object["inserted_at"]) + if active: + lines = textwrap.dedent( + f""" + =============== + Status: **Active** + Date: {start_date} + Actor: {actor.mention if actor else actor_id} + Reason: {nomination_object["reason"]} + Nomination ID: `{nomination_object["id"]}` + =============== + """ + ) + else: + end_date = self._get_human_readable(nomination_object["ended_at"]) + lines = textwrap.dedent( + f""" + =============== + Status: Inactive + Date: {start_date} + Actor: {actor.mention if actor else actor_id} + Reason: {nomination_object["reason"]} + + End date: {end_date} + Unwatch reason: {nomination_object["end_reason"]} + Nomination ID: `{nomination_object["id"]}` + =============== + """ + ) + + return lines.strip() diff --git a/bot/cogs/watchchannels/watchchannel.py b/bot/cogs/watchchannels/watchchannel.py new file mode 100644 index 000000000..fe6d6bb6e --- /dev/null +++ b/bot/cogs/watchchannels/watchchannel.py @@ -0,0 +1,353 @@ +import asyncio +import datetime +import logging +import re +import textwrap +from abc import ABC, abstractmethod +from collections import defaultdict, deque +from dataclasses import dataclass +from typing import Optional + +import aiohttp +import discord +from discord import Color, Embed, Message, Object, errors +from discord.ext.commands import BadArgument, Bot, Context + +from bot.cogs.modlog import ModLog +from bot.constants import BigBrother as BigBrotherConfig, Guild as GuildConfig, Icons +from bot.pagination import LinePaginator +from bot.utils import messages +from bot.utils.time import time_since + +log = logging.getLogger(__name__) + +URL_RE = re.compile(r"(https?://[^\s]+)") + + +def proxy_user(user_id: str) -> Object: + """A proxy user object that mocks a real User instance for when the later is not available.""" + try: + user_id = int(user_id) + except ValueError: + raise BadArgument + + user = Object(user_id) + user.mention = user.id + user.display_name = f"<@{user.id}>" + user.avatar_url_as = lambda static_format: None + user.bot = False + + return user + + +@dataclass +class MessageHistory: + last_author: Optional[int] = None + last_channel: Optional[int] = None + message_count: int = 0 + + +class WatchChannel(ABC): + """ABC with functionality for relaying users' messages to a certain channel.""" + + @abstractmethod + def __init__(self, bot: Bot, destination, webhook_id, api_endpoint, api_default_params, logger) -> None: + self.bot = bot + + self.destination = destination # E.g., Channels.big_brother_logs + self.webhook_id = webhook_id # E.g., Webhooks.big_brother + self.api_endpoint = api_endpoint # E.g., 'bot/infractions' + self.api_default_params = api_default_params # E.g., {'active': 'true', 'type': 'watch'} + self.log = logger # Logger of the child cog for a correct name in the logs + + self._consume_task = None + self.watched_users = defaultdict(dict) + self.message_queue = defaultdict(lambda: defaultdict(deque)) + self.consumption_queue = {} + self.retries = 5 + self.retry_delay = 10 + self.channel = None + self.webhook = None + self.message_history = MessageHistory() + + self._start = self.bot.loop.create_task(self.start_watchchannel()) + + @property + def modlog(self) -> ModLog: + """Provides access to the ModLog cog for alert purposes.""" + return self.bot.get_cog("ModLog") + + @property + def consuming_messages(self) -> bool: + """Checks if a consumption task is currently running.""" + if self._consume_task is None: + return False + + if self._consume_task.done(): + exc = self._consume_task.exception() + if exc: + self.log.exception( + f"The message queue consume task has failed with:", + exc_info=exc + ) + return False + + return True + + async def start_watchchannel(self) -> None: + """Starts the watch channel by getting the channel, webhook, and user cache ready.""" + await self.bot.wait_until_ready() + + # After updating d.py, this block can be replaced by `fetch_channel` with a try-except + for attempt in range(1, self.retries+1): + self.channel = self.bot.get_channel(self.destination) + if self.channel is None: + if attempt < self.retries: + await asyncio.sleep(self.retry_delay) + else: + break + else: + self.log.error(f"Failed to retrieve the text channel with id {self.destination}") + + # `get_webhook_info` has been renamed to `fetch_webhook` in newer versions of d.py + try: + self.webhook = await self.bot.get_webhook_info(self.webhook_id) + except (discord.HTTPException, discord.NotFound, discord.Forbidden): + self.log.exception(f"Failed to fetch webhook with id `{self.webhook_id}`") + + if self.channel is None or self.webhook is None: + self.log.error("Failed to start the watch channel; unloading the cog.") + + message = textwrap.dedent( + f""" + An error occurred while loading the text channel or webhook. + + TextChannel: {"**Failed to load**" if self.channel is None else "Loaded successfully"} + Webhook: {"**Failed to load**" if self.webhook is None else "Loaded successfully"} + + The Cog has been unloaded. + """ + ) + + await self.modlog.send_log_message( + title=f"Error: Failed to initialize the {self.__class__.__name__} watch channel", + text=message, + ping_everyone=True, + icon_url=Icons.token_removed, + colour=Color.red() + ) + + self.bot.remove_cog(self.__class__.__name__) + return + + if not await self.fetch_user_cache(): + await self.modlog.send_log_message( + title=f"Warning: Failed to retrieve user cache for the {self.__class__.__name__} watch channel", + text="Could not retrieve the list of watched users from the API and messages will not be relayed.", + ping_everyone=True, + icon=Icons.token_removed, + color=Color.red() + ) + + async def fetch_user_cache(self) -> bool: + """ + Fetches watched users from the API and updates the watched user cache accordingly. + + This function returns `True` if the update succeeded. + """ + try: + data = await self.bot.api_client.get(self.api_endpoint, params=self.api_default_params) + except aiohttp.ClientResponseError as e: + self.log.exception(f"Failed to fetch the watched users from the API", exc_info=e) + return False + + self.watched_users = defaultdict(dict) + + for entry in data: + user_id = entry.pop('user') + self.watched_users[user_id] = entry + + return True + + async def on_message(self, msg: Message) -> None: + """Queues up messages sent by watched users.""" + if msg.author.id in self.watched_users: + if not self.consuming_messages: + self._consume_task = self.bot.loop.create_task(self.consume_messages()) + + self.log.trace(f"Received message: {msg.content} ({len(msg.attachments)} attachments)") + self.message_queue[msg.author.id][msg.channel.id].append(msg) + + async def consume_messages(self, delay_consumption: bool = True) -> None: + """Consumes the message queues to log watched users' messages.""" + if delay_consumption: + self.log.trace(f"Sleeping {BigBrotherConfig.log_delay} seconds before consuming message queue") + await asyncio.sleep(BigBrotherConfig.log_delay) + + self.log.trace(f"Started consuming the message queue") + + # If the previous consumption Task failed, first consume the existing comsumption_queue + if not self.consumption_queue: + self.consumption_queue = self.message_queue.copy() + self.message_queue.clear() + + for user_channel_queues in self.consumption_queue.values(): + for channel_queue in user_channel_queues.values(): + while channel_queue: + msg = channel_queue.popleft() + + self.log.trace(f"Consuming message {msg.id} ({len(msg.attachments)} attachments)") + await self.relay_message(msg) + + self.consumption_queue.clear() + + if self.message_queue: + self.log.trace("Channel queue not empty: Continuing consuming queues") + self._consume_task = self.bot.loop.create_task(self.consume_messages(delay_consumption=False)) + else: + self.log.trace("Done consuming messages.") + + async def webhook_send( + self, + content: Optional[str] = None, + username: Optional[str] = None, + avatar_url: Optional[str] = None, + embed: Optional[Embed] = None, + ) -> None: + """Sends a message to the webhook with the specified kwargs.""" + try: + await self.webhook.send(content=content, username=username, avatar_url=avatar_url, embed=embed) + except discord.HTTPException as exc: + self.log.exception( + f"Failed to send a message to the webhook", + exc_info=exc + ) + + async def relay_message(self, msg: Message) -> None: + """Relays the message to the relevant watch channel.""" + limit = BigBrotherConfig.header_message_limit + + if ( + msg.author.id != self.message_history.last_author + or msg.channel.id != self.message_history.last_channel + or self.message_history.message_count >= limit + ): + self.message_history = MessageHistory(last_author=msg.author.id, last_channel=msg.channel.id) + + await self.send_header(msg) + + cleaned_content = msg.clean_content + + if cleaned_content: + # Put all non-media URLs in a code block to prevent embeds + media_urls = {embed.url for embed in msg.embeds if embed.type in ("image", "video")} + for url in URL_RE.findall(cleaned_content): + if url not in media_urls: + cleaned_content = cleaned_content.replace(url, f"`{url}`") + await self.webhook_send( + cleaned_content, + username=msg.author.display_name, + avatar_url=msg.author.avatar_url + ) + + if msg.attachments: + try: + await messages.send_attachments(msg, self.webhook) + except (errors.Forbidden, errors.NotFound): + e = Embed( + description=":x: **This message contained an attachment, but it could not be retrieved**", + color=Color.red() + ) + await self.webhook_send( + embed=e, + username=msg.author.display_name, + avatar_url=msg.author.avatar_url + ) + except discord.HTTPException as exc: + self.log.exception( + f"Failed to send an attachment to the webhook", + exc_info=exc + ) + + self.message_history.message_count += 1 + + async def send_header(self, msg) -> None: + """Sends a header embed with information about the relayed messages to the watch channel.""" + user_id = msg.author.id + + guild = self.bot.get_guild(GuildConfig.id) + actor = guild.get_member(self.watched_users[user_id]['actor']) + actor = actor.display_name if actor else self.watched_users[user_id]['actor'] + + inserted_at = self.watched_users[user_id]['inserted_at'] + time_delta = self._get_time_delta(inserted_at) + + reason = self.watched_users[user_id]['reason'] + + embed = Embed(description=f"{msg.author.mention} in [#{msg.channel.name}]({msg.jump_url})") + embed.set_footer(text=f"Added {time_delta} by {actor} | Reason: {reason}") + + await self.webhook_send(embed=embed, username=msg.author.display_name, avatar_url=msg.author.avatar_url) + + async def list_watched_users(self, ctx: Context, update_cache: bool = True) -> None: + """ + Gives an overview of the watched user list for this channel. + + The optional kwarg `update_cache` specifies whether the cache should + be refreshed by polling the API. + """ + if update_cache: + if not await self.fetch_user_cache(): + await ctx.send(f":x: Failed to update {self.__class__.__name__} user cache, serving from cache") + update_cache = False + + lines = [] + for user_id, user_data in self.watched_users.items(): + inserted_at = user_data['inserted_at'] + time_delta = self._get_time_delta(inserted_at) + lines.append(f"• <@{user_id}> (added {time_delta})") + + lines = lines or ("There's nothing here yet.",) + embed = Embed( + title=f"{self.__class__.__name__} watched users ({'updated' if update_cache else 'cached'})", + color=Color.blue() + ) + await LinePaginator.paginate(lines, ctx, embed, empty=False) + + @staticmethod + def _get_time_delta(time_string: str) -> str: + """Returns the time in human-readable time delta format.""" + date_time = datetime.datetime.strptime( + time_string, + "%Y-%m-%dT%H:%M:%S.%fZ" + ).replace(tzinfo=None) + time_delta = time_since(date_time, precision="minutes", max_units=1) + + return time_delta + + @staticmethod + def _get_human_readable(time_string: str, output_format: str = "%Y-%m-%d %H:%M:%S") -> str: + date_time = datetime.datetime.strptime( + time_string, + "%Y-%m-%dT%H:%M:%S.%fZ" + ).replace(tzinfo=None) + return date_time.strftime(output_format) + + def _remove_user(self, user_id: int) -> None: + """Removes a user from a watch channel.""" + self.watched_users.pop(user_id, None) + self.message_queue.pop(user_id, None) + self.consumption_queue.pop(user_id, None) + + def cog_unload(self) -> None: + """Takes care of unloading the cog and canceling the consumption task.""" + self.log.trace(f"Unloading the cog") + if not self._consume_task.done(): + self._consume_task.cancel() + try: + self._consume_task.result() + except asyncio.CancelledError as e: + self.log.exception( + f"The consume task was canceled. Messages may be lost.", + exc_info=e + ) diff --git a/bot/constants.py b/bot/constants.py index d2c953276..033d19f7b 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -355,6 +355,14 @@ class Channels(metaclass=YAMLGetter): verification: int +class Webhooks(metaclass=YAMLGetter): + section = "guild" + subsection = "webhooks" + + talent_pool: int + big_brother: int + + class Roles(metaclass=YAMLGetter): section = "guild" subsection = "roles" @@ -389,18 +397,12 @@ class Keys(metaclass=YAMLGetter): site_api: str -class RabbitMQ(metaclass=YAMLGetter): - section = "rabbitmq" - - host: str - password: str - port: int - username: str - - class URLs(metaclass=YAMLGetter): section = "urls" + # Snekbox endpoints + snekbox_eval_api: str + # Discord API endpoints discord_api: str discord_invite_api: str diff --git a/bot/utils/messages.py b/bot/utils/messages.py index fc38b0127..94a8b36ed 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -1,9 +1,9 @@ import asyncio import contextlib from io import BytesIO -from typing import Sequence +from typing import Sequence, Union -from discord import Embed, File, Message, TextChannel +from discord import Embed, File, Message, TextChannel, Webhook from discord.abc import Snowflake from discord.errors import HTTPException @@ -78,9 +78,9 @@ async def wait_for_deletion( await message.delete() -async def send_attachments(message: Message, destination: TextChannel): +async def send_attachments(message: Message, destination: Union[TextChannel, Webhook]): """ - Re-uploads each attachment in a message to the given channel. + Re-uploads each attachment in a message to the given channel or webhook. Each attachment is sent as a separate message to more easily comply with the 8 MiB request size limit. If attachments are too large, they are instead grouped into a single embed which links to them. @@ -97,7 +97,16 @@ async def send_attachments(message: Message, destination: TextChannel): if attachment.size <= MAX_SIZE - 512: with BytesIO() as file: await attachment.save(file) - await destination.send(file=File(file, filename=attachment.filename)) + attachment_file = File(file, filename=attachment.filename) + + if isinstance(destination, TextChannel): + await destination.send(file=attachment_file) + else: + await destination.send( + file=attachment_file, + username=message.author.display_name, + avatar_url=message.author.avatar_url + ) else: large.append(attachment) except HTTPException as e: @@ -109,4 +118,11 @@ async def send_attachments(message: Message, destination: TextChannel): if large: embed = Embed(description=f"\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large)) embed.set_footer(text="Attachments exceed upload size limit.") - await destination.send(embed=embed) + if isinstance(destination, TextChannel): + await destination.send(embed=embed) + else: + await destination.send( + embed=embed, + username=message.author.display_name, + avatar_url=message.author.avatar_url + ) diff --git a/bot/utils/moderation.py b/bot/utils/moderation.py index fcdf3c4d5..c1eb98dd6 100644 --- a/bot/utils/moderation.py +++ b/bot/utils/moderation.py @@ -14,8 +14,13 @@ HEADERS = {"X-API-KEY": Keys.site_api} async def post_infraction( - ctx: Context, user: Union[Member, Object, User], - type: str, reason: str, expires_at: datetime = None, hidden: bool = False + ctx: Context, + user: Union[Member, Object, User], + type: str, + reason: str, + expires_at: datetime = None, + hidden: bool = False, + active: bool = True, ): payload = { @@ -23,7 +28,8 @@ async def post_infraction( "hidden": hidden, "reason": reason, "type": type, - "user": user.id + "user": user.id, + "active": active } if expires_at: payload['expires_at'] = expires_at.isoformat() diff --git a/bot/utils/service_discovery.py b/bot/utils/service_discovery.py deleted file mode 100644 index 8d79096bd..000000000 --- a/bot/utils/service_discovery.py +++ /dev/null @@ -1,22 +0,0 @@ -import datetime -import socket -import time -from contextlib import closing - -from bot.constants import RabbitMQ - -THIRTY_SECONDS = datetime.timedelta(seconds=30) - - -def wait_for_rmq(): - start = datetime.datetime.now() - - while True: - if datetime.datetime.now() - start > THIRTY_SECONDS: - return False - - with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: - if sock.connect_ex((RabbitMQ.host, RabbitMQ.port)) == 0: - return True - - time.sleep(0.5) diff --git a/config-default.yml b/config-default.yml index d8016875b..dd5126230 100644 --- a/config-default.yml +++ b/config-default.yml @@ -135,6 +135,10 @@ guild: rockstars: &ROCKSTARS_ROLE 458226413825294336 team_leader: 501324292341104650 + webhooks: + talent_pool: 569145364800602132 + big_brother: 569133704568373283 + filter: @@ -165,6 +169,8 @@ filter: - 249111029668249601 # Gentoo - 327254708534116352 # Adafruit - 544525886180032552 # kennethreitz.org + - 590806733924859943 # Discord Hack Week + - 423249981340778496 # Kivy domain_blacklist: - pornhub.com @@ -224,13 +230,6 @@ keys: site_api: !ENV "BOT_API_KEY" -rabbitmq: - host: "pdrmq" - password: !ENV ["RABBITMQ_DEFAULT_PASS", "guest"] - port: 5672 - username: !ENV ["RABBITMQ_DEFAULT_USER", "guest"] - - urls: # PyDis site vars site: &DOMAIN "pythondiscord.com" @@ -258,6 +257,9 @@ urls: site_user_complete_api: !JOIN [*SCHEMA, *API, "/bot/users/complete"] paste_service: !JOIN [*SCHEMA, *PASTE, "/{key}"] + # Snekbox + snekbox_eval_api: "http://localhost:8060/eval" + # Env vars deploy: !ENV "DEPLOY_URL" status: !ENV "STATUS_URL" |