diff options
| author | 2019-07-16 15:02:43 -0700 | |
|---|---|---|
| committer | 2019-07-16 15:02:43 -0700 | |
| commit | 3232fd566d5b35f10ffff0d47129f9c052df1303 (patch) | |
| tree | 5c7eec255a7973afaba5b033e9feb7f3c4d6ad70 | |
| parent | Merge remote-tracking branch 'remotes/origin/master' into django (diff) | |
| parent | Merge pull request #376 from python-discord/django-watchchannels (diff) | |
Merge remote-tracking branch 'origin/django' into django-merge
Merge in RMQ removal, Snekbox changes, and watch channel rework, among
other minor things.
Diffstat (limited to '')
| -rw-r--r-- | bot/__init__.py | 2 | ||||
| -rw-r--r-- | bot/__main__.py | 14 | ||||
| -rw-r--r-- | bot/cogs/alias.py | 26 | ||||
| -rw-r--r-- | bot/cogs/bigbrother.py | 258 | ||||
| -rw-r--r-- | bot/cogs/nominations.py | 120 | ||||
| -rw-r--r-- | bot/cogs/rmq.py | 229 | ||||
| -rw-r--r-- | bot/cogs/snekbox.py | 264 | ||||
| -rw-r--r-- | bot/cogs/watchchannels/__init__.py | 15 | ||||
| -rw-r--r-- | bot/cogs/watchchannels/bigbrother.py | 100 | ||||
| -rw-r--r-- | bot/cogs/watchchannels/talentpool.py | 233 | ||||
| -rw-r--r-- | bot/cogs/watchchannels/watchchannel.py | 353 | ||||
| -rw-r--r-- | bot/constants.py | 20 | ||||
| -rw-r--r-- | bot/utils/messages.py | 28 | ||||
| -rw-r--r-- | bot/utils/moderation.py | 12 | ||||
| -rw-r--r-- | bot/utils/service_discovery.py | 22 | ||||
| -rw-r--r-- | config-default.yml | 16 | 
16 files changed, 927 insertions, 785 deletions
| diff --git a/bot/__init__.py b/bot/__init__.py index a088138a0..8efa5e53c 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -55,7 +55,7 @@ else:  logging.basicConfig( -    format="%(asctime)s pd.beardfist.com Bot: | %(name)30s | %(levelname)8s | %(message)s", +    format="%(asctime)s pd.beardfist.com Bot: | %(name)33s | %(levelname)8s | %(message)s",      datefmt="%b %d %H:%M:%S",      level=logging.TRACE if DEBUG_MODE else logging.INFO,      handlers=logging_handlers diff --git a/bot/__main__.py b/bot/__main__.py index ab212a8a1..b3f80ef55 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -8,7 +8,6 @@ from discord.ext.commands import Bot, when_mentioned_or  from bot.api import APIClient  from bot.constants import Bot as BotConfig, DEBUG_MODE -from bot.utils.service_discovery import wait_for_rmq  log = logging.getLogger(__name__) @@ -31,14 +30,6 @@ bot.http_session = ClientSession(  )  bot.api_client = APIClient(loop=asyncio.get_event_loop()) -log.info("Waiting for RabbitMQ...") -has_rmq = wait_for_rmq() - -if has_rmq: -    log.info("RabbitMQ found") -else: -    log.warning("Timed out while waiting for RabbitMQ") -  # Internal/debug  bot.load_extension("bot.cogs.filtering")  bot.load_extension("bot.cogs.logging") @@ -47,7 +38,6 @@ bot.load_extension("bot.cogs.security")  # Commands, etc  bot.load_extension("bot.cogs.antispam") -bot.load_extension("bot.cogs.bigbrother")  bot.load_extension("bot.cogs.bot")  bot.load_extension("bot.cogs.clean")  bot.load_extension("bot.cogs.cogs") @@ -78,11 +68,9 @@ bot.load_extension("bot.cogs.sync")  bot.load_extension("bot.cogs.tags")  bot.load_extension("bot.cogs.token_remover")  bot.load_extension("bot.cogs.utils") +bot.load_extension("bot.cogs.watchchannels")  bot.load_extension("bot.cogs.wolfram") -if has_rmq: -    bot.load_extension("bot.cogs.rmq") -  bot.run(BotConfig.token)  bot.http_session.close()  # Close the aiohttp session when the bot finishes running diff --git a/bot/cogs/alias.py b/bot/cogs/alias.py index 02f92a79d..85d101448 100644 --- a/bot/cogs/alias.py +++ b/bot/cogs/alias.py @@ -1,11 +1,13 @@  import inspect  import logging +from typing import Union -from discord import Colour, Embed, User +from discord import Colour, Embed, Member, User  from discord.ext.commands import (      Command, Context, clean_content, command, group  ) +from bot.cogs.watchchannels.watchchannel import proxy_user  from bot.converters import TagNameConverter  from bot.pagination import LinePaginator @@ -70,9 +72,7 @@ class Alias:          await self.invoke(ctx, "site resources")      @command(name="watch", hidden=True) -    async def bigbrother_watch_alias( -            self, ctx: Context, user: User, *, reason: str -    ): +    async def bigbrother_watch_alias(self, ctx, user: Union[Member, User, proxy_user], *, reason: str):          """          Alias for invoking <prefix>bigbrother watch [user] [reason].          """ @@ -80,7 +80,7 @@ class Alias:          await self.invoke(ctx, "bigbrother watch", user, reason=reason)      @command(name="unwatch", hidden=True) -    async def bigbrother_unwatch_alias(self, ctx, user: User, *, reason: str): +    async def bigbrother_unwatch_alias(self, ctx, user: Union[User, proxy_user], *, reason: str):          """          Alias for invoking <prefix>bigbrother unwatch [user] [reason].          """ @@ -181,6 +181,22 @@ class Alias:          await self.invoke(ctx, "docs get", symbol) +    @command(name="nominate", hidden=True) +    async def nomination_add_alias(self, ctx, user: Union[Member, User, proxy_user], *, reason: str): +        """ +        Alias for invoking <prefix>talentpool add [user] [reason]. +        """ + +        await self.invoke(ctx, "talentpool add", user, reason=reason) + +    @command(name="unnominate", hidden=True) +    async def nomination_end_alias(self, ctx, user: Union[User, proxy_user], *, reason: str): +        """ +        Alias for invoking <prefix>nomination end [user] [reason]. +        """ + +        await self.invoke(ctx, "nomination end", user, reason=reason) +  def setup(bot):      bot.add_cog(Alias(bot)) diff --git a/bot/cogs/bigbrother.py b/bot/cogs/bigbrother.py deleted file mode 100644 index df7a0b576..000000000 --- a/bot/cogs/bigbrother.py +++ /dev/null @@ -1,258 +0,0 @@ -import asyncio -import logging -import re -from collections import defaultdict, deque -from typing import List, Union - -from discord import Color, Embed, Guild, Member, Message, User -from discord.ext.commands import Bot, Context, group - -from bot.constants import ( -    BigBrother as BigBrotherConfig, Channels, Emojis, Guild as GuildConfig, Roles -) -from bot.decorators import with_role -from bot.pagination import LinePaginator -from bot.utils import messages -from bot.utils.moderation import post_infraction - -log = logging.getLogger(__name__) - -URL_RE = re.compile(r"(https?://[^\s]+)") - - -class BigBrother: -    """User monitoring to assist with moderation.""" - -    def __init__(self, bot: Bot): -        self.bot = bot -        self.watched_users = set()  # { user_id } -        self.channel_queues = defaultdict(lambda: defaultdict(deque))  # { user_id: { channel_id: queue(messages) } -        self.last_log = [None, None, 0]  # [user_id, channel_id, message_count] -        self.consuming = False - -    def update_cache(self, api_response: List[dict]): -        """ -        Updates the internal cache of watched users from the given `api_response`. -        This function will only add (or update) existing keys, it will not delete -        keys that were not present in the API response. -        A user is only added if the bot can find a channel -        with the given `channel_id` in its channel cache. -        """ - -        for entry in api_response: -            user_id = entry['user'] -            self.watched_users.add(user_id) - -    async def on_ready(self): -        """Retrieves watched users from the API.""" - -        self.channel = self.bot.get_channel(Channels.big_brother_logs) -        if self.channel is None: -            log.error("Cannot find Big Brother channel. Cannot watch users.") -        else: -            data = await self.bot.api_client.get( -                'bot/infractions', -                params={ -                    'active': 'true', -                    'type': 'watch' -                } -            ) -            self.update_cache(data) - -    async def on_member_ban(self, guild: Guild, user: Union[User, Member]): -        if guild.id == GuildConfig.id and user.id in self.watched_users: -            [active_watch] = await self.bot.api_client.get( -                'bot/infractions', -                params={ -                    'active': 'true', -                    'type': 'watch', -                    'user__id': str(user.id) -                } -            ) -            await self.bot.api_client.put( -                'bot/infractions/' + str(active_watch['id']), -                json={'active': False} -            ) -            self.watched_users.remove(user.id) -            del self.channel_queues[user.id] -            await self.channel.send( -                f"{Emojis.bb_message}:hammer: {user} got banned, so " -                f"`BigBrother` will no longer relay their messages." -            ) - -    async def on_message(self, msg: Message): -        """Queues up messages sent by watched users.""" - -        if msg.author.id in self.watched_users: -            if not self.consuming: -                self.bot.loop.create_task(self.consume_messages()) - -            log.trace(f"Received message: {msg.content} ({len(msg.attachments)} attachments)") -            self.channel_queues[msg.author.id][msg.channel.id].append(msg) - -    async def consume_messages(self): -        """Consumes the message queues to log watched users' messages.""" - -        if not self.consuming: -            self.consuming = True -            log.trace("Sleeping before consuming...") -            await asyncio.sleep(BigBrotherConfig.log_delay) - -        log.trace("Begin consuming messages.") -        channel_queues = self.channel_queues.copy() -        self.channel_queues.clear() -        for _, queues in channel_queues.items(): -            for queue in queues.values(): -                while queue: -                    msg = queue.popleft() -                    log.trace(f"Consuming message: {msg.clean_content} ({len(msg.attachments)} attachments)") - -                    self.last_log[2] += 1  # Increment message count. -                    await self.send_header(msg) -                    await self.log_message(msg) - -        if self.channel_queues: -            log.trace("Queue not empty; continue consumption.") -            self.bot.loop.create_task(self.consume_messages()) -        else: -            log.trace("Done consuming messages.") -            self.consuming = False - -    async def send_header(self, message: Message): -        """ -        Sends a log message header to the given channel. - -        A header is only sent if the user or channel are different than the previous, or if the configured message -        limit for a single header has been exceeded. - -        :param message: the first message in the queue -        """ - -        last_user, last_channel, msg_count = self.last_log -        limit = BigBrotherConfig.header_message_limit - -        # Send header if user/channel are different or if message limit exceeded. -        if message.author.id != last_user or message.channel.id != last_channel or msg_count > limit: -            self.last_log = [message.author.id, message.channel.id, 0] - -            embed = Embed(description=f"{message.author.mention} in [#{message.channel.name}]({message.jump_url})") -            embed.set_author(name=message.author.nick or message.author.name, icon_url=message.author.avatar_url) -            await self.channel.send(embed=embed) - -    async def log_message(self, message: Message): -        """ -        Logs a watched user's message in the given channel. - -        Attachments are also sent. All non-image or non-video URLs are put in inline code blocks to prevent preview -        embeds from being automatically generated. - -        :param message: the message to log -        """ - -        content = message.clean_content -        if content: -            # Put all non-media URLs in inline code blocks. -            media_urls = {embed.url for embed in message.embeds if embed.type in ("image", "video")} -            for url in URL_RE.findall(content): -                if url not in media_urls: -                    content = content.replace(url, f"`{url}`") - -            await self.channel.send(content) - -        await messages.send_attachments(message, self.channel) - -    @group(name='bigbrother', aliases=('bb',), invoke_without_command=True) -    @with_role(Roles.owner, Roles.admin, Roles.moderator) -    async def bigbrother_group(self, ctx: Context): -        """Monitor users, NSA-style.""" - -        await ctx.invoke(self.bot.get_command("help"), "bigbrother") - -    @bigbrother_group.command(name='watched', aliases=('all',)) -    @with_role(Roles.owner, Roles.admin, Roles.moderator) -    async def watched_command(self, ctx: Context, from_cache: bool = True): -        """ -        Shows all users that are currently monitored and in which channel. -        By default, the users are returned from the cache. -        If this is not desired, `from_cache` can be given as a falsy value, e.g. e.g. 'no'. -        """ - -        if from_cache: -            lines = tuple(f"• <@{user_id}>" for user_id in self.watched_users) -            await LinePaginator.paginate( -                lines or ("There's nothing here yet.",), -                ctx, -                Embed(title="Watched users (cached)", color=Color.blue()), -                empty=False -            ) - -        else: -            active_watches = await self.bot.api_client.get( -                'bot/infractions', -                params={ -                    'active': 'true', -                    'type': 'watch' -                } -            ) -            self.update_cache(active_watches) -            lines = tuple( -                f"• <@{entry['user']}>: {entry['reason'] or '*no reason provided*'}" -                for entry in active_watches -            ) - -            await LinePaginator.paginate( -                lines or ("There's nothing here yet.",), -                ctx, -                Embed(title="Watched users", color=Color.blue()), -                empty=False -            ) - -    @bigbrother_group.command(name='watch', aliases=('w',)) -    @with_role(Roles.owner, Roles.admin, Roles.moderator) -    async def watch_command(self, ctx: Context, user: User, *, reason: str): -        """ -        Relay messages sent by the given `user` to the `#big-brother-logs` channel - -        A `reason` for watching is required, which is added for the user to be watched as a -        note (aka: shadow warning) -        """ - -        if user.id in self.watched_users: -            return await ctx.send(":x: That user is already watched.") - -        await post_infraction( -            ctx, user, type='watch', reason=reason, hidden=True -        ) -        self.watched_users.add(user.id) -        await ctx.send(f":ok_hand: will now relay messages sent by {user}") - -    @bigbrother_group.command(name='unwatch', aliases=('uw',)) -    @with_role(Roles.owner, Roles.admin, Roles.moderator) -    async def unwatch_command(self, ctx: Context, user: User): -        """Stop relaying messages by the given `user`.""" - -        active_watches = await self.bot.api_client.get( -            'bot/infractions', -            params={ -                'active': 'true', -                'type': 'watch', -                'user__id': str(user.id) -            } -        ) -        if active_watches: -            [infraction] = active_watches -            await self.bot.api_client.patch( -                'bot/infractions/' + str(infraction['id']), -                json={'active': False} -            ) -            await ctx.send(f":ok_hand: will no longer relay messages sent by {user}") -            self.watched_users.remove(user.id) -            if user.id in self.channel_queues: -                del self.channel_queues[user.id] -        else: -            await ctx.send(":x: that user is currently not being watched") - - -def setup(bot: Bot): -    bot.add_cog(BigBrother(bot)) -    log.info("Cog loaded: BigBrother") diff --git a/bot/cogs/nominations.py b/bot/cogs/nominations.py deleted file mode 100644 index 93ee0d885..000000000 --- a/bot/cogs/nominations.py +++ /dev/null @@ -1,120 +0,0 @@ -import logging - -from discord import Color, Embed, User -from discord.ext.commands import Context, group - -from bot.cogs.bigbrother import BigBrother, Roles -from bot.constants import Channels -from bot.decorators import with_role -from bot.pagination import LinePaginator - - -log = logging.getLogger(__name__) - - -class Nominations(BigBrother): -    """Monitor potential helpers, NSA-style.""" - -    async def on_ready(self): -        """Retrieve nominees from the API.""" - -        self.channel = self.bot.get_channel(Channels.talent_pool) -        if self.channel is None: -            log.error("Cannot find talent pool channel. Cannot watch nominees.") -        else: -            nominations = await self.bot.api_client.get( -                'bot/nominations', -                params={'active': 'true'} -            ) -            self.update_cache(nominations) - -    async def on_member_ban(self, *_): -        pass - -    @group(name='nominations', aliases=('n',), invoke_without_command=True) -    @with_role(Roles.owner, Roles.admin, Roles.moderator) -    async def bigbrother_group(self, ctx: Context): -        """Nominate helpers, NSA-style.""" - -        await ctx.invoke(self.bot.get_command("help"), "nominations") - -    @bigbrother_group.command(name='nominated', aliases=('nominees', 'all')) -    @with_role(Roles.owner, Roles.admin, Roles.moderator) -    async def watched_command(self, ctx: Context, from_cache: bool = True): -        if from_cache: -            lines = tuple(f"• <@{user_id}>" for user_id in self.watched_users) - -        else: -            active_nominations = await self.bot.api_client.get( -                'bot/nominations', -                params={'active': 'true'} -            ) -            self.update_cache(active_nominations) -            lines = tuple( -                f"• <@{entry['user']}>: {entry['reason'] or '*no reason provided*'}" -                for entry in active_nominations -            ) - -        await LinePaginator.paginate( -            lines or ("There's nothing here yet.",), -            ctx, -            Embed( -                title="Nominated users" + " (cached)" * from_cache, -                color=Color.blue() -            ), -            empty=False -        ) - -    @bigbrother_group.command(name='nominate', aliases=('n',)) -    @with_role(Roles.owner, Roles.admin, Roles.moderator) -    async def watch_command(self, ctx: Context, user: User, *, reason: str): -        """Talent pool the given `user`.""" - -        active_nominations = await self.bot.api_client.get( -            'bot/nominations/' + str(user.id), -        ) -        if active_nominations: -            active_nominations = await self.bot.api_client.put( -                'bot/nominations/' + str(user.id), -                json={'active': True} -            ) -            await ctx.send(":ok_hand: user's watch was updated") - -        else: -            active_nominations = await self.bot.api_client.post( -                'bot/nominations/' + str(user.id), -                json={ -                    'active': True, -                    'author': ctx.author.id, -                    'reason': reason, -                } -            ) -            self.watched_users.add(user.id) -            await ctx.send(":ok_hand: user added to talent pool") - -    @bigbrother_group.command(name='unnominate', aliases=('un',)) -    @with_role(Roles.owner, Roles.admin, Roles.moderator) -    async def unwatch_command(self, ctx: Context, user: User): -        """Stop talent pooling the given `user`.""" - -        nomination = await self.bot.api_client.get( -            'bot/nominations/' + str(user.id) -        ) - -        if not nomination['active']: -            await ctx.send(":x: the nomination is already inactive") - -        else: -            await self.bot.api_client.put( -                'bot/nominations/' + str(user.id), -                json={'active': False} -            ) -            self.watched_users.remove(user.id) -            if user.id in self.channel_queues: -                del self.channel_queues[user.id] -            await ctx.send(f":ok_hand: {user} is no longer part of the talent pool") - - -def setup(bot): -    bot.add_cog(Nominations(bot)) -    log.info("Cog loaded: Nominations") diff --git a/bot/cogs/rmq.py b/bot/cogs/rmq.py deleted file mode 100644 index 585eacc25..000000000 --- a/bot/cogs/rmq.py +++ /dev/null @@ -1,229 +0,0 @@ -import asyncio -import datetime -import json -import logging -import pprint - -import aio_pika -from aio_pika import Message -from dateutil import parser as date_parser -from discord import Colour, Embed -from discord.ext.commands import Bot -from discord.utils import get - -from bot.constants import Channels, Guild, RabbitMQ - -log = logging.getLogger(__name__) - -LEVEL_COLOURS = { -    "debug": Colour.blue(), -    "info": Colour.green(), -    "warning": Colour.gold(), -    "error": Colour.red() -} - -DEFAULT_LEVEL_COLOUR = Colour.greyple() -EMBED_PARAMS = ( -    "colour", "title", "url", "description", "timestamp" -) - -CONSUME_TIMEOUT = datetime.timedelta(seconds=10) - - -class RMQ: -    """ -    RabbitMQ event handling -    """ - -    rmq = None  # type: aio_pika.Connection -    channel = None  # type: aio_pika.Channel -    queue = None  # type: aio_pika.Queue - -    def __init__(self, bot: Bot): -        self.bot = bot - -    async def on_ready(self): -        self.rmq = await aio_pika.connect_robust( -            host=RabbitMQ.host, port=RabbitMQ.port, login=RabbitMQ.username, password=RabbitMQ.password -        ) - -        log.info("Connected to RabbitMQ") - -        self.channel = await self.rmq.channel() -        self.queue = await self.channel.declare_queue("bot_events", durable=True) - -        log.debug("Channel opened, queue declared") - -        async for message in self.queue: -            with message.process(): -                message.ack() -                await self.handle_message(message, message.body.decode()) - -    async def send_text(self, queue: str, data: str): -        message = Message(data.encode("utf-8")) -        await self.channel.default_exchange.publish(message, queue) - -    async def send_json(self, queue: str, **data): -        message = Message(json.dumps(data).encode("utf-8")) -        await self.channel.default_exchange.publish(message, queue) - -    async def consume(self, queue: str, **kwargs): -        queue_obj = await self.channel.declare_queue(queue, **kwargs) - -        result = None -        start_time = datetime.datetime.now() - -        while result is None: -            if datetime.datetime.now() - start_time >= CONSUME_TIMEOUT: -                result = "Timed out while waiting for a response." -            else: -                result = await queue_obj.get(timeout=5, fail=False) -                await asyncio.sleep(0.5) - -        if result: -            result.ack() - -        return result - -    async def handle_message(self, message, data): -        log.debug(f"Message: {message}") -        log.debug(f"Data: {data}") - -        try: -            data = json.loads(data) -        except Exception: -            await self.do_mod_log("error", "Unable to parse event", data) -        else: -            event = data["event"] -            event_data = data["data"] - -            try: -                func = getattr(self, f"do_{event}") -                await func(**event_data) -            except Exception as e: -                await self.do_mod_log( -                    "error", f"Unable to handle event: {event}", -                    str(e) -                ) - -    async def do_mod_log(self, level: str, title: str, message: str): -        colour = LEVEL_COLOURS.get(level, DEFAULT_LEVEL_COLOUR) -        embed = Embed( -            title=title, description=f"```\n{message}\n```", -            colour=colour, timestamp=datetime.datetime.utcnow() -        ) - -        await self.bot.get_channel(Channels.devlog).send(embed=embed) -        log.log(logging._nameToLevel[level.upper()], f"Modlog: {title} | {message}") - -    async def do_send_message(self, target: int, message: str): -        channel = self.bot.get_channel(target) - -        if channel is None: -            await self.do_mod_log( -                "error", "Failed: Send Message", -                f"Unable to find channel: {target}" -            ) -        else: -            await channel.send(message) - -            await self.do_mod_log( -                "info", "Succeeded: Send Embed", -                f"Message sent to channel {target}\n\n{message}" -            ) - -    async def do_send_embed(self, target: int, **embed_params): -        for param, value in list(embed_params.items()):  # To keep a full copy -            if param not in EMBED_PARAMS: -                await self.do_mod_log( -                    "warning", "Warning: Send Embed", -                    f"Unknown embed parameter: {param}" -                ) -                del embed_params[param] - -            if param == "timestamp": -                embed_params[param] = date_parser.parse(value) -            elif param == "colour": -                embed_params[param] = Colour(value) - -        channel = self.bot.get_channel(target) - -        if channel is None: -            await self.do_mod_log( -                "error", "Failed: Send Embed", -                f"Unable to find channel: {target}" -            ) -        else: -            await channel.send(embed=Embed(**embed_params)) - -            await self.do_mod_log( -                "info", "Succeeded: Send Embed", -                f"Embed sent to channel {target}\n\n{pprint.pformat(embed_params, 4)}" -            ) - -    async def do_add_role(self, target: int, role_id: int, reason: str): -        guild = self.bot.get_guild(Guild.id) -        member = guild.get_member(int(target)) - -        if member is None: -            return await self.do_mod_log( -                "error", "Failed: Add Role", -                f"Unable to find member: {target}" -            ) - -        role = get(guild.roles, id=int(role_id)) - -        if role is None: -            return await self.do_mod_log( -                "error", "Failed: Add Role", -                f"Unable to find role: {role_id}" -            ) - -        try: -            await member.add_roles(role, reason=reason) -        except Exception as e: -            await self.do_mod_log( -                "error", "Failed: Add Role", -                f"Error while adding role {role.name}: {e}" -            ) -        else: -            await self.do_mod_log( -                "info", "Succeeded: Add Role", -                f"Role {role.name} added to member {target}" -            ) - -    async def do_remove_role(self, target: int, role_id: int, reason: str): -        guild = self.bot.get_guild(Guild.id) -        member = guild.get_member(int(target)) - -        if member is None: -            return await self.do_mod_log( -                "error", "Failed: Remove Role", -                f"Unable to find member: {target}" -            ) - -        role = get(guild.roles, id=int(role_id)) - -        if role is None: -            return await self.do_mod_log( -                "error", "Failed: Remove Role", -                f"Unable to find role: {role_id}" -            ) - -        try: -            await member.remove_roles(role, reason=reason) -        except Exception as e: -            await self.do_mod_log( -                "error", "Failed: Remove Role", -                f"Error while adding role {role.name}: {e}" -            ) -        else: -            await self.do_mod_log( -                "info", "Succeeded: Remove Role", -                f"Role {role.name} removed from member {target}" -            ) - - -def setup(bot): -    bot.add_cog(RMQ(bot)) -    log.info("Cog loaded: RMQ") diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index cc18c0041..05834e421 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -3,13 +3,14 @@ import logging  import random  import re  import textwrap +from signal import Signals +from typing import Optional, Tuple  from discord import Colour, Embed  from discord.ext.commands import (      Bot, CommandError, Context, NoPrivateMessage, command, guild_only  ) -from bot.cogs.rmq import RMQ  from bot.constants import Channels, ERROR_REPLIES, NEGATIVE_REPLIES, STAFF_ROLES, URLs  from bot.decorators import InChannelCheckFailure, in_channel  from bot.utils.messages import wait_for_deletion @@ -17,23 +18,6 @@ from bot.utils.messages import wait_for_deletion  log = logging.getLogger(__name__) -RMQ_ARGS = { -    "durable": False, -    "arguments": {"x-message-ttl": 5000}, -    "auto_delete": True -} - -CODE_TEMPLATE = """ -venv_file = "/snekbox/.venv/bin/activate_this.py" -exec(open(venv_file).read(), dict(__file__=venv_file)) - -try: -{CODE} -except: -    import traceback -    print(traceback.format_exc()) -""" -  ESCAPE_REGEX = re.compile("[`\u202E\u200B]{3,}")  FORMATTED_CODE_REGEX = re.compile(      r"^\s*"                                 # any leading whitespace from the beginning of the string @@ -53,42 +37,47 @@ RAW_CODE_REGEX = re.compile(      re.DOTALL                               # "." also matches newlines  ) +MAX_PASTE_LEN = 1000 +  class Snekbox:      """ -    Safe evaluation using Snekbox +    Safe evaluation of Python code using Snekbox      """      def __init__(self, bot: Bot):          self.bot = bot          self.jobs = {} -    @property -    def rmq(self) -> RMQ: -        return self.bot.get_cog("RMQ") - -    @command(name='eval', aliases=('e',)) -    @guild_only() -    @in_channel(Channels.bot, bypass_roles=STAFF_ROLES) -    async def eval_command(self, ctx: Context, *, code: str = None): -        """ -        Run some code. get the result back. We've done our best to make this safe, but do let us know if you -        manage to find an issue with it! - -        This command supports multiple lines of code, including code wrapped inside a formatted code block. -        """ +    async def post_eval(self, code: str) -> dict: +        """Send a POST request to the Snekbox API to evaluate code and return the results.""" +        url = URLs.snekbox_eval_api +        data = {"input": code} +        async with self.bot.http_session.post(url, json=data, raise_for_status=True) as resp: +            return await resp.json() -        if ctx.author.id in self.jobs: -            await ctx.send(f"{ctx.author.mention} You've already got a job running - please wait for it to finish!") -            return - -        if not code:  # None or empty string -            return await ctx.invoke(self.bot.get_command("help"), "eval") +    async def upload_output(self, output: str) -> Optional[str]: +        """Upload the eval output to a paste service and return a URL to it if successful.""" +        log.trace("Uploading full output to paste service...") -        log.info(f"Received code from {ctx.author.name}#{ctx.author.discriminator} for evaluation:\n{code}") -        self.jobs[ctx.author.id] = datetime.datetime.now() +        if len(output) > MAX_PASTE_LEN: +            log.info("Full output is too long to upload") +            return "too long to upload" -        # Strip whitespace and inline or block code markdown and extract the code and some formatting info +        url = URLs.paste_service.format(key="documents") +        try: +            async with self.bot.http_session.post(url, data=output, raise_for_status=True) as resp: +                data = await resp.json() + +            if "key" in data: +                return URLs.paste_service.format(key=data["key"]) +        except Exception: +            # 400 (Bad Request) means there are too many characters +            log.exception("Failed to upload full output to paste service!") + +    @staticmethod +    def prepare_input(code: str) -> str: +        """Extract code from the Markdown, format it, and insert it into the code template."""          match = FORMATTED_CODE_REGEX.fullmatch(code)          if match:              code, block, lang, delim = match.group("code", "block", "lang", "delim") @@ -100,87 +89,138 @@ class Snekbox:              log.trace(f"Extracted {info} for evaluation:\n{code}")          else:              code = textwrap.dedent(RAW_CODE_REGEX.fullmatch(code).group("code")) -            log.trace(f"Eval message contains not or badly formatted code, stripping whitespace only:\n{code}") +            log.trace( +                f"Eval message contains unformatted or badly formatted code, " +                f"stripping whitespace only:\n{code}" +            ) -        try: -            stripped_lines = [ln.strip() for ln in code.split('\n')] -            if all(line.startswith('#') for line in stripped_lines): -                return await ctx.send( -                    f"{ctx.author.mention} Your eval job has completed.\n\n```[No output]```" -                ) +        return code + +    @staticmethod +    def get_results_message(results: dict) -> Tuple[str, str]: +        """Return a user-friendly message and error corresponding to the process's return code.""" +        stdout, returncode = results["stdout"], results["returncode"] +        msg = f"Your eval job has completed with return code {returncode}" +        error = "" + +        if returncode is None: +            msg = "Your eval job has failed" +            error = stdout.strip() +        elif returncode == 128 + Signals.SIGKILL: +            msg = "Your eval job timed out or ran out of memory" +        elif returncode == 255: +            msg = "Your eval job has failed" +            error = "A fatal NsJail error occurred" +        else: +            # Try to append signal's name if one exists +            try: +                name = Signals(returncode - 128).name +                msg = f"{msg} ({name})" +            except ValueError: +                pass + +        return msg, error + +    async def format_output(self, output: str) -> Tuple[str, Optional[str]]: +        """ +        Format the output and return a tuple of the formatted output and a URL to the full output. + +        Prepend each line with a line number. Truncate if there are over 10 lines or 1000 characters +        and upload the full output to a paste service. +        """ +        log.trace("Formatting output...") + +        output = output.strip(" \n") +        original_output = output  # To be uploaded to a pasting service if needed +        paste_link = None + +        if "<@" in output: +            output = output.replace("<@", "<@\u200B")  # Zero-width space -            code = textwrap.indent(code, "    ") -            code = CODE_TEMPLATE.replace("{CODE}", code) +        if "<!@" in output: +            output = output.replace("<!@", "<!@\u200B")  # Zero-width space -            await self.rmq.send_json( -                "input", -                snekid=str(ctx.author.id), message=code +        if ESCAPE_REGEX.findall(output): +            return "Code block escape attempt detected; will not output result", paste_link + +        truncated = False +        lines = output.count("\n") + +        if lines > 0: +            output = output.split("\n")[:10]  # Only first 10 cause the rest is truncated anyway +            output = (f"{i:03d} | {line}" for i, line in enumerate(output, 1)) +            output = "\n".join(output) + +        if lines > 10: +            truncated = True +            if len(output) >= 1000: +                output = f"{output[:1000]}\n... (truncated - too long, too many lines)" +            else: +                output = f"{output}\n... (truncated - too many lines)" +        elif len(output) >= 1000: +            truncated = True +            output = f"{output[:1000]}\n... (truncated - too long)" + +        if truncated: +            paste_link = await self.upload_output(original_output) + +        output = output.strip() +        if not output: +            output = "[No output]" + +        return output, paste_link + +    @command(name="eval", aliases=("e",)) +    @guild_only() +    @in_channel(Channels.bot, bypass_roles=STAFF_ROLES) +    async def eval_command(self, ctx: Context, *, code: str = None): +        """ +        Run Python code and get the results. + +        This command supports multiple lines of code, including code wrapped inside a formatted code +        block. We've done our best to make this safe, but do let us know if you manage to find an +        issue with it! +        """ +        if ctx.author.id in self.jobs: +            return await ctx.send( +                f"{ctx.author.mention} You've already got a job running - " +                f"please wait for it to finish!"              ) -            async with ctx.typing(): -                message = await self.rmq.consume(str(ctx.author.id), **RMQ_ARGS) -                paste_link = None +        if not code:  # None or empty string +            return await ctx.invoke(self.bot.get_command("help"), "eval") -                if isinstance(message, str): -                    output = str.strip(" \n") -                else: -                    output = message.body.decode().strip(" \n") +        log.info( +            f"Received code from {ctx.author.name}#{ctx.author.discriminator} " +            f"for evaluation:\n{code}" +        ) -                if "<@" in output: -                    output = output.replace("<@", "<@\u200B")  # Zero-width space +        self.jobs[ctx.author.id] = datetime.datetime.now() +        code = self.prepare_input(code) -                if "<!@" in output: -                    output = output.replace("<!@", "<!@\u200B")  # Zero-width space +        try: +            async with ctx.typing(): +                results = await self.post_eval(code) +                msg, error = self.get_results_message(results) -                if ESCAPE_REGEX.findall(output): -                    output = "Code block escape attempt detected; will not output result" +                if error: +                    output, paste_link = error, None                  else: -                    # the original output, to send to a pasting service if needed -                    full_output = output -                    truncated = False -                    if output.count("\n") > 0: -                        output = [f"{i:03d} | {line}" for i, line in enumerate(output.split("\n"), start=1)] -                        output = "\n".join(output) - -                    if output.count("\n") > 10: -                        output = "\n".join(output.split("\n")[:10]) - -                        if len(output) >= 1000: -                            output = f"{output[:1000]}\n... (truncated - too long, too many lines)" -                        else: -                            output = f"{output}\n... (truncated - too many lines)" -                        truncated = True - -                    elif len(output) >= 1000: -                        output = f"{output[:1000]}\n... (truncated - too long)" -                        truncated = True - -                    if truncated: -                        try: -                            response = await self.bot.http_session.post( -                                URLs.paste_service.format(key="documents"), -                                data=full_output -                            ) -                            data = await response.json() -                            if "key" in data: -                                paste_link = URLs.paste_service.format(key=data["key"]) -                        except Exception: -                            log.exception("Failed to upload full output to paste service!") - -                if output.strip(): -                    if paste_link: -                        msg = f"{ctx.author.mention} Your eval job has completed.\n\n```py\n{output}\n```" \ -                              f"\nFull output: {paste_link}" -                    else: -                        msg = f"{ctx.author.mention} Your eval job has completed.\n\n```py\n{output}\n```" - -                    response = await ctx.send(msg) -                    self.bot.loop.create_task(wait_for_deletion(response, user_ids=(ctx.author.id,), client=ctx.bot)) +                    output, paste_link = await self.format_output(results["stdout"]) -                else: -                    await ctx.send( -                        f"{ctx.author.mention} Your eval job has completed.\n\n```[No output]```" -                    ) +                msg = f"{ctx.author.mention} {msg}.\n\n```py\n{output}\n```" +                if paste_link: +                    msg = f"{msg}\nFull output: {paste_link}" + +                response = await ctx.send(msg) +                self.bot.loop.create_task( +                    wait_for_deletion(response, user_ids=(ctx.author.id,), client=ctx.bot) +                ) + +                log.info( +                    f"{ctx.author.name}#{ctx.author.discriminator}'s job had a return code of " +                    f"{results['returncode']}" +                )          finally:              del self.jobs[ctx.author.id] diff --git a/bot/cogs/watchchannels/__init__.py b/bot/cogs/watchchannels/__init__.py new file mode 100644 index 000000000..ac7713803 --- /dev/null +++ b/bot/cogs/watchchannels/__init__.py @@ -0,0 +1,15 @@ +import logging + +from .bigbrother import BigBrother +from .talentpool import TalentPool + + +log = logging.getLogger(__name__) + + +def setup(bot): +    bot.add_cog(BigBrother(bot)) +    log.info("Cog loaded: BigBrother") + +    bot.add_cog(TalentPool(bot)) +    log.info("Cog loaded: TalentPool") diff --git a/bot/cogs/watchchannels/bigbrother.py b/bot/cogs/watchchannels/bigbrother.py new file mode 100644 index 000000000..e7b3d70bc --- /dev/null +++ b/bot/cogs/watchchannels/bigbrother.py @@ -0,0 +1,100 @@ +import logging +from collections import ChainMap +from typing import Union + +from discord import User +from discord.ext.commands import Context, group + +from bot.constants import Channels, Roles, Webhooks +from bot.decorators import with_role +from bot.utils.moderation import post_infraction +from .watchchannel import WatchChannel, proxy_user + +log = logging.getLogger(__name__) + + +class BigBrother(WatchChannel): +    """Monitors users by relaying their messages to a watch channel to assist with moderation.""" + +    def __init__(self, bot) -> None: +        super().__init__( +            bot, +            destination=Channels.big_brother_logs, +            webhook_id=Webhooks.big_brother, +            api_endpoint='bot/infractions', +            api_default_params={'active': 'true', 'type': 'watch', 'ordering': '-inserted_at'}, +            logger=log +        ) + +    @group(name='bigbrother', aliases=('bb',), invoke_without_command=True) +    @with_role(Roles.owner, Roles.admin, Roles.moderator) +    async def bigbrother_group(self, ctx: Context) -> None: +        """Monitors users by relaying their messages to the Big Brother watch channel.""" +        await ctx.invoke(self.bot.get_command("help"), "bigbrother") + +    @bigbrother_group.command(name='watched', aliases=('all', 'list')) +    @with_role(Roles.owner, Roles.admin, Roles.moderator) +    async def watched_command(self, ctx: Context, update_cache: bool = True) -> None: +        """ +        Shows the users that are currently being monitored by Big Brother. + +        The optional kwarg `update_cache` can be used to update the user +        cache using the API before listing the users. +        """ +        await self.list_watched_users(ctx, update_cache) + +    @bigbrother_group.command(name='watch', aliases=('w',)) +    @with_role(Roles.owner, Roles.admin, Roles.moderator) +    async def watch_command(self, ctx: Context, user: Union[User, proxy_user], *, reason: str) -> None: +        """ +        Relay messages sent by the given `user` to the `#big-brother` channel. + +        A `reason` for adding the user to Big Brother is required and will be displayed +        in the header when relaying messages of this user to the watchchannel. +        """ +        if user.bot: +            await ctx.send(f":x: I'm sorry {ctx.author}, I'm afraid I can't do that. I only watch humans.") +            return + +        if not await self.fetch_user_cache(): +            await ctx.send(f":x: Updating the user cache failed, can't watch user {user}") +            return + +        if user.id in self.watched_users: +            await ctx.send(":x: The specified user is already being watched.") +            return + +        response = await post_infraction( +            ctx, user, type='watch', reason=reason, hidden=True +        ) + +        if response is not None: +            self.watched_users[user.id] = response +            await ctx.send(f":white_check_mark: Messages sent by {user} will now be relayed to Big Brother.") + +    @bigbrother_group.command(name='unwatch', aliases=('uw',)) +    @with_role(Roles.owner, Roles.admin, Roles.moderator) +    async def unwatch_command(self, ctx: Context, user: Union[User, proxy_user], *, reason: str) -> None: +        """Stop relaying messages by the given `user`.""" +        active_watches = await self.bot.api_client.get( +            self.api_endpoint, +            params=ChainMap( +                self.api_default_params, +                {"user__id": str(user.id)} +            ) +        ) +        if active_watches: +            [infraction] = active_watches + +            await self.bot.api_client.patch( +                f"{self.api_endpoint}/{infraction['id']}", +                json={'active': False} +            ) + +            await post_infraction(ctx, user, type='watch', reason=f"Unwatched: {reason}", hidden=True, active=False) + +            await ctx.send(f":white_check_mark: Messages sent by {user} will no longer be relayed.") + +            self._remove_user(user.id) +        else: +            await ctx.send(":x: The specified user is currently not being watched.") diff --git a/bot/cogs/watchchannels/talentpool.py b/bot/cogs/watchchannels/talentpool.py new file mode 100644 index 000000000..6fbe2bc03 --- /dev/null +++ b/bot/cogs/watchchannels/talentpool.py @@ -0,0 +1,233 @@ +import logging +import textwrap +from collections import ChainMap +from typing import Union + +from aiohttp.client_exceptions import ClientResponseError +from discord import Color, Embed, Member, User +from discord.ext.commands import Context, group + +from bot.constants import Channels, Guild, Roles, Webhooks +from bot.decorators import with_role +from bot.pagination import LinePaginator +from .watchchannel import WatchChannel, proxy_user + +log = logging.getLogger(__name__) +STAFF_ROLES = Roles.owner, Roles.admin, Roles.moderator, Roles.helpers    # <- In constants after the merge? + + +class TalentPool(WatchChannel): +    """Relays messages of helper candidates to a watch channel to observe them.""" + +    def __init__(self, bot) -> None: +        super().__init__( +            bot, +            destination=Channels.talent_pool, +            webhook_id=Webhooks.talent_pool, +            api_endpoint='bot/nominations', +            api_default_params={'active': 'true', 'ordering': '-inserted_at'}, +            logger=log, +        ) + +    @group(name='talentpool', aliases=('tp', 'talent', 'nomination', 'n'), invoke_without_command=True) +    @with_role(Roles.owner, Roles.admin, Roles.moderator) +    async def nomination_group(self, ctx: Context) -> None: +        """Highlights the activity of helper nominees by relaying their messages to the talent pool channel.""" + +        await ctx.invoke(self.bot.get_command("help"), "talentpool") + +    @nomination_group.command(name='watched', aliases=('all', 'list')) +    @with_role(Roles.owner, Roles.admin, Roles.moderator) +    async def watched_command(self, ctx: Context, update_cache: bool = True) -> None: +        """ +        Shows the users that are currently being monitored in the talent pool. + +        The optional kwarg `update_cache` can be used to update the user +        cache using the API before listing the users. +        """ +        await self.list_watched_users(ctx, update_cache) + +    @nomination_group.command(name='watch', aliases=('w', 'add', 'a')) +    @with_role(Roles.owner, Roles.admin, Roles.moderator) +    async def watch_command(self, ctx: Context, user: Union[Member, User, proxy_user], *, reason: str) -> None: +        """ +        Relay messages sent by the given `user` to the `#talent-pool` channel. + +        A `reason` for adding the user to the talent pool is required and will be displayed +        in the header when relaying messages of this user to the channel. +        """ +        if user.bot: +            await ctx.send(f":x: I'm sorry {ctx.author}, I'm afraid I can't do that. I only watch humans.") +            return + +        if isinstance(user, Member) and any(role.id in STAFF_ROLES for role in user.roles): +            await ctx.send(f":x: Nominating staff members, eh? Here's a cookie :cookie:") +            return + +        if not await self.fetch_user_cache(): +            await ctx.send(f":x: Failed to update the user cache; can't add {user}") +            return + +        if user.id in self.watched_users: +            await ctx.send(":x: The specified user is already being watched in the talent pool") +            return + +        # Manual request with `raise_for_status` as False because we want the actual response +        session = self.bot.api_client.session +        url = self.bot.api_client._url_for(self.api_endpoint) +        kwargs = { +            'json': { +                'actor': ctx.author.id, +                'reason': reason, +                'user': user.id +            }, +            'raise_for_status': False, +        } +        async with session.post(url, **kwargs) as resp: +            response_data = await resp.json() + +            if resp.status == 400 and response_data.get('user', False): +                await ctx.send(":x: The specified user can't be found in the database tables") +                return +            else: +                resp.raise_for_status() + +        self.watched_users[user.id] = response_data +        await ctx.send(f":white_check_mark: Messages sent by {user} will now be relayed to the talent pool channel") + +    @nomination_group.command(name='history', aliases=('info', 'search')) +    @with_role(Roles.owner, Roles.admin, Roles.moderator) +    async def history_command(self, ctx: Context, user: Union[User, proxy_user]) -> None: +        """Shows the specified user's nomination history.""" +        result = await self.bot.api_client.get( +            self.api_endpoint, +            params={ +                'user__id': str(user.id), +                'ordering': "-active,-inserted_at" +            } +        ) +        if not result: +            await ctx.send(":warning: This user has never been nominated") +            return + +        embed = Embed( +            title=f"Nominations for {user.display_name} `({user.id})`", +            color=Color.blue() +        ) +        lines = [self._nomination_to_string(nomination) for nomination in result] +        await LinePaginator.paginate( +            lines, +            ctx=ctx, +            embed=embed, +            empty=True, +            max_lines=3, +            max_size=1000 +        ) + +    @nomination_group.command(name='unwatch', aliases=('end', )) +    @with_role(Roles.owner, Roles.admin, Roles.moderator) +    async def unwatch_command(self, ctx: Context, user: Union[User, proxy_user], *, reason: str) -> None: +        """ +        Ends the active nomination of the specified user with the given reason. + +        Providing a `reason` is required. +        """ +        active_nomination = await self.bot.api_client.get( +            self.api_endpoint, +            params=ChainMap( +                self.api_default_params, +                {"user__id": str(user.id)} +            ) +        ) + +        if not active_nomination: +            await ctx.send(":x: The specified user does not have an active nomination") +            return + +        [nomination] = active_nomination +        await self.bot.api_client.patch( +            f"{self.api_endpoint}/{nomination['id']}", +            json={'end_reason': reason, 'active': False} +        ) +        await ctx.send(f":white_check_mark: Messages sent by {user} will no longer be relayed") +        self._remove_user(user.id) + +    @nomination_group.group(name='edit', aliases=('e',), invoke_without_command=True) +    @with_role(Roles.owner, Roles.admin, Roles.moderator) +    async def nomination_edit_group(self, ctx: Context) -> None: +        """Commands to edit nominations.""" + +        await ctx.invoke(self.bot.get_command("help"), "talentpool", "edit") + +    @nomination_edit_group.command(name='reason') +    @with_role(Roles.owner, Roles.admin, Roles.moderator) +    async def edit_reason_command(self, ctx: Context, nomination_id: int, *, reason: str) -> None: +        """ +        Edits the reason/unnominate reason for the nomination with the given `id` depending on the status. + +        If the nomination is active, the reason for nominating the user will be edited; +        If the nomination is no longer active, the reason for ending the nomination will be edited instead. +        """ +        try: +            nomination = await self.bot.api_client.get(f"{self.api_endpoint}/{nomination_id}") +        except ClientResponseError as e: +            if e.status == 404: +                self.log.trace(f"Nomination API 404: Can't nomination with id {nomination_id}") +                await ctx.send(f":x: Can't find a nomination with id `{nomination_id}`") +                return +            else: +                raise + +        field = "reason" if nomination["active"] else "end_reason" + +        self.log.trace(f"Changing {field} for nomination with id {nomination_id} to {reason}") + +        await self.bot.api_client.patch( +            f"{self.api_endpoint}/{nomination_id}", +            json={field: reason} +        ) + +        await ctx.send(f":white_check_mark: Updated the {field} of the nomination!") + +    def _nomination_to_string(self, nomination_object: dict) -> str: +        """Creates a string representation of a nomination.""" +        guild = self.bot.get_guild(Guild.id) + +        actor_id = nomination_object["actor"] +        actor = guild.get_member(actor_id) + +        active = nomination_object["active"] +        log.debug(active) +        log.debug(type(nomination_object["inserted_at"])) + +        start_date = self._get_human_readable(nomination_object["inserted_at"]) +        if active: +            lines = textwrap.dedent( +                f""" +                =============== +                Status: **Active** +                Date: {start_date} +                Actor: {actor.mention if actor else actor_id} +                Reason: {nomination_object["reason"]} +                Nomination ID: `{nomination_object["id"]}` +                =============== +                """ +            ) +        else: +            end_date = self._get_human_readable(nomination_object["ended_at"]) +            lines = textwrap.dedent( +                f""" +                =============== +                Status: Inactive +                Date: {start_date} +                Actor: {actor.mention if actor else actor_id} +                Reason: {nomination_object["reason"]} + +                End date: {end_date} +                Unwatch reason: {nomination_object["end_reason"]} +                Nomination ID: `{nomination_object["id"]}` +                =============== +                """ +            ) + +        return lines.strip() diff --git a/bot/cogs/watchchannels/watchchannel.py b/bot/cogs/watchchannels/watchchannel.py new file mode 100644 index 000000000..fe6d6bb6e --- /dev/null +++ b/bot/cogs/watchchannels/watchchannel.py @@ -0,0 +1,353 @@ +import asyncio +import datetime +import logging +import re +import textwrap +from abc import ABC, abstractmethod +from collections import defaultdict, deque +from dataclasses import dataclass +from typing import Optional + +import aiohttp +import discord +from discord import Color, Embed, Message, Object, errors +from discord.ext.commands import BadArgument, Bot, Context + +from bot.cogs.modlog import ModLog +from bot.constants import BigBrother as BigBrotherConfig, Guild as GuildConfig, Icons +from bot.pagination import LinePaginator +from bot.utils import messages +from bot.utils.time import time_since + +log = logging.getLogger(__name__) + +URL_RE = re.compile(r"(https?://[^\s]+)") + + +def proxy_user(user_id: str) -> Object: +    """A proxy user object that mocks a real User instance for when the later is not available.""" +    try: +        user_id = int(user_id) +    except ValueError: +        raise BadArgument + +    user = Object(user_id) +    user.mention = user.id +    user.display_name = f"<@{user.id}>" +    user.avatar_url_as = lambda static_format: None +    user.bot = False + +    return user + + +@dataclass +class MessageHistory: +    last_author: Optional[int] = None +    last_channel: Optional[int] = None +    message_count: int = 0 + + +class WatchChannel(ABC): +    """ABC with functionality for relaying users' messages to a certain channel.""" + +    @abstractmethod +    def __init__(self, bot: Bot, destination, webhook_id, api_endpoint, api_default_params, logger) -> None: +        self.bot = bot + +        self.destination = destination  # E.g., Channels.big_brother_logs +        self.webhook_id = webhook_id  # E.g.,  Webhooks.big_brother +        self.api_endpoint = api_endpoint  # E.g., 'bot/infractions' +        self.api_default_params = api_default_params  # E.g., {'active': 'true', 'type': 'watch'} +        self.log = logger  # Logger of the child cog for a correct name in the logs + +        self._consume_task = None +        self.watched_users = defaultdict(dict) +        self.message_queue = defaultdict(lambda: defaultdict(deque)) +        self.consumption_queue = {} +        self.retries = 5 +        self.retry_delay = 10 +        self.channel = None +        self.webhook = None +        self.message_history = MessageHistory() + +        self._start = self.bot.loop.create_task(self.start_watchchannel()) + +    @property +    def modlog(self) -> ModLog: +        """Provides access to the ModLog cog for alert purposes.""" +        return self.bot.get_cog("ModLog") + +    @property +    def consuming_messages(self) -> bool: +        """Checks if a consumption task is currently running.""" +        if self._consume_task is None: +            return False + +        if self._consume_task.done(): +            exc = self._consume_task.exception() +            if exc: +                self.log.exception( +                    f"The message queue consume task has failed with:", +                    exc_info=exc +                ) +            return False + +        return True + +    async def start_watchchannel(self) -> None: +        """Starts the watch channel by getting the channel, webhook, and user cache ready.""" +        await self.bot.wait_until_ready() + +        # After updating d.py, this block can be replaced by `fetch_channel` with a try-except +        for attempt in range(1, self.retries+1): +            self.channel = self.bot.get_channel(self.destination) +            if self.channel is None: +                if attempt < self.retries: +                    await asyncio.sleep(self.retry_delay) +            else: +                break +        else: +            self.log.error(f"Failed to retrieve the text channel with id {self.destination}") + +        # `get_webhook_info` has been renamed to `fetch_webhook` in newer versions of d.py +        try: +            self.webhook = await self.bot.get_webhook_info(self.webhook_id) +        except (discord.HTTPException, discord.NotFound, discord.Forbidden): +            self.log.exception(f"Failed to fetch webhook with id `{self.webhook_id}`") + +        if self.channel is None or self.webhook is None: +            self.log.error("Failed to start the watch channel; unloading the cog.") + +            message = textwrap.dedent( +                f""" +                An error occurred while loading the text channel or webhook. + +                TextChannel: {"**Failed to load**" if self.channel is None else "Loaded successfully"} +                Webhook: {"**Failed to load**" if self.webhook is None else "Loaded successfully"} + +                The Cog has been unloaded. +                """ +            ) + +            await self.modlog.send_log_message( +                title=f"Error: Failed to initialize the {self.__class__.__name__} watch channel", +                text=message, +                ping_everyone=True, +                icon_url=Icons.token_removed, +                colour=Color.red() +            ) + +            self.bot.remove_cog(self.__class__.__name__) +            return + +        if not await self.fetch_user_cache(): +            await self.modlog.send_log_message( +                title=f"Warning: Failed to retrieve user cache for the {self.__class__.__name__} watch channel", +                text="Could not retrieve the list of watched users from the API and messages will not be relayed.", +                ping_everyone=True, +                icon=Icons.token_removed, +                color=Color.red() +            ) + +    async def fetch_user_cache(self) -> bool: +        """ +        Fetches watched users from the API and updates the watched user cache accordingly. + +        This function returns `True` if the update succeeded. +        """ +        try: +            data = await self.bot.api_client.get(self.api_endpoint, params=self.api_default_params) +        except aiohttp.ClientResponseError as e: +            self.log.exception(f"Failed to fetch the watched users from the API", exc_info=e) +            return False + +        self.watched_users = defaultdict(dict) + +        for entry in data: +            user_id = entry.pop('user') +            self.watched_users[user_id] = entry + +        return True + +    async def on_message(self, msg: Message) -> None: +        """Queues up messages sent by watched users.""" +        if msg.author.id in self.watched_users: +            if not self.consuming_messages: +                self._consume_task = self.bot.loop.create_task(self.consume_messages()) + +            self.log.trace(f"Received message: {msg.content} ({len(msg.attachments)} attachments)") +            self.message_queue[msg.author.id][msg.channel.id].append(msg) + +    async def consume_messages(self, delay_consumption: bool = True) -> None: +        """Consumes the message queues to log watched users' messages.""" +        if delay_consumption: +            self.log.trace(f"Sleeping {BigBrotherConfig.log_delay} seconds before consuming message queue") +            await asyncio.sleep(BigBrotherConfig.log_delay) + +        self.log.trace(f"Started consuming the message queue") + +        # If the previous consumption Task failed, first consume the existing comsumption_queue +        if not self.consumption_queue: +            self.consumption_queue = self.message_queue.copy() +            self.message_queue.clear() + +        for user_channel_queues in self.consumption_queue.values(): +            for channel_queue in user_channel_queues.values(): +                while channel_queue: +                    msg = channel_queue.popleft() + +                    self.log.trace(f"Consuming message {msg.id} ({len(msg.attachments)} attachments)") +                    await self.relay_message(msg) + +        self.consumption_queue.clear() + +        if self.message_queue: +            self.log.trace("Channel queue not empty: Continuing consuming queues") +            self._consume_task = self.bot.loop.create_task(self.consume_messages(delay_consumption=False)) +        else: +            self.log.trace("Done consuming messages.") + +    async def webhook_send( +        self, +        content: Optional[str] = None, +        username: Optional[str] = None, +        avatar_url: Optional[str] = None, +        embed: Optional[Embed] = None, +    ) -> None: +        """Sends a message to the webhook with the specified kwargs.""" +        try: +            await self.webhook.send(content=content, username=username, avatar_url=avatar_url, embed=embed) +        except discord.HTTPException as exc: +            self.log.exception( +                f"Failed to send a message to the webhook", +                exc_info=exc +            ) + +    async def relay_message(self, msg: Message) -> None: +        """Relays the message to the relevant watch channel.""" +        limit = BigBrotherConfig.header_message_limit + +        if ( +            msg.author.id != self.message_history.last_author +            or msg.channel.id != self.message_history.last_channel +            or self.message_history.message_count >= limit +        ): +            self.message_history = MessageHistory(last_author=msg.author.id, last_channel=msg.channel.id) + +            await self.send_header(msg) + +        cleaned_content = msg.clean_content + +        if cleaned_content: +            # Put all non-media URLs in a code block to prevent embeds +            media_urls = {embed.url for embed in msg.embeds if embed.type in ("image", "video")} +            for url in URL_RE.findall(cleaned_content): +                if url not in media_urls: +                    cleaned_content = cleaned_content.replace(url, f"`{url}`") +            await self.webhook_send( +                cleaned_content, +                username=msg.author.display_name, +                avatar_url=msg.author.avatar_url +            ) + +        if msg.attachments: +            try: +                await messages.send_attachments(msg, self.webhook) +            except (errors.Forbidden, errors.NotFound): +                e = Embed( +                    description=":x: **This message contained an attachment, but it could not be retrieved**", +                    color=Color.red() +                ) +                await self.webhook_send( +                    embed=e, +                    username=msg.author.display_name, +                    avatar_url=msg.author.avatar_url +                ) +            except discord.HTTPException as exc: +                self.log.exception( +                    f"Failed to send an attachment to the webhook", +                    exc_info=exc +                ) + +        self.message_history.message_count += 1 + +    async def send_header(self, msg) -> None: +        """Sends a header embed with information about the relayed messages to the watch channel.""" +        user_id = msg.author.id + +        guild = self.bot.get_guild(GuildConfig.id) +        actor = guild.get_member(self.watched_users[user_id]['actor']) +        actor = actor.display_name if actor else self.watched_users[user_id]['actor'] + +        inserted_at = self.watched_users[user_id]['inserted_at'] +        time_delta = self._get_time_delta(inserted_at) + +        reason = self.watched_users[user_id]['reason'] + +        embed = Embed(description=f"{msg.author.mention} in [#{msg.channel.name}]({msg.jump_url})") +        embed.set_footer(text=f"Added {time_delta} by {actor} | Reason: {reason}") + +        await self.webhook_send(embed=embed, username=msg.author.display_name, avatar_url=msg.author.avatar_url) + +    async def list_watched_users(self, ctx: Context, update_cache: bool = True) -> None: +        """ +        Gives an overview of the watched user list for this channel. + +        The optional kwarg `update_cache` specifies whether the cache should +        be refreshed by polling the API. +        """ +        if update_cache: +            if not await self.fetch_user_cache(): +                await ctx.send(f":x: Failed to update {self.__class__.__name__} user cache, serving from cache") +                update_cache = False + +        lines = [] +        for user_id, user_data in self.watched_users.items(): +            inserted_at = user_data['inserted_at'] +            time_delta = self._get_time_delta(inserted_at) +            lines.append(f"• <@{user_id}> (added {time_delta})") + +        lines = lines or ("There's nothing here yet.",) +        embed = Embed( +            title=f"{self.__class__.__name__} watched users ({'updated' if update_cache else 'cached'})", +            color=Color.blue() +        ) +        await LinePaginator.paginate(lines, ctx, embed, empty=False) + +    @staticmethod +    def _get_time_delta(time_string: str) -> str: +        """Returns the time in human-readable time delta format.""" +        date_time = datetime.datetime.strptime( +            time_string, +            "%Y-%m-%dT%H:%M:%S.%fZ" +        ).replace(tzinfo=None) +        time_delta = time_since(date_time, precision="minutes", max_units=1) + +        return time_delta + +    @staticmethod +    def _get_human_readable(time_string: str, output_format: str = "%Y-%m-%d %H:%M:%S") -> str: +        date_time = datetime.datetime.strptime( +            time_string, +            "%Y-%m-%dT%H:%M:%S.%fZ" +        ).replace(tzinfo=None) +        return date_time.strftime(output_format) + +    def _remove_user(self, user_id: int) -> None: +        """Removes a user from a watch channel.""" +        self.watched_users.pop(user_id, None) +        self.message_queue.pop(user_id, None) +        self.consumption_queue.pop(user_id, None) + +    def cog_unload(self) -> None: +        """Takes care of unloading the cog and canceling the consumption task.""" +        self.log.trace(f"Unloading the cog") +        if not self._consume_task.done(): +            self._consume_task.cancel() +            try: +                self._consume_task.result() +            except asyncio.CancelledError as e: +                self.log.exception( +                    f"The consume task was canceled. Messages may be lost.", +                    exc_info=e +                ) diff --git a/bot/constants.py b/bot/constants.py index d2c953276..033d19f7b 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -355,6 +355,14 @@ class Channels(metaclass=YAMLGetter):      verification: int +class Webhooks(metaclass=YAMLGetter): +    section = "guild" +    subsection = "webhooks" + +    talent_pool: int +    big_brother: int + +  class Roles(metaclass=YAMLGetter):      section = "guild"      subsection = "roles" @@ -389,18 +397,12 @@ class Keys(metaclass=YAMLGetter):      site_api: str -class RabbitMQ(metaclass=YAMLGetter): -    section = "rabbitmq" - -    host: str -    password: str -    port: int -    username: str - -  class URLs(metaclass=YAMLGetter):      section = "urls" +    # Snekbox endpoints +    snekbox_eval_api: str +      # Discord API endpoints      discord_api: str      discord_invite_api: str diff --git a/bot/utils/messages.py b/bot/utils/messages.py index fc38b0127..94a8b36ed 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -1,9 +1,9 @@  import asyncio  import contextlib  from io import BytesIO -from typing import Sequence +from typing import Sequence, Union -from discord import Embed, File, Message, TextChannel +from discord import Embed, File, Message, TextChannel, Webhook  from discord.abc import Snowflake  from discord.errors import HTTPException @@ -78,9 +78,9 @@ async def wait_for_deletion(          await message.delete() -async def send_attachments(message: Message, destination: TextChannel): +async def send_attachments(message: Message, destination: Union[TextChannel, Webhook]):      """ -    Re-uploads each attachment in a message to the given channel. +    Re-uploads each attachment in a message to the given channel or webhook.      Each attachment is sent as a separate message to more easily comply with the 8 MiB request size limit.      If attachments are too large, they are instead grouped into a single embed which links to them. @@ -97,7 +97,16 @@ async def send_attachments(message: Message, destination: TextChannel):              if attachment.size <= MAX_SIZE - 512:                  with BytesIO() as file:                      await attachment.save(file) -                    await destination.send(file=File(file, filename=attachment.filename)) +                    attachment_file = File(file, filename=attachment.filename) + +                    if isinstance(destination, TextChannel): +                        await destination.send(file=attachment_file) +                    else: +                        await destination.send( +                            file=attachment_file, +                            username=message.author.display_name, +                            avatar_url=message.author.avatar_url +                        )              else:                  large.append(attachment)          except HTTPException as e: @@ -109,4 +118,11 @@ async def send_attachments(message: Message, destination: TextChannel):      if large:          embed = Embed(description=f"\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large))          embed.set_footer(text="Attachments exceed upload size limit.") -        await destination.send(embed=embed) +        if isinstance(destination, TextChannel): +            await destination.send(embed=embed) +        else: +            await destination.send( +                embed=embed, +                username=message.author.display_name, +                avatar_url=message.author.avatar_url +            ) diff --git a/bot/utils/moderation.py b/bot/utils/moderation.py index fcdf3c4d5..c1eb98dd6 100644 --- a/bot/utils/moderation.py +++ b/bot/utils/moderation.py @@ -14,8 +14,13 @@ HEADERS = {"X-API-KEY": Keys.site_api}  async def post_infraction( -    ctx: Context, user: Union[Member, Object, User], -    type: str, reason: str, expires_at: datetime = None, hidden: bool = False +    ctx: Context, +    user: Union[Member, Object, User], +    type: str, +    reason: str, +    expires_at: datetime = None, +    hidden: bool = False, +    active: bool = True,  ):      payload = { @@ -23,7 +28,8 @@ async def post_infraction(          "hidden": hidden,          "reason": reason,          "type": type, -        "user": user.id +        "user": user.id, +        "active": active      }      if expires_at:          payload['expires_at'] = expires_at.isoformat() diff --git a/bot/utils/service_discovery.py b/bot/utils/service_discovery.py deleted file mode 100644 index 8d79096bd..000000000 --- a/bot/utils/service_discovery.py +++ /dev/null @@ -1,22 +0,0 @@ -import datetime -import socket -import time -from contextlib import closing - -from bot.constants import RabbitMQ - -THIRTY_SECONDS = datetime.timedelta(seconds=30) - - -def wait_for_rmq(): -    start = datetime.datetime.now() - -    while True: -        if datetime.datetime.now() - start > THIRTY_SECONDS: -            return False - -        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: -            if sock.connect_ex((RabbitMQ.host, RabbitMQ.port)) == 0: -                return True - -        time.sleep(0.5) diff --git a/config-default.yml b/config-default.yml index d8016875b..dd5126230 100644 --- a/config-default.yml +++ b/config-default.yml @@ -135,6 +135,10 @@ guild:          rockstars:         &ROCKSTARS_ROLE  458226413825294336          team_leader:                        501324292341104650 +    webhooks: +        talent_pool:                        569145364800602132 +        big_brother:                        569133704568373283 +  filter: @@ -165,6 +169,8 @@ filter:          - 249111029668249601  # Gentoo          - 327254708534116352  # Adafruit          - 544525886180032552  # kennethreitz.org +        - 590806733924859943  # Discord Hack Week +        - 423249981340778496  # Kivy      domain_blacklist:          - pornhub.com @@ -224,13 +230,6 @@ keys:      site_api:    !ENV "BOT_API_KEY" -rabbitmq: -    host:          "pdrmq" -    password: !ENV ["RABBITMQ_DEFAULT_PASS", "guest"] -    port:          5672 -    username: !ENV ["RABBITMQ_DEFAULT_USER", "guest"] - -  urls:      # PyDis site vars      site:        &DOMAIN       "pythondiscord.com" @@ -258,6 +257,9 @@ urls:      site_user_complete_api:             !JOIN [*SCHEMA, *API, "/bot/users/complete"]      paste_service:                      !JOIN [*SCHEMA, *PASTE, "/{key}"] +    # Snekbox +    snekbox_eval_api: "http://localhost:8060/eval" +      # Env vars      deploy: !ENV "DEPLOY_URL"      status: !ENV "STATUS_URL" | 
