diff options
41 files changed, 2616 insertions, 638 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 88ab5d927..f7aee8165 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,15 @@ image: pythondiscord/bot-ci:latest +variables: + PIPENV_CACHE_DIR: "/root/.cache/pipenv" + PIP_CACHE_DIR: "/root/.cache/pip" + +cache: + paths: + - "/root/.cache/pip/" + - "/root/.cache/pipenv/" + - "/usr/local/lib/python3.6/site-packages/" + stages: - test - build @@ -11,8 +21,10 @@ test: stage: test script: - - pipenv install --dev --deploy - - pipenv run lint + - ls /root/.cache/ + - pipenv install --dev --deploy --system + - python -m flake8 + - ls /root/.cache/ build: tags: @@ -18,7 +18,6 @@ lxml = "*" pyyaml = "*" yarl = "==1.1.1" fuzzywuzzy = "*" -python-levenshtein = "*" pillow = "*" aio-pika = "*" python-dateutil = "*" @@ -41,9 +40,12 @@ python_version = "3.6" [scripts] start = "python -m bot" lint = "python -m flake8" -build = "docker build -t pythondiscord/bot:latest -f docker/Dockerfile ." + +build = "docker build -t pythondiscord/bot:latest -f docker/bot.Dockerfile ." push = "docker push pythondiscord/bot:latest" -buildbase = "docker build -t pythondiscord/bot-base:latest -f docker/Dockerfile.base ." + +buildbase = "docker build -t pythondiscord/bot-base:latest -f docker/base.Dockerfile ." pushbase = "docker push pythondiscord/bot-base:latest" + buildci = "docker build -t pythondiscord/bot-ci:latest -f docker/ci.Dockerfile ." pushci = "docker push pythondiscord/bot-ci:latest" diff --git a/Pipfile.lock b/Pipfile.lock index 864eb574a..8b43235bb 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c7d1bad1549c322484f6751447115ded9299df039cb6321bcc1a1fa1359481dc" + "sha256": "9c22a342245c638b196b519a8afb8a2c66410d76283746cfdd89f19ff7dce94c" }, "pipfile-spec": 6, "requires": { @@ -74,11 +74,13 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:11a9a27b7d3bddc6d86f59fb76afb70e921a25ac2d6cc55b40d072bd68435a76", - "sha256:7015e76bf32f1f574636c4288399a6de66ce08fb7b2457f628a8d70c0fbabb11", - "sha256:808b6ac932dccb0a4126558f7dfdcf41710dd44a4ef497a0bb59a77f9f078e89" + "sha256:2545357585a6cc7d050d3c43a86eba2c0b91b9e7ac8a3965e64a6ead6a1a9a3d", + "sha256:272081ad78c5495ba67083a0e50920163701fa6fe67fbb5eefeb21b5dd88c40b", + "sha256:4ddc90ad88bccc005a71d8ef32f7b1cd8f935475cd561c4122b2f87de45d28ab", + "sha256:5a3d659840960a4107047b6328d6d4cdaaee69939bf11adc07466a1856c99a80", + "sha256:bd43a3b26d2886acd63070c43da821b60dea603eb6d45bab0294aac6129adbfa" ], - "version": "==4.6.0" + "version": "==4.6.1" }, "certifi": { "hashes": [ @@ -247,18 +249,30 @@ "sha256:026449b64e559226cdb8e6d8c931b5965d8fc90ec18ebbb0baa04c5b36503c72", "sha256:03dbb224ee196ef30ed2156d41b579143e1efeb422974719a5392fc035e4f574", "sha256:03eb0e04f929c102ae24bc436bf1c0c60a4e63b07ebd388e84d8b219df3e6acd", + "sha256:087b0551ce2d19b3f092f2b5f071a065f7379e748867d070b29999cc83db15e3", + "sha256:091a0656688d85fd6e10f49a73fa3ab9b37dbfcb2151f5a3ab17f8b879f467ee", + "sha256:0f3e2d0a9966161b7dfd06d147f901d72c3a88ea1a833359b92193b8e1f68e1c", + "sha256:114398d0e073b93e1d7da5b5ab92ff4b83c0180625c8031911425e51f4365d2e", "sha256:1be66b9a89e367e7d20d6cae419794997921fe105090fafd86ef39e20a3baab2", + "sha256:1c5e93c40d4ce8cb133d3b105a869be6fa767e703f6eb1003eb4b90583e08a59", "sha256:1e977a3ed998a599bda5021fb2c2889060617627d3ae228297a529a082a3cd5c", "sha256:22cf3406d135cfcc13ec6228ade774c8461e125c940e80455f500638429be273", "sha256:24adccf1e834f82718c7fc8e3ec1093738da95144b8b1e44c99d5fc7d3e9c554", "sha256:2a3e362c97a5e6a259ee9cd66553292a1f8928a5bdfa3622fdb1501570834612", + "sha256:3518f9fc666cbc58a5c1f48a6a23e9e6ceef69665eab43cdad5144de9383e72c", + "sha256:3709339f4619e8c9b00f53079e40b964f43c5af61fb89a923fe24437167298bb", "sha256:3832e26ecbc9d8a500821e3a1d3765bda99d04ae29ffbb2efba49f5f788dc934", + "sha256:452d159024faf37cc080537df308e8fa0026076eb38eb75185d96ed9642bd6d7", "sha256:4fd1f0c2dc02aaec729d91c92cd85a2df0289d88e9f68d1e8faba750bb9c4786", "sha256:4fda62030f2c515b6e2e673c57caa55cb04026a81968f3128aae10fc28e5cc27", "sha256:5044d75a68b49ce36a813c82d8201384207112d5d81643937fc758c05302f05b", "sha256:522184556921512ec484cb93bd84e0bab915d0ac5a372d49571c241a7f73db62", "sha256:5914cff11f3e920626da48e564be6818831713a3087586302444b9c70e8552d9", + "sha256:653d48fe46378f40e3c2b892be88d8440efbb2c9df78559da44c63ad5ecb4142", "sha256:6661a7908d68c4a133e03dac8178287aa20a99f841ea90beeb98a233ae3fd710", + "sha256:6735a7e560df6f0deb78246a6fe056cf2ae392ba2dc060ea8a6f2535aec924f1", + "sha256:6d26a475a19cb294225738f5c974b3a24599438a67a30ed2d25638f012668026", + "sha256:791f07fe13937e65285f9ef30664ddf0e10a0230bdb236751fa0ca67725740dd", "sha256:79258a8df3e309a54c7ef2ef4a59bb8e28f7e4a8992a3ad17c24b1889ced44f3", "sha256:7d74c20b8f1c3e99d3f781d3b8ff5abfefdd7363d61e23bdeba9992ff32cc4b4", "sha256:81918afeafc16ba5d9d0d4e9445905f21aac969a4ebb6f2bff4b9886da100f4b", @@ -266,12 +280,20 @@ "sha256:84d5d31200b11b3c76fab853b89ac898bf2d05c8b3da07c1fcc23feb06359d6e", "sha256:989981db57abffb52026b114c9a1f114c7142860a6d30a352d28f8cbf186500b", "sha256:a3d7511d3fad1618a82299aab71a5fceee5c015653a77ffea75ced9ef917e71a", + "sha256:a4a6ac01b8c2f9d2d83719f193e6dea493e18445ce5bfd743d739174daa974d9", + "sha256:acb90eb6c7ed6526551a78211d84c81e33082a35642ff5fe57489abc14e6bf6e", "sha256:b3ef168d4d6fd4fa6685aef7c91400f59f7ab1c0da734541f7031699741fb23f", "sha256:c1c5792b6e74bbf2af0f8e892272c2a6c48efa895903211f11b8342e03129fea", "sha256:c5dcb5a56aebb8a8f2585042b2f5c496d7624f0bcfe248f0cc33ceb2fd8d39e7", + "sha256:d16f90810106822833a19bdb24c7cb766959acf791ca0edf5edfec674d55c8ee", + "sha256:dcdc9cd9880027688007ff8f7c8e7ae6f24e81fae33bfd18d1e691e7bda4855f", + "sha256:e2807aad4565d8de15391a9548f97818a14ef32624015c7bf3095171e314445e", "sha256:e2bed4a04e2ca1050bb5f00865cf2f83c0b92fd62454d9244f690fcd842e27a4", "sha256:e87a527c06319428007e8c30511e1f0ce035cb7f14bb4793b003ed532c3b9333", + "sha256:ebcfc33a6c34984086451e230253bc33727bd17b4cdc4b39ec03032c3a6fc9e9", "sha256:f63e420180cbe22ff6e32558b612e75f50616fc111c5e095a4631946c782e109", + "sha256:f7717eb360d40e7598c30cc44b33d98f79c468d9279379b66c1e28c568e0bf47", + "sha256:f8582e1ab155302ea9ef1235441a0214919f4f79c4c7c21833ce9eec58181781", "sha256:f8b3d413c5a8f84b12cd4c5df1d8e211777c9852c6be3ee9c094b626644d3eab" ], "index": "pypi", @@ -313,6 +335,11 @@ "pyparsing": { "hashes": [ "sha256:0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04", + "sha256:281683241b25fe9b80ec9d66017485f6deff1af5cde372469134b56ca8447a07", + "sha256:8f1e18d3fd36c6795bb7e02a39fd05c611ffc2596c1e0d995d34d67630426c18", + "sha256:9e8143a3e15c13713506886badd96ca4b579a87fbdf49e550dbfc057d6cb218e", + "sha256:b8b3117ed9bdf45e14dcc89345ce638ec7e0e29b2b579fa1ecf32ce45ebac8a5", + "sha256:e4d45427c6e20a59bf4f88c639dcc03ce30d193112047f94012102f235853a58", "sha256:fee43f17a9c4087e7ed1605bd6df994c6173c1e977d7ade7b651292fab2bd010" ], "version": "==2.2.0" @@ -330,16 +357,8 @@ "sha256:a292e22c5e03105a05a746ade6209d43db1c4c763b91c75c8486e81d10904d85", "sha256:e3636824d35ba6a15fc39f573588cba63cf46322a5dc86fb2f280229077e9fbe" ], - "markers": "python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.2.*'", "version": "==0.1.9" }, - "python-levenshtein": { - "hashes": [ - "sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1" - ], - "index": "pypi", - "version": "==0.12.0" - }, "pytz": { "hashes": [ "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", @@ -404,7 +423,6 @@ "sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd", "sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9" ], - "markers": "python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.2.*'", "version": "==1.1.0" }, "sympy": { @@ -419,7 +437,6 @@ "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" ], - "markers": "python_version != '3.0.*' and python_version < '4' and python_version != '3.3.*' and python_version != '3.2.*' and python_version >= '2.6' and python_version != '3.1.*'", "version": "==1.23" }, "websockets": { @@ -592,6 +609,11 @@ "pyparsing": { "hashes": [ "sha256:0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04", + "sha256:281683241b25fe9b80ec9d66017485f6deff1af5cde372469134b56ca8447a07", + "sha256:8f1e18d3fd36c6795bb7e02a39fd05c611ffc2596c1e0d995d34d67630426c18", + "sha256:9e8143a3e15c13713506886badd96ca4b579a87fbdf49e550dbfc057d6cb218e", + "sha256:b8b3117ed9bdf45e14dcc89345ce638ec7e0e29b2b579fa1ecf32ce45ebac8a5", + "sha256:e4d45427c6e20a59bf4f88c639dcc03ce30d193112047f94012102f235853a58", "sha256:fee43f17a9c4087e7ed1605bd6df994c6173c1e977d7ade7b651292fab2bd010" ], "version": "==2.2.0" @@ -622,11 +644,11 @@ }, "safety": { "hashes": [ - "sha256:32d41b8bbd736db749aa2162de6c0bb11c2113c7bc0357476491f96cd5d58299", - "sha256:34227360409ffb1bc2657e5b6ff3472a32d72b917617cd3d2914ddf078c263b9" + "sha256:2689fe629bafe9450796d36578aa112820ff65038578aee004f60b9db1ba4ae8", + "sha256:cd04e57ff8cf8984ff2cb11973e1d5469dae681e25d4edfccb1ef08cc107b2c0" ], "index": "pypi", - "version": "==1.8.2" + "version": "==1.8.3" }, "six": { "hashes": [ @@ -640,7 +662,6 @@ "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" ], - "markers": "python_version != '3.0.*' and python_version < '4' and python_version != '3.3.*' and python_version != '3.2.*' and python_version >= '2.6' and python_version != '3.1.*'", "version": "==1.23" } } diff --git a/bot/__init__.py b/bot/__init__.py index a87d31541..5a446d71c 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -88,9 +88,12 @@ for key, value in logging.Logger.manager.loggerDict.items(): value.addHandler(handler) -# Silence discord and websockets +# Silence aio_pika.pika.{callback,channel}, discord, PIL, and and websockets +logging.getLogger("aio_pika.pika.callback").setLevel(logging.ERROR) +logging.getLogger("aio_pika.pika.channel").setLevel(logging.ERROR) logging.getLogger("discord.client").setLevel(logging.ERROR) logging.getLogger("discord.gateway").setLevel(logging.ERROR) logging.getLogger("discord.state").setLevel(logging.ERROR) logging.getLogger("discord.http").setLevel(logging.ERROR) +logging.getLogger("PIL.PngImagePlugin").setLevel(logging.ERROR) logging.getLogger("websockets.protocol").setLevel(logging.ERROR) diff --git a/bot/__main__.py b/bot/__main__.py index b9e6001ac..30d1b4c9a 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -5,7 +5,7 @@ from aiohttp import AsyncResolver, ClientSession, TCPConnector from discord import Game from discord.ext.commands import Bot, when_mentioned_or -from bot.constants import Bot as BotConfig # , ClickUp +from bot.constants import Bot as BotConfig, DEBUG_MODE from bot.utils.service_discovery import wait_for_rmq @@ -38,36 +38,40 @@ else: # Internal/debug bot.load_extension("bot.cogs.logging") -bot.load_extension("bot.cogs.modlog") bot.load_extension("bot.cogs.security") bot.load_extension("bot.cogs.events") +bot.load_extension("bot.cogs.filtering") +bot.load_extension("bot.cogs.modlog") # Commands, etc +bot.load_extension("bot.cogs.antispam") bot.load_extension("bot.cogs.bigbrother") bot.load_extension("bot.cogs.bot") +bot.load_extension("bot.cogs.clean") bot.load_extension("bot.cogs.cogs") -# Local setups usually don't have the clickup key set, -# and loading the cog would simply spam errors in the console. -# if ClickUp.key is not None: -# bot.load_extension("bot.cogs.clickup") -# else: -# log.info("`CLICKUP_KEY` not set in the environment, not loading the ClickUp cog.") +# Only load this in production +if not DEBUG_MODE: + bot.load_extension("bot.cogs.verification") + +# Feature cogs bot.load_extension("bot.cogs.deployment") bot.load_extension("bot.cogs.defcon") bot.load_extension("bot.cogs.doc") bot.load_extension("bot.cogs.eval") bot.load_extension("bot.cogs.fun") bot.load_extension("bot.cogs.hiphopify") +bot.load_extension("bot.cogs.information") bot.load_extension("bot.cogs.moderation") bot.load_extension("bot.cogs.off_topic_names") +bot.load_extension("bot.cogs.reddit") +bot.load_extension("bot.cogs.site") bot.load_extension("bot.cogs.snakes") bot.load_extension("bot.cogs.snekbox") bot.load_extension("bot.cogs.tags") bot.load_extension("bot.cogs.token_remover") bot.load_extension("bot.cogs.utils") -bot.load_extension("bot.cogs.verification") if has_rmq: bot.load_extension("bot.cogs.rmq") diff --git a/bot/cogs/antispam.py b/bot/cogs/antispam.py new file mode 100644 index 000000000..7a33ba9e8 --- /dev/null +++ b/bot/cogs/antispam.py @@ -0,0 +1,203 @@ +import asyncio +import logging +import textwrap +from datetime import datetime, timedelta +from typing import List + +from dateutil.relativedelta import relativedelta +from discord import Colour, Member, Message, Object, TextChannel +from discord.ext.commands import Bot + +from bot import rules +from bot.cogs.modlog import ModLog +from bot.constants import ( + AntiSpam as AntiSpamConfig, Channels, + Colours, DEBUG_MODE, Event, + Guild as GuildConfig, Icons, Roles, +) +from bot.utils.time import humanize_delta + + +log = logging.getLogger(__name__) + +RULE_FUNCTION_MAPPING = { + 'attachments': rules.apply_attachments, + 'burst': rules.apply_burst, + 'burst_shared': rules.apply_burst_shared, + 'chars': rules.apply_chars, + 'discord_emojis': rules.apply_discord_emojis, + 'duplicates': rules.apply_duplicates, + 'links': rules.apply_links, + 'mentions': rules.apply_mentions, + 'newlines': rules.apply_newlines, + 'role_mentions': rules.apply_role_mentions +} +WHITELISTED_CHANNELS = ( + Channels.admins, Channels.announcements, Channels.big_brother_logs, + Channels.devalerts, Channels.devlog, Channels.devtest, + Channels.helpers, Channels.message_log, + Channels.mod_alerts, Channels.modlog, Channels.staff_lounge +) +WHITELISTED_ROLES = (Roles.owner, Roles.admin, Roles.moderator, Roles.helpers) + + +class AntiSpam: + def __init__(self, bot: Bot): + self.bot = bot + self.muted_role = None + + @property + def mod_log(self) -> ModLog: + return self.bot.get_cog("ModLog") + + async def on_ready(self): + role_id = AntiSpamConfig.punishment['role_id'] + self.muted_role = Object(role_id) + + async def on_message(self, message: Message): + if ( + message.guild.id != GuildConfig.id + or message.author.bot + or (message.channel.id in WHITELISTED_CHANNELS and not DEBUG_MODE) + or (message.author.top_role.id in WHITELISTED_ROLES and not DEBUG_MODE) + ): + return + + # Fetch the rule configuration with the highest rule interval. + max_interval_config = max( + AntiSpamConfig.rules.values(), + key=lambda config: config['interval'] + ) + max_interval = max_interval_config['interval'] + + # Store history messages since `interval` seconds ago in a list to prevent unnecessary API calls. + earliest_relevant_at = datetime.utcnow() - timedelta(seconds=max_interval) + relevant_messages = [ + msg async for msg in message.channel.history(after=earliest_relevant_at, reverse=False) + ] + + for rule_name in AntiSpamConfig.rules: + rule_config = AntiSpamConfig.rules[rule_name] + rule_function = RULE_FUNCTION_MAPPING[rule_name] + + # Create a list of messages that were sent in the interval that the rule cares about. + latest_interesting_stamp = datetime.utcnow() - timedelta(seconds=rule_config['interval']) + messages_for_rule = [ + msg for msg in relevant_messages if msg.created_at > latest_interesting_stamp + ] + result = await rule_function(message, messages_for_rule, rule_config) + + # If the rule returns `None`, that means the message didn't violate it. + # If it doesn't, it returns a tuple in the form `(str, Iterable[discord.Member])` + # which contains the reason for why the message violated the rule and + # an iterable of all members that violated the rule. + if result is not None: + reason, members, relevant_messages = result + full_reason = f"`{rule_name}` rule: {reason}" + for member in members: + + # Fire it off as a background task to ensure + # that the sleep doesn't block further tasks + self.bot.loop.create_task( + self.punish(message, member, full_reason, relevant_messages) + ) + + await self.maybe_delete_messages(message.channel, relevant_messages) + break + + async def punish(self, msg: Message, member: Member, reason: str, messages: List[Message]): + # Sanity check to ensure we're not lagging behind + if self.muted_role not in member.roles: + remove_role_after = AntiSpamConfig.punishment['remove_after'] + duration_delta = relativedelta(seconds=remove_role_after) + human_duration = humanize_delta(duration_delta) + + mod_alert_message = ( + f"**Triggered by:** {member.display_name}#{member.discriminator} (`{member.id}`)\n" + f"**Channel:** {msg.channel.mention}\n" + f"**Reason:** {reason}\n" + ) + + # For multiple messages, use the logs API + if len(messages) > 1: + url = await self.mod_log.upload_log(messages) + mod_alert_message += f"A complete log of the offending messages can be found [here]({url})" + else: + mod_alert_message += "Message:\n" + content = messages[0].clean_content + remaining_chars = 2040 - len(mod_alert_message) + + if len(content) > remaining_chars: + content = content[:remaining_chars] + "..." + + mod_alert_message += f"{content}" + + await self.mod_log.send_log_message( + icon_url=Icons.filtering, + colour=Colour(Colours.soft_red), + title=f"Spam detected!", + text=mod_alert_message, + thumbnail=msg.author.avatar_url_as(static_format="png"), + channel_id=Channels.mod_alerts, + ping_everyone=AntiSpamConfig.ping_everyone + ) + + await member.add_roles(self.muted_role, reason=reason) + description = textwrap.dedent(f""" + **Channel**: {msg.channel.mention} + **User**: {msg.author.mention} (`{msg.author.id}`) + **Reason**: {reason} + Role will be removed after {human_duration}. + """) + + await self.mod_log.send_log_message( + icon_url=Icons.user_mute, colour=Colour(Colours.soft_red), + title="User muted", text=description + ) + + await asyncio.sleep(remove_role_after) + await member.remove_roles(self.muted_role, reason="AntiSpam mute expired") + + await self.mod_log.send_log_message( + icon_url=Icons.user_mute, colour=Colour(Colours.soft_green), + title="User unmuted", + text=f"Was muted by `AntiSpam` cog for {human_duration}." + ) + + async def maybe_delete_messages(self, channel: TextChannel, messages: List[Message]): + # Is deletion of offending messages actually enabled? + if AntiSpamConfig.clean_offending: + + # If we have more than one message, we can use bulk delete. + if len(messages) > 1: + message_ids = [message.id for message in messages] + self.mod_log.ignore(Event.message_delete, *message_ids) + await channel.delete_messages(messages) + + # Otherwise, the bulk delete endpoint will throw up. + # Delete the message directly instead. + else: + self.mod_log.ignore(Event.message_delete, messages[0].id) + await messages[0].delete() + + +def validate_config(): + for name, config in AntiSpamConfig.rules.items(): + if name not in RULE_FUNCTION_MAPPING: + raise ValueError( + f"Unrecognized antispam rule `{name}`. " + f"Valid rules are: {', '.join(RULE_FUNCTION_MAPPING)}" + ) + + for required_key in ('interval', 'max'): + if required_key not in config: + raise ValueError( + f"`{required_key}` is required but was not " + f"set in rule `{name}`'s configuration." + ) + + +def setup(bot: Bot): + validate_config() + bot.add_cog(AntiSpam(bot)) + log.info("Cog loaded: AntiSpam") diff --git a/bot/cogs/bot.py b/bot/cogs/bot.py index 2f8600c06..fcc642313 100644 --- a/bot/cogs/bot.py +++ b/bot/cogs/bot.py @@ -3,12 +3,12 @@ import logging import re import time -from discord import Embed, Message +from discord import Embed, Member, Message, Reaction from discord.ext.commands import Bot, Context, command, group from dulwich.repo import Repo from bot.constants import ( - Channels, Guild, Roles, URLs + Channels, Emojis, Guild, Roles, URLs ) from bot.decorators import with_role @@ -40,6 +40,9 @@ class Bot: Channels.devtest, ) + # Stores improperly formatted Python codeblock message ids and the corresponding bot message + self.codeblock_message_ids = {} + @group(invoke_without_command=True, name="bot", hidden=True) @with_role(Roles.verified) async def bot_group(self, ctx: Context): @@ -168,6 +171,7 @@ class Bot: """ Attempts to fix badly indented code. """ + def unindent(code, skip_spaces=0): """ Unindents all code down to the number of spaces given ins skip_spaces @@ -178,7 +182,7 @@ class Bot: # Get numbers of spaces before code in the first line. while current == " ": - current = code[leading_spaces+1] + current = code[leading_spaces + 1] leading_spaces += 1 leading_spaces -= skip_spaces @@ -225,6 +229,16 @@ class Bot: log.trace(f"Found REPL code in \n\n{msg}\n\n") return final.rstrip(), True + def has_bad_ticks(self, msg: Message): + not_backticks = [ + "'''", '"""', "\u00b4\u00b4\u00b4", "\u2018\u2018\u2018", "\u2019\u2019\u2019", + "\u2032\u2032\u2032", "\u201c\u201c\u201c", "\u201d\u201d\u201d", "\u2033\u2033\u2033", + "\u3003\u3003\u3003" + ] + + has_bad_ticks = msg.content[:3] in not_backticks + return has_bad_ticks + async def on_message(self, msg: Message): """ Detect poorly formatted Python code and send the user @@ -245,14 +259,7 @@ class Bot: on_cooldown = (time.time() - self.channel_cooldowns.get(msg.channel.id, 0)) < 300 if not on_cooldown: try: - not_backticks = [ - "'''", '"""', "\u00b4\u00b4\u00b4", "\u2018\u2018\u2018", "\u2019\u2019\u2019", - "\u2032\u2032\u2032", "\u201c\u201c\u201c", "\u201d\u201d\u201d", "\u2033\u2033\u2033", - "\u3003\u3003\u3003" - ] - - bad_ticks = msg.content[:3] in not_backticks - if bad_ticks: + if self.has_bad_ticks(msg): ticks = msg.content[:3] content = self.codeblock_stripping(f"```{msg.content[3:-3]}```", True) if content is None: @@ -270,7 +277,7 @@ class Bot: current_length = 0 lines_walked = 0 for line in content.splitlines(keepends=True): - if current_length+len(line) > space_left or lines_walked == 10: + if current_length + len(line) > space_left or lines_walked == 10: break current_length += len(line) lines_walked += 1 @@ -311,11 +318,11 @@ class Bot: current_length = 0 lines_walked = 0 for line in content.splitlines(keepends=True): - if current_length+len(line) > space_left or lines_walked == 10: + if current_length + len(line) > space_left or lines_walked == 10: break current_length += len(line) lines_walked += 1 - content = content[:current_length]+"#..." + content = content[:current_length] + "#..." howto += ( "It looks like you're trying to paste code into this channel.\n\n" @@ -334,7 +341,9 @@ class Bot: if howto != "": howto_embed = Embed(description=howto) - await msg.channel.send(f"Hey {msg.author.mention}!", embed=howto_embed) + bot_message = await msg.channel.send(f"Hey {msg.author.mention}!", embed=howto_embed) + self.codeblock_message_ids[msg.id] = bot_message.id + await bot_message.add_reaction(Emojis.cross_mark) else: return @@ -348,6 +357,43 @@ class Bot: f"The message that was posted was:\n\n{msg.content}\n\n" ) + async def on_message_edit(self, before: Message, after: Message): + has_fixed_codeblock = ( + # Checks if the original message was previously called out by the bot + before.id in self.codeblock_message_ids + # Checks to see if the user has corrected their codeblock + and self.codeblock_stripping(after.content, self.has_bad_ticks(after)) is None + ) + if has_fixed_codeblock: + bot_message = await after.channel.get_message(self.codeblock_message_ids[after.id]) + await bot_message.delete() + del self.codeblock_message_ids[after.id] + + async def on_reaction_add(self, reaction: Reaction, user: Member): + # Ignores reactions added by the bot or added to non-codeblock correction embed messages + if user.bot or reaction.message.id not in self.codeblock_message_ids.values(): + return + + # Finds the appropriate bot message/ user message pair and assigns them to variables + for user_message_id, bot_message_id in self.codeblock_message_ids.items(): + if bot_message_id == reaction.message.id: + user_message = await reaction.message.channel.get_message(user_message_id) + bot_message = await reaction.message.channel.get_message(bot_message_id) + break + + # If the reaction was clicked on by the author of the user message, deletes the bot message + if user.id == user_message.author.id: + await bot_message.delete() + del self.codeblock_message_ids[user_message_id] + return + + # If the reaction was clicked by staff (mod or higher), deletes the bot message + for role in user.roles: + if role.id in (Roles.owner, Roles.admin, Roles.moderator): + await bot_message.delete() + del self.codeblock_message_ids[user_message_id] + return + def setup(bot): bot.add_cog(Bot(bot)) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py new file mode 100644 index 000000000..8a9b01d07 --- /dev/null +++ b/bot/cogs/clean.py @@ -0,0 +1,260 @@ +import logging +import random +import re +from typing import Optional + +from discord import Colour, Embed, Message, User +from discord.ext.commands import Bot, Context, group + +from bot.cogs.modlog import ModLog +from bot.constants import ( + Channels, CleanMessages, Colours, Event, + Icons, NEGATIVE_REPLIES, Roles +) +from bot.decorators import with_role + +log = logging.getLogger(__name__) + + +class Clean: + """ + A cog that allows messages to be deleted in + bulk, while applying various filters. + + You can delete messages sent by a specific user, + messages sent by bots, all messages, or messages + that match a specific regular expression. + + The deleted messages are saved and uploaded + to the database via an API endpoint, and a URL is + returned which can be used to view the messages + in the Discord dark theme style. + """ + + def __init__(self, bot: Bot): + self.bot = bot + self.cleaning = False + + @property + def mod_log(self) -> ModLog: + return self.bot.get_cog("ModLog") + + async def _clean_messages( + self, amount: int, ctx: Context, + bots_only: bool = False, user: User = None, + regex: Optional[str] = None + ): + """ + A helper function that does the actual message cleaning. + + :param bots_only: Set this to True if you only want to delete bot messages. + :param user: Specify a user and it will only delete messages by this user. + :param regular_expression: Specify a regular expression and it will only + delete messages that match this. + """ + + def predicate_bots_only(message: Message) -> bool: + """ + Returns true if the message was sent by a bot + """ + + return message.author.bot + + def predicate_specific_user(message: Message) -> bool: + """ + Return True if the message was sent by the + user provided in the _clean_messages call. + """ + + return message.author == user + + def predicate_regex(message: Message): + """ + Returns True if the regex provided in the + _clean_messages matches the message content + or any embed attributes the message may have. + """ + + content = [message.content] + + # Add the content for all embed attributes + for embed in message.embeds: + content.append(embed.title) + content.append(embed.description) + content.append(embed.footer.text) + content.append(embed.author.name) + for field in embed.fields: + content.append(field.name) + content.append(field.value) + + # Get rid of empty attributes and turn it into a string + content = [attr for attr in content if attr] + content = "\n".join(content) + + # Now let's see if there's a regex match + if not content: + return False + else: + return bool(re.search(regex.lower(), content.lower())) + + # Is this an acceptable amount of messages to clean? + if amount > CleanMessages.message_limit: + embed = Embed( + color=Colour(Colours.soft_red), + title=random.choice(NEGATIVE_REPLIES), + description=f"You cannot clean more than {CleanMessages.message_limit} messages." + ) + await ctx.send(embed=embed) + return + + # Are we already performing a clean? + if self.cleaning: + embed = Embed( + color=Colour(Colours.soft_red), + title=random.choice(NEGATIVE_REPLIES), + description="Please wait for the currently ongoing clean operation to complete." + ) + await ctx.send(embed=embed) + return + + # Set up the correct predicate + if bots_only: + predicate = predicate_bots_only # Delete messages from bots + elif user: + predicate = predicate_specific_user # Delete messages from specific user + elif regex: + predicate = predicate_regex # Delete messages that match regex + else: + predicate = None # Delete all messages + + # Look through the history and retrieve message data + messages = [] + message_ids = [] + self.cleaning = True + invocation_deleted = False + + async for message in ctx.channel.history(limit=amount): + + # If at any point the cancel command is invoked, we should stop. + if not self.cleaning: + return + + # Always start by deleting the invocation + if not invocation_deleted: + self.mod_log.ignore(Event.message_delete, message.id) + await message.delete() + invocation_deleted = True + continue + + # If the message passes predicate, let's save it. + if predicate is None or predicate(message): + message_ids.append(message.id) + messages.append(message) + + self.cleaning = False + + # We should ignore the ID's we stored, so we don't get mod-log spam. + self.mod_log.ignore(Event.message_delete, *message_ids) + + # Use bulk delete to actually do the cleaning. It's far faster. + await ctx.channel.purge( + limit=amount, + check=predicate + ) + + # Reverse the list to restore chronological order + if messages: + messages = list(reversed(messages)) + log_url = await self.mod_log.upload_log(messages) + else: + # Can't build an embed, nothing to clean! + embed = Embed( + color=Colour(Colours.soft_red), + description="No matching messages could be found." + ) + await ctx.send(embed=embed, delete_after=10) + return + + # Build the embed and send it + message = ( + f"**{len(message_ids)}** messages deleted in <#{ctx.channel.id}> by **{ctx.author.name}**\n\n" + f"A log of the deleted messages can be found [here]({log_url})." + ) + + await self.mod_log.send_log_message( + icon_url=Icons.message_bulk_delete, + colour=Colour(Colours.soft_red), + title="Bulk message delete", + text=message, + channel_id=Channels.modlog, + ) + + @group(invoke_without_command=True, name="clean", hidden=True) + @with_role(Roles.moderator, Roles.admin, Roles.owner) + async def clean_group(self, ctx: Context): + """ + Commands for cleaning messages in channels + """ + + await ctx.invoke(self.bot.get_command("help"), "clean") + + @clean_group.command(name="user", aliases=["users"]) + @with_role(Roles.moderator, Roles.admin, Roles.owner) + async def clean_user(self, ctx: Context, user: User, amount: int = 10): + """ + Delete messages posted by the provided user, + and stop cleaning after traversing `amount` messages. + """ + + await self._clean_messages(amount, ctx, user=user) + + @clean_group.command(name="all", aliases=["everything"]) + @with_role(Roles.moderator, Roles.admin, Roles.owner) + async def clean_all(self, ctx: Context, amount: int = 10): + """ + Delete all messages, regardless of poster, + and stop cleaning after traversing `amount` messages. + """ + + await self._clean_messages(amount, ctx) + + @clean_group.command(name="bots", aliases=["bot"]) + @with_role(Roles.moderator, Roles.admin, Roles.owner) + async def clean_bots(self, ctx: Context, amount: int = 10): + """ + Delete all messages posted by a bot, + and stop cleaning after traversing `amount` messages. + """ + + await self._clean_messages(amount, ctx, bots_only=True) + + @clean_group.command(name="regex", aliases=["word", "expression"]) + @with_role(Roles.moderator, Roles.admin, Roles.owner) + async def clean_regex(self, ctx: Context, regex, amount: int = 10): + """ + Delete all messages that match a certain regex, + and stop cleaning after traversing `amount` messages. + """ + + await self._clean_messages(amount, ctx, regex=regex) + + @clean_group.command(name="stop", aliases=["cancel", "abort"]) + @with_role(Roles.moderator, Roles.admin, Roles.owner) + async def clean_cancel(self, ctx: Context): + """ + If there is an ongoing cleaning process, + attempt to immediately cancel it. + """ + + self.cleaning = False + + embed = Embed( + color=Colour.blurple(), + description="Clean interrupted." + ) + await ctx.send(embed=embed, delete_after=10) + + +def setup(bot): + bot.add_cog(Clean(bot)) + log.info("Cog loaded: Clean") diff --git a/bot/cogs/clickup.py b/bot/cogs/clickup.py deleted file mode 100644 index 3509c001e..000000000 --- a/bot/cogs/clickup.py +++ /dev/null @@ -1,380 +0,0 @@ -import logging - -from discord import Colour, Embed -from discord.ext.commands import Bot, Context, command -from multidict import MultiDict - -from bot.constants import ClickUp as ClickUpConfig, Roles -from bot.decorators import with_role -from bot.pagination import LinePaginator -from bot.utils import CaseInsensitiveDict - -CREATE_TASK_URL = "https://api.clickup.com/api/v1/list/{list_id}/task" -EDIT_TASK_URL = "https://api.clickup.com/api/v1/task/{task_id}" -GET_TASKS_URL = "https://api.clickup.com/api/v1/team/{team_id}/task" -PROJECTS_URL = "https://api.clickup.com/api/v1/space/{space_id}/project" -SPACES_URL = "https://api.clickup.com/api/v1/team/{team_id}/space" -TEAM_URL = "https://api.clickup.com/api/v1/team/{team_id}" - -HEADERS = { - "Authorization": ClickUpConfig.key, - "Content-Type": "application/json" -} - -STATUSES = ["open", "in progress", "review", "closed"] - -log = logging.getLogger(__name__) - - -class ClickUp: - """ - ClickUp management commands - """ - - def __init__(self, bot: Bot): - self.bot = bot - self.lists = CaseInsensitiveDict() - - async def on_ready(self): - response = await self.bot.http_session.get( - PROJECTS_URL.format(space_id=ClickUpConfig.space), headers=HEADERS - ) - result = await response.json() - - if "err" in result: - log.error(f"Failed to get ClickUp lists: `{result['ECODE']}`: {result['err']}") - else: - # Save all the lists with their IDs so that we can get at them later - for project in result["projects"]: - for list_ in project["lists"]: - self.lists[list_["name"]] = list_["id"] - self.lists[f"{project['name']}/{list_['name']}"] = list_["id"] # Just in case we have duplicates - - # Add the reverse so we can look up by ID as well - self.lists.update({v: k for k, v in self.lists.items()}) - - @command(name="clickup.tasks()", aliases=["clickup.tasks", "tasks", "list_tasks"]) - @with_role(Roles.moderator, Roles.admin, Roles.owner, Roles.devops, Roles.contributor) - async def tasks_command(self, ctx: Context, status: str = None, task_list: str = None): - """ - Get a list of tasks, optionally on a specific list or with a specific status - - Provide "*" for the status to match everything except for "Closed". - - When specifying a list you may use the list name on its own, but it is preferable to give the project name - as well - for example, "Bot/Cogs". This is case-insensitive. - """ - - params = {} - - embed = Embed(colour=Colour.blurple()) - embed.set_author( - name="ClickUp Tasks", - icon_url="https://clickup.com/landing/favicons/favicon-32x32.png", - url=f"https://app.clickup.com/{ClickUpConfig.team}/{ClickUpConfig.space}/" - ) - - if task_list: - if task_list in self.lists: - params["list_ids[]"] = self.lists[task_list] - else: - log.warning(f"{ctx.author} requested '{task_list}', but that list is unknown. Rejecting request.") - embed.description = f"Unknown list: {task_list}" - embed.colour = Colour.red() - return await ctx.send(embed=embed) - - if status and status != "*": - params["statuses[]"] = status - - response = await self.bot.http_session.get( - GET_TASKS_URL.format(team_id=ClickUpConfig.team), headers=HEADERS, params=params - ) - result = await response.json() - - if "err" in result: - log.error("ClickUp responded to the task list request with an error!\n" - f"error code: '{result['ECODE']}'\n" - f"error: {result['err']}") - embed.description = f"`{result['ECODE']}`: {result['err']}" - embed.colour = Colour.red() - - else: - tasks = result["tasks"] - - if not tasks: - log.debug(f"{ctx.author} requested a list of ClickUp tasks, but no ClickUp tasks were found.") - embed.description = "No tasks found." - embed.colour = Colour.red() - - else: - lines = [] - - for task in tasks: - task_url = f"http://app.clickup.com/{ClickUpConfig.team}/{ClickUpConfig.space}/t/{task['id']}" - id_fragment = f"[`#{task['id']: <5}`]({task_url})" - status = f"{task['status']['status'].title()}" - - lines.append(f"{id_fragment} ({status})\n\u00BB {task['name']}") - - log.debug(f"{ctx.author} requested a list of ClickUp tasks. Returning list.") - return await LinePaginator.paginate(lines, ctx, embed, max_size=750) - return await ctx.send(embed=embed) - - @command(name="clickup.task()", aliases=["clickup.task", "task", "get_task"]) - @with_role(Roles.moderator, Roles.admin, Roles.owner, Roles.devops, Roles.contributor) - async def task_command(self, ctx: Context, task_id: str): - """ - Get a task and return information specific to it - """ - - if task_id.startswith("#"): - task_id = task_id[1:] - - embed = Embed(colour=Colour.blurple()) - embed.set_author( - name=f"ClickUp Task: #{task_id}", - icon_url="https://clickup.com/landing/favicons/favicon-32x32.png", - url=f"https://app.clickup.com/{ClickUpConfig.team}/{ClickUpConfig.space}/t/{task_id}" - ) - - params = MultiDict() - params.add("statuses[]", "Open") - params.add("statuses[]", "in progress") - params.add("statuses[]", "review") - params.add("statuses[]", "Closed") - - response = await self.bot.http_session.get( - GET_TASKS_URL.format(team_id=ClickUpConfig.team), headers=HEADERS, params=params - ) - result = await response.json() - - if "err" in result: - log.error("ClickUp responded to the get task request with an error!\n" - f"error code: '{result['ECODE']}'\n" - f"error: {result['err']}") - embed.description = f"`{result['ECODE']}`: {result['err']}" - embed.colour = Colour.red() - else: - task = None - - for task_ in result["tasks"]: - if task_["id"] == task_id: - task = task_ - break - - if task is None: - log.warning(f"{ctx.author} requested the task '#{task_id}', but it could not be found.") - embed.description = f"Unable to find task with ID `#{task_id}`:" - embed.colour = Colour.red() - else: - status = task['status']['status'].title() - project, list_ = self.lists[task['list']['id']].split("/", 1) - list_ = f"{project.title()}/{list_.title()}" - first_line = f"**{list_}** \u00BB *{task['name']}* \n**Status**: {status}" - - if task.get("tags"): - tags = ", ".join(tag["name"].title() for tag in task["tags"]) - first_line += f" / **Tags**: {tags}" - - lines = [first_line] - - if task.get("text_content"): - text = task["text_content"] - - if len(text) >= 1500: - text = text[:1497] + "..." - - lines.append(text) - - if task.get("assignees"): - assignees = ", ".join(user["username"] for user in task["assignees"]) - lines.append( - f"**Assignees**\n{assignees}" - ) - - log.debug(f"{ctx.author} requested the task '#{task_id}'. Returning the task data.") - return await LinePaginator.paginate(lines, ctx, embed, max_size=1500) - return await ctx.send(embed=embed) - - @command(name="clickup.team()", aliases=["clickup.team", "team", "list_team"]) - @with_role(Roles.moderator, Roles.admin, Roles.owner, Roles.devops) - async def team_command(self, ctx: Context): - """ - Get a list of every member of the team - """ - - response = await self.bot.http_session.get( - TEAM_URL.format(team_id=ClickUpConfig.team), headers=HEADERS - ) - result = await response.json() - - if "err" in result: - log.error("ClickUp responded to the team request with an error!\n" - f"error code: '{result['ECODE']}'\n" - f"error: {result['err']}") - embed = Embed( - colour=Colour.red(), - description=f"`{result['ECODE']}`: {result['err']}" - ) - else: - log.debug(f"{ctx.author} requested a list of team members. Preparing the list...") - embed = Embed( - colour=Colour.blurple() - ) - - for member in result["team"]["members"]: - embed.add_field( - name=member["user"]["username"], - value=member["user"]["id"] - ) - - embed.set_author( - name="ClickUp Members", - icon_url="https://clickup.com/landing/favicons/favicon-32x32.png", - url=f"https://app.clickup.com/{ClickUpConfig.team}/{ClickUpConfig.space}/" - ) - - log.debug("List fully prepared, returning list to channel.") - await ctx.send(embed=embed) - - @command(name="clickup.lists()", aliases=["clickup.lists", "lists"]) - @with_role(Roles.moderator, Roles.admin, Roles.owner, Roles.devops, Roles.contributor) - async def lists_command(self, ctx: Context): - """ - Get all the lists belonging to the ClickUp space - """ - - response = await self.bot.http_session.get( - PROJECTS_URL.format(space_id=ClickUpConfig.space), headers=HEADERS - ) - result = await response.json() - - if "err" in result: - log.error("ClickUp responded to the lists request with an error!\n" - f"error code: '{result['ECODE']}'\n" - f"error: {result['err']}") - embed = Embed( - colour=Colour.red(), - description=f"`{result['ECODE']}`: {result['err']}" - ) - else: - log.debug(f"{ctx.author} requested a list of all ClickUp lists. Preparing the list...") - embed = Embed( - colour=Colour.blurple() - ) - - for project in result["projects"]: - lists = [] - - for list_ in project["lists"]: - lists.append(f"{list_['name']} ({list_['id']})") - - lists = "\n".join(lists) - - embed.add_field( - name=f"{project['name']} ({project['id']})", - value=lists - ) - - embed.set_author( - name="ClickUp Projects", - icon_url="https://clickup.com/landing/favicons/favicon-32x32.png", - url=f"https://app.clickup.com/{ClickUpConfig.team}/{ClickUpConfig.space}/" - ) - - log.debug(f"List fully prepared, returning list to channel.") - await ctx.send(embed=embed) - - @command(name="clickup.open()", aliases=["clickup.open", "open", "open_task"]) - @with_role(Roles.moderator, Roles.admin, Roles.owner, Roles.devops, Roles.contributor) - async def open_command(self, ctx: Context, task_list: str, title: str): - """ - Open a new task under a specific task list, with a title - - When specifying a list you may use the list name on its own, but it is preferable to give the project name - as well - for example, "Bot/Cogs". This is case-insensitive. - """ - - embed = Embed(colour=Colour.blurple()) - embed.set_author( - name="ClickUp Tasks", - icon_url="https://clickup.com/landing/favicons/favicon-32x32.png", - url=f"https://app.clickup.com/{ClickUpConfig.team}/{ClickUpConfig.space}/" - ) - - if task_list in self.lists: - task_list = self.lists[task_list] - else: - log.warning(f"{ctx.author} tried to open a new task on ClickUp, " - f"but '{task_list}' is not a known list. Rejecting request.") - embed.description = f"Unknown list: {task_list}" - embed.colour = Colour.red() - return await ctx.send(embed=embed) - - response = await self.bot.http_session.post( - CREATE_TASK_URL.format(list_id=task_list), headers=HEADERS, json={ - "name": title, - "status": "Open" - } - ) - result = await response.json() - - if "err" in result: - log.error("ClickUp responded to the get task request with an error!\n" - f"error code: '{result['ECODE']}'\n" - f"error: {result['err']}") - embed.colour = Colour.red() - embed.description = f"`{result['ECODE']}`: {result['err']}" - else: - task_id = result.get("id") - task_url = f"https://app.clickup.com/{ClickUpConfig.team}/{ClickUpConfig.space}/t/{task_id}" - project, task_list = self.lists[task_list].split("/", 1) - task_list = f"{project.title()}/{task_list.title()}" - - log.debug(f"{ctx.author} opened a new task on ClickUp: \n" - f"{task_list} - #{task_id}") - embed.description = f"New task created: [{task_list} \u00BB `#{task_id}`]({task_url})" - - await ctx.send(embed=embed) - - @command(name="clickup.set_status()", aliases=["clickup.set_status", "set_status", "set_task_status"]) - @with_role(Roles.moderator, Roles.admin, Roles.owner, Roles.devops, Roles.contributor) - async def set_status_command(self, ctx: Context, task_id: str, status: str): - """ - Update the status of a specific task - """ - - embed = Embed(colour=Colour.blurple()) - embed.set_author( - name="ClickUp Tasks", - icon_url="https://clickup.com/landing/favicons/favicon-32x32.png", - url=f"https://app.clickup.com/{ClickUpConfig.team}/{ClickUpConfig.space}/" - ) - - if status.lower() not in STATUSES: - log.warning(f"{ctx.author} tried to update a task on ClickUp, but '{status}' is not a known status.") - embed.description = f"Unknown status: {status}" - embed.colour = Colour.red() - else: - response = await self.bot.http_session.put( - EDIT_TASK_URL.format(task_id=task_id), headers=HEADERS, json={"status": status} - ) - result = await response.json() - - if "err" in result: - log.error("ClickUp responded to the get task request with an error!\n" - f"error code: '{result['ECODE']}'\n" - f"error: {result['err']}") - embed.description = f"`{result['ECODE']}`: {result['err']}" - embed.colour = Colour.red() - else: - log.debug(f"{ctx.author} updated a task on ClickUp: #{task_id}") - task_url = f"https://app.clickup.com/{ClickUpConfig.team}/{ClickUpConfig.space}/t/{task_id}" - embed.description = f"Task updated: [`#{task_id}`]({task_url})" - - await ctx.send(embed=embed) - - -def setup(bot): - bot.add_cog(ClickUp(bot)) - log.info("Cog loaded: ClickUp") diff --git a/bot/cogs/defcon.py b/bot/cogs/defcon.py index 8ca59b058..beb05ba46 100644 --- a/bot/cogs/defcon.py +++ b/bot/cogs/defcon.py @@ -36,7 +36,7 @@ class Defcon: self.headers = {"X-API-KEY": Keys.site_api} @property - def modlog(self) -> ModLog: + def mod_log(self) -> ModLog: return self.bot.get_cog("ModLog") async def on_ready(self): @@ -92,7 +92,7 @@ class Defcon: if not message_sent: message = f"{message}\n\nUnable to send rejection message via DM; they probably have DMs disabled." - await self.modlog.send_log_message( + await self.mod_log.send_log_message( Icons.defcon_denied, COLOUR_RED, "Entry denied", message, member.avatar_url_as(static_format="png") ) @@ -133,7 +133,7 @@ class Defcon: f"```py\n{e}\n```" ) - await self.modlog.send_log_message( + await self.mod_log.send_log_message( Icons.defcon_enabled, COLOUR_GREEN, "DEFCON enabled", f"**Staffer:** {ctx.author.name}#{ctx.author.discriminator} (`{ctx.author.id}`)\n" f"**Days:** {self.days.days}\n\n" @@ -144,7 +144,7 @@ class Defcon: else: await ctx.send(f"{Emojis.defcon_enabled} DEFCON enabled.") - await self.modlog.send_log_message( + await self.mod_log.send_log_message( Icons.defcon_enabled, COLOUR_GREEN, "DEFCON enabled", f"**Staffer:** {ctx.author.name}#{ctx.author.discriminator} (`{ctx.author.id}`)\n" f"**Days:** {self.days.days}\n\n" @@ -176,7 +176,7 @@ class Defcon: f"```py\n{e}\n```" ) - await self.modlog.send_log_message( + await self.mod_log.send_log_message( Icons.defcon_disabled, COLOUR_RED, "DEFCON disabled", f"**Staffer:** {ctx.author.name}#{ctx.author.discriminator} (`{ctx.author.id}`)\n" "**There was a problem updating the site** - This setting may be reverted when the bot is " @@ -186,7 +186,7 @@ class Defcon: else: await ctx.send(f"{Emojis.defcon_disabled} DEFCON disabled.") - await self.modlog.send_log_message( + await self.mod_log.send_log_message( Icons.defcon_disabled, COLOUR_RED, "DEFCON disabled", f"**Staffer:** {ctx.author.name}#{ctx.author.discriminator} (`{ctx.author.id}`)" ) @@ -233,7 +233,7 @@ class Defcon: f"```py\n{e}\n```" ) - await self.modlog.send_log_message( + await self.mod_log.send_log_message( Icons.defcon_updated, Colour.blurple(), "DEFCON updated", f"**Staffer:** {ctx.author.name}#{ctx.author.discriminator} (`{ctx.author.id}`)\n" f"**Days:** {self.days.days}\n\n" @@ -246,7 +246,7 @@ class Defcon: f"{Emojis.defcon_updated} DEFCON days updated; accounts must be {days} days old to join to the server" ) - await self.modlog.send_log_message( + await self.mod_log.send_log_message( Icons.defcon_updated, Colour.blurple(), "DEFCON updated", f"**Staffer:** {ctx.author.name}#{ctx.author.discriminator} (`{ctx.author.id}`)\n" f"**Days:** {self.days.days}" diff --git a/bot/cogs/events.py b/bot/cogs/events.py index a7111b8a0..0b9b75a00 100644 --- a/bot/cogs/events.py +++ b/bot/cogs/events.py @@ -1,20 +1,24 @@ import logging -from discord import Embed, Member +from discord import Colour, Embed, Member, Object from discord.ext.commands import ( BadArgument, Bot, BotMissingPermissions, CommandError, CommandInvokeError, Context, NoPrivateMessage, UserInputError ) +from bot.cogs.modlog import ModLog from bot.constants import ( - Channels, DEBUG_MODE, Guild, - Keys, Roles, URLs + Channels, Colours, DEBUG_MODE, + Guild, Icons, Keys, + Roles, URLs ) from bot.utils import chunks log = logging.getLogger(__name__) +RESTORE_ROLES = (str(Roles.muted), str(Roles.announcements)) + class Events: """No commands, just event handlers.""" @@ -22,6 +26,10 @@ class Events: def __init__(self, bot: Bot): self.bot = bot + @property + def mod_log(self) -> ModLog: + return self.bot.get_cog("ModLog") + async def send_updated_users(self, *users, replace_all=False): users = list(filter(lambda user: str(Roles.verified) in user["roles"], users)) @@ -85,6 +93,16 @@ class Events: log.exception(f"Failed to delete {len(users)} users") return {} + async def get_user(self, user_id): + response = await self.bot.http_session.get( + url=URLs.site_user_api, + params={"user_id": user_id}, + headers={"X-API-Key": Keys.site_api} + ) + + resp = await response.json() + return resp["data"] + async def on_command_error(self, ctx: Context, e: CommandError): command = ctx.command parent = None @@ -194,6 +212,29 @@ class Events: async def on_member_join(self, member: Member): role_ids = [str(r.id) for r in member.roles] # type: List[str] + new_roles = [] + + try: + user_objs = await self.get_user(str(member.id)) + except Exception as e: + log.exception("Failed to persist roles") + + await self.mod_log.send_log_message( + Icons.crown_red, Colour(Colours.soft_red), "Failed to persist roles", + f"```py\n{e}\n```", + member.avatar_url_as(static_format="png") + ) + else: + if user_objs: + old_roles = user_objs[0].get("roles", []) + + for role in RESTORE_ROLES: + if role in old_roles: + new_roles.append(Object(int(role))) + + for role in new_roles: + if str(role) not in role_ids: + role_ids.append(str(role.id)) changes = await self.send_updated_users({ "avatar": member.avatar_url_as(format="png"), @@ -205,6 +246,18 @@ class Events: log.debug(f"User {member.id} joined; changes: {changes}") + if new_roles: + await member.add_roles( + *new_roles, + reason="Roles restored" + ) + + await self.mod_log.send_log_message( + Icons.crown_blurple, Colour.blurple(), "Roles restored", + f"Restored {len(new_roles)} roles", + member.avatar_url_as(static_format="png") + ) + async def on_member_remove(self, member: Member): changes = await self.send_delete_users({ "user_id": str(member.id) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py new file mode 100644 index 000000000..70254fd88 --- /dev/null +++ b/bot/cogs/filtering.py @@ -0,0 +1,232 @@ +import logging +import re + +from discord import Colour, Member, Message +from discord.ext.commands import Bot + +from bot.cogs.modlog import ModLog +from bot.constants import ( + Channels, Colours, DEBUG_MODE, + Filter, Icons +) + +log = logging.getLogger(__name__) + +INVITE_RE = ( + r"(?:discord(?:[\.,]|dot)gg|" # Could be discord.gg/ + r"discord(?:[\.,]|dot)com(?:\/|slash)invite|" # or discord.com/invite/ + r"discordapp(?:[\.,]|dot)com(?:\/|slash)invite|" # or discordapp.com/invite/ + r"discord(?:[\.,]|dot)me|" # or discord.me + r"discord(?:[\.,]|dot)io" # or discord.io. + r")(?:[\/]|slash)" # / or 'slash' + r"([a-zA-Z0-9]+)" # the invite code itself +) + +URL_RE = "(https?://[^\s]+)" +ZALGO_RE = r"[\u0300-\u036F\u0489]" + + +class Filtering: + """ + Filtering out invites, blacklisting domains, + and warning us of certain regular expressions + """ + + def __init__(self, bot: Bot): + self.bot = bot + + self.filters = { + "filter_zalgo": { + "enabled": Filter.filter_zalgo, + "function": self._has_zalgo, + "type": "filter" + }, + "filter_invites": { + "enabled": Filter.filter_invites, + "function": self._has_invites, + "type": "filter" + }, + "filter_domains": { + "enabled": Filter.filter_domains, + "function": self._has_urls, + "type": "filter" + }, + "watch_words": { + "enabled": Filter.watch_words, + "function": self._has_watchlist_words, + "type": "watchlist" + }, + "watch_tokens": { + "enabled": Filter.watch_tokens, + "function": self._has_watchlist_tokens, + "type": "watchlist" + }, + } + + @property + def mod_log(self) -> ModLog: + return self.bot.get_cog("ModLog") + + async def on_message(self, msg: Message): + await self._filter_message(msg) + + async def on_message_edit(self, _: Message, after: Message): + await self._filter_message(after) + + async def _filter_message(self, msg: Message): + """ + Whenever a message is sent or edited, + run it through our filters to see if it + violates any of our rules, and then respond + accordingly. + """ + + # Should we filter this message? + role_whitelisted = False + + if type(msg.author) is Member: # Only Member has roles, not User. + for role in msg.author.roles: + if role.id in Filter.role_whitelist: + role_whitelisted = True + + filter_message = ( + msg.channel.id not in Filter.channel_whitelist # Channel not in whitelist + and not role_whitelisted # Role not in whitelist + and not msg.author.bot # Author not a bot + ) + + # If we're running the bot locally, ignore role whitelist and only listen to #dev-test + if DEBUG_MODE: + filter_message = not msg.author.bot and msg.channel.id == Channels.devtest + + # If none of the above, we can start filtering. + if filter_message: + for filter_name, _filter in self.filters.items(): + + # Is this specific filter enabled in the config? + if _filter["enabled"]: + triggered = await _filter["function"](msg.content) + + if triggered: + message = ( + f"The {filter_name} {_filter['type']} was triggered " + f"by **{msg.author.name}#{msg.author.discriminator}** " + f"(`{msg.author.id}`) in <#{msg.channel.id}> with [the " + f"following message]({msg.jump_url}):\n\n" + f"{msg.content}" + ) + + log.debug(message) + + # Send pretty mod log embed to mod-alerts + await self.mod_log.send_log_message( + icon_url=Icons.filtering, + colour=Colour(Colours.soft_red), + title=f"{_filter['type'].title()} triggered!", + text=message, + thumbnail=msg.author.avatar_url_as(static_format="png"), + channel_id=Channels.mod_alerts, + ping_everyone=Filter.ping_everyone, + ) + + # If this is a filter (not a watchlist), we should delete the message. + if _filter["type"] == "filter": + await msg.delete() + + break # We don't want multiple filters to trigger + + @staticmethod + async def _has_watchlist_words(text: str) -> bool: + """ + Returns True if the text contains + one of the regular expressions from the + word_watchlist in our filter config. + + Only matches words with boundaries before + and after the expression. + """ + + for expression in Filter.word_watchlist: + if re.search(fr"\b{expression}\b", text, re.IGNORECASE): + return True + + return False + + @staticmethod + async def _has_watchlist_tokens(text: str) -> bool: + """ + Returns True if the text contains + one of the regular expressions from the + token_watchlist in our filter config. + + This will match the expression even if it + does not have boundaries before and after + """ + + for expression in Filter.token_watchlist: + if re.search(fr"{expression}", text, re.IGNORECASE): + return True + + return False + + @staticmethod + async def _has_urls(text: str) -> bool: + """ + Returns True if the text contains one of + the blacklisted URLs from the config file. + """ + + if not re.search(URL_RE, text, re.IGNORECASE): + return False + + text = text.lower() + + for url in Filter.domain_blacklist: + if url.lower() in text: + return True + + return False + + @staticmethod + async def _has_zalgo(text: str) -> bool: + """ + Returns True if the text contains zalgo characters. + + Zalgo range is \u0300 – \u036F and \u0489. + """ + + return bool(re.search(ZALGO_RE, text)) + + @staticmethod + async def _has_invites(text: str) -> bool: + """ + Returns True if the text contains an invite which + is not on the guild_invite_whitelist in config.yml. + + Also catches a lot of common ways to try to cheat the system. + """ + + # Remove spaces to prevent cases like + # d i s c o r d . c o m / i n v i t e / p y t h o n + text = text.replace(" ", "") + + # Remove backslashes to prevent escape character aroundfuckery like + # discord\.gg/gdudes-pony-farm + text = text.replace("\\", "") + + invites = re.findall(INVITE_RE, text, re.IGNORECASE) + for invite in invites: + + filter_invite = ( + invite not in Filter.guild_invite_whitelist + and invite.lower() not in Filter.vanity_url_whitelist + ) + + if filter_invite: + return True + return False + + +def setup(bot: Bot): + bot.add_cog(Filtering(bot)) + log.info("Cog loaded: Filtering") diff --git a/bot/cogs/information.py b/bot/cogs/information.py new file mode 100644 index 000000000..a313d2379 --- /dev/null +++ b/bot/cogs/information.py @@ -0,0 +1,191 @@ +import logging +import textwrap + +from discord import CategoryChannel, Colour, Embed, Member, TextChannel, VoiceChannel +from discord.ext.commands import Bot, Context, command + +from bot.constants import Emojis, Keys, Roles, URLs +from bot.decorators import with_role +from bot.utils.time import time_since + +log = logging.getLogger(__name__) + + +class Information: + """ + A cog with commands for generating embeds with + server information, such as server statistics + and user information. + """ + + def __init__(self, bot: Bot): + self.bot = bot + self.headers = {"X-API-Key": Keys.site_api} + + @with_role(Roles.owner, Roles.admin, Roles.moderator) + @command(name="roles") + async def roles_info(self, ctx: Context): + """ + Returns a list of all roles and their + corresponding IDs. + """ + + # Sort the roles alphabetically and remove the @everyone role + roles = sorted(ctx.guild.roles, key=lambda role: role.name) + roles = [role for role in roles if role.name != "@everyone"] + + # Build a string + role_string = "" + for role in roles: + role_string += f"`{role.id}` - {role.mention}\n" + + # Build an embed + embed = Embed( + title="Role information", + colour=Colour.blurple(), + description=role_string + ) + + embed.set_footer(text=f"Total roles: {len(roles)}") + + await ctx.send(embed=embed) + + @command(name="server", aliases=["server_info", "guild", "guild_info"]) + async def server_info(self, ctx: Context): + """ + Returns an embed full of + server information. + """ + + created = time_since(ctx.guild.created_at, precision="days") + features = ", ".join(ctx.guild.features) + region = ctx.guild.region + + # How many of each type of channel? + roles = len(ctx.guild.roles) + channels = ctx.guild.channels + text_channels = 0 + category_channels = 0 + voice_channels = 0 + for channel in channels: + if type(channel) == TextChannel: + text_channels += 1 + elif type(channel) == CategoryChannel: + category_channels += 1 + elif type(channel) == VoiceChannel: + voice_channels += 1 + + # How many of each user status? + member_count = ctx.guild.member_count + members = ctx.guild.members + online = 0 + dnd = 0 + idle = 0 + offline = 0 + for member in members: + if str(member.status) == "online": + online += 1 + elif str(member.status) == "offline": + offline += 1 + elif str(member.status) == "idle": + idle += 1 + elif str(member.status) == "dnd": + dnd += 1 + + embed = Embed( + colour=Colour.blurple(), + description=textwrap.dedent(f""" + **Server information** + Created: {created} + Voice region: {region} + Features: {features} + + **Counts** + Members: {member_count} + Roles: {roles} + Text: {text_channels} + Voice: {voice_channels} + Channel categories: {category_channels} + + **Members** + {Emojis.status_online} {online} + {Emojis.status_idle} {idle} + {Emojis.status_dnd} {dnd} + {Emojis.status_offline} {offline} + """) + ) + + embed.set_thumbnail(url=ctx.guild.icon_url) + + await ctx.send(embed=embed) + + @command(name="user", aliases=["user_info", "member", "member_info"]) + async def user_info(self, ctx: Context, user: Member = None): + """ + Returns info about a user. + """ + + if user is None: + user = ctx.author + + # User information + created = time_since(user.created_at, max_units=3) + + name = f"{user.name}#{user.discriminator}" + if user.nick: + name = f"{user.nick} ({name})" + + # Member information + joined = time_since(user.joined_at, precision="days") + + # You're welcome, Volcyyyyyyyyyyyyyyyy + roles = ", ".join( + role.mention for role in user.roles if role.name != "@everyone" + ) + + # Infractions + api_response = await self.bot.http_session.get( + url=URLs.site_infractions_user.format(user_id=user.id), + headers=self.headers + ) + + infractions = await api_response.json() + + infr_total = 0 + infr_active = 0 + + # At least it's readable. + for infr in infractions: + if infr["active"]: + infr_active += 1 + + infr_total += 1 + + # Let's build the embed now + embed = Embed( + title=name, + description=textwrap.dedent(f""" + **User Information** + Created: {created} + Profile: {user.mention} + ID: {user.id} + + **Member Information** + Joined: {joined} + Roles: {roles or None} + + **Infractions** + Total: {infr_total} + Active: {infr_active} + """) + ) + + embed.set_thumbnail(url=user.avatar_url_as(format="png")) + embed.colour = user.top_role.colour if roles else Colour.blurple() + + await ctx.send(embed=embed) + + +def setup(bot): + bot.add_cog(Information(bot)) + log.info("Cog loaded: Information") diff --git a/bot/cogs/moderation.py b/bot/cogs/moderation.py index 245f17fda..ee28a3600 100644 --- a/bot/cogs/moderation.py +++ b/bot/cogs/moderation.py @@ -1,6 +1,7 @@ import asyncio import datetime import logging +import textwrap from typing import Dict from aiohttp import ClientError @@ -8,7 +9,8 @@ from discord import Colour, Embed, Guild, Member, Object, User from discord.ext.commands import Bot, Context, command, group from bot import constants -from bot.constants import Keys, Roles, URLs +from bot.cogs.modlog import ModLog +from bot.constants import Colours, Event, Icons, Keys, Roles, URLs from bot.converters import InfractionSearchQuery from bot.decorators import with_role from bot.pagination import LinePaginator @@ -29,6 +31,10 @@ class Moderation: self.expiration_tasks: Dict[str, asyncio.Task] = {} self._muted_role = Object(constants.Roles.muted) + @property + def mod_log(self) -> ModLog: + return self.bot.get_cog("ModLog") + async def on_ready(self): # Schedule expiration for previous infractions response = await self.bot.http_session.get( @@ -111,6 +117,7 @@ class Moderation: await ctx.send(f":x: There was an error adding the infraction: {response_object['error_message']}") return + self.mod_log.ignore(Event.member_remove, user.id) await user.kick(reason=reason) if reason is None: @@ -120,6 +127,19 @@ class Moderation: await ctx.send(result_message) + # Send a log message to the mod log + await self.mod_log.send_log_message( + icon_url=Icons.sign_out, + colour=Colour(Colours.soft_red), + title="Member kicked", + thumbnail=user.avatar_url_as(static_format="png"), + text=textwrap.dedent(f""" + Member: {user.mention} (`{user.id}`) + Actor: {ctx.message.author} + Reason: {reason} + """) + ) + @with_role(*MODERATION_ROLES) @command(name="ban") async def ban(self, ctx: Context, user: User, *, reason: str = None): @@ -150,7 +170,9 @@ class Moderation: await ctx.send(f":x: There was an error adding the infraction: {response_object['error_message']}") return - await ctx.guild.ban(user, reason=reason) + self.mod_log.ignore(Event.member_ban, user.id) + self.mod_log.ignore(Event.member_remove, user.id) + await ctx.guild.ban(user, reason=reason, delete_message_days=0) if reason is None: result_message = f":ok_hand: permanently banned {user.mention}." @@ -159,6 +181,19 @@ class Moderation: await ctx.send(result_message) + # Send a log message to the mod log + await self.mod_log.send_log_message( + icon_url=Icons.user_ban, + colour=Colour(Colours.soft_red), + title="Member permanently banned", + thumbnail=user.avatar_url_as(static_format="png"), + text=textwrap.dedent(f""" + Member: {user.mention} (`{user.id}`) + Actor: {ctx.message.author} + Reason: {reason} + """) + ) + @with_role(*MODERATION_ROLES) @command(name="mute") async def mute(self, ctx: Context, user: Member, *, reason: str = None): @@ -190,6 +225,7 @@ class Moderation: return # add the mute role + self.mod_log.ignore(Event.member_update, user.id) await user.add_roles(self._muted_role, reason=reason) if reason is None: @@ -199,6 +235,19 @@ class Moderation: await ctx.send(result_message) + # Send a log message to the mod log + await self.mod_log.send_log_message( + icon_url=Icons.user_mute, + colour=Colour(Colours.soft_red), + title="Member permanently muted", + thumbnail=user.avatar_url_as(static_format="png"), + text=textwrap.dedent(f""" + Member: {user.mention} (`{user.id}`) + Actor: {ctx.message.author} + Reason: {reason} + """) + ) + # endregion # region: Temporary infractions @@ -234,6 +283,7 @@ class Moderation: await ctx.send(f":x: There was an error adding the infraction: {response_object['error_message']}") return + self.mod_log.ignore(Event.member_update, user.id) await user.add_roles(self._muted_role, reason=reason) infraction_object = response_object["infraction"] @@ -249,6 +299,21 @@ class Moderation: await ctx.send(result_message) + # Send a log message to the mod log + await self.mod_log.send_log_message( + icon_url=Icons.user_mute, + colour=Colour(Colours.soft_red), + title="Member temporarily muted", + thumbnail=user.avatar_url_as(static_format="png"), + text=textwrap.dedent(f""" + Member: {user.mention} (`{user.id}`) + Actor: {ctx.message.author} + Reason: {reason} + Duration: {duration} + Expires: {infraction_expiration} + """) + ) + @with_role(*MODERATION_ROLES) @command(name="tempban") async def tempban(self, ctx, user: User, duration: str, *, reason: str = None): @@ -281,8 +346,10 @@ class Moderation: await ctx.send(f":x: There was an error adding the infraction: {response_object['error_message']}") return + self.mod_log.ignore(Event.member_ban, user.id) + self.mod_log.ignore(Event.member_remove, user.id) guild: Guild = ctx.guild - await guild.ban(user, reason=reason) + await guild.ban(user, reason=reason, delete_message_days=0) infraction_object = response_object["infraction"] infraction_expiration = infraction_object["expires_at"] @@ -297,6 +364,21 @@ class Moderation: await ctx.send(result_message) + # Send a log message to the mod log + await self.mod_log.send_log_message( + icon_url=Icons.user_ban, + colour=Colour(Colours.soft_red), + thumbnail=user.avatar_url_as(static_format="png"), + title="Member temporarily banned", + text=textwrap.dedent(f""" + Member: {user.mention} (`{user.id}`) + Actor: {ctx.message.author} + Reason: {reason} + Duration: {duration} + Expires: {infraction_expiration} + """) + ) + # endregion # region: Remove infractions (un- commands) @@ -333,6 +415,19 @@ class Moderation: self.cancel_expiration(infraction_object["id"]) await ctx.send(f":ok_hand: Un-muted {user.mention}.") + + # Send a log message to the mod log + await self.mod_log.send_log_message( + icon_url=Icons.user_unmute, + colour=Colour(Colours.soft_green), + title="Member unmuted", + thumbnail=user.avatar_url_as(static_format="png"), + text=textwrap.dedent(f""" + Member: {user.mention} (`{user.id}`) + Actor: {ctx.message.author} + Intended expiry: {infraction_object['expires_at']} + """) + ) except Exception: log.exception("There was an error removing an infraction.") await ctx.send(":x: There was an error removing the infraction.") @@ -371,6 +466,19 @@ class Moderation: self.cancel_expiration(infraction_object["id"]) await ctx.send(f":ok_hand: Un-banned {user.mention}.") + + # Send a log message to the mod log + await self.mod_log.send_log_message( + icon_url=Icons.user_unban, + colour=Colour(Colours.soft_green), + title="Member unbanned", + thumbnail=user.avatar_url_as(static_format="png"), + text=textwrap.dedent(f""" + Member: {user.mention} (`{user.id}`) + Actor: {ctx.message.author} + Intended expiry: {infraction_object['expires_at']} + """) + ) except Exception: log.exception("There was an error removing an infraction.") await ctx.send(":x: There was an error removing the infraction.") @@ -400,6 +508,15 @@ class Moderation: """ try: + previous = await self.bot.http_session.get( + URLs.site_infractions_by_id.format( + infraction_id=infraction_id + ), + headers=self.headers + ) + + previous_object = await previous.json() + if duration == "permanent": duration = None # check the current active infraction @@ -432,6 +549,37 @@ class Moderation: await ctx.send(":x: There was an error updating the infraction.") return + prev_infraction = previous_object["infraction"] + + # Get information about the infraction's user + user_id = int(infraction_object["user"]["user_id"]) + user = ctx.guild.get_member(user_id) + + if user: + member_text = f"{user.mention} (`{user.id}`)" + thumbnail = user.avatar_url_as(static_format="png") + else: + member_text = f"`{user_id}`" + thumbnail = None + + # The infraction's actor + actor_id = int(infraction_object["actor"]["user_id"]) + actor = ctx.guild.get_member(actor_id) or f"`{actor_id}`" + + await self.mod_log.send_log_message( + icon_url=Icons.pencil, + colour=Colour.blurple(), + title="Infraction edited", + thumbnail=thumbnail, + text=textwrap.dedent(f""" + Member: {member_text} + Actor: {actor} + Edited by: {ctx.message.author} + Previous expiry: {prev_infraction['expires_at']} + New expiry: {infraction_object['expires_at']} + """) + ) + @with_role(*MODERATION_ROLES) @infraction_edit_group.command(name="reason") async def edit_reason(self, ctx, infraction_id: str, *, reason: str): @@ -442,6 +590,15 @@ class Moderation: """ try: + previous = await self.bot.http_session.get( + URLs.site_infractions_by_id.format( + infraction_id=infraction_id + ), + headers=self.headers + ) + + previous_object = await previous.json() + response = await self.bot.http_session.patch( URLs.site_infractions, json={ @@ -461,6 +618,38 @@ class Moderation: await ctx.send(":x: There was an error updating the infraction.") return + new_infraction = response_object["infraction"] + prev_infraction = previous_object["infraction"] + + # Get information about the infraction's user + user_id = int(new_infraction["user"]["user_id"]) + user = ctx.guild.get_member(user_id) + + if user: + user_text = f"{user.mention} (`{user.id}`)" + thumbnail = user.avatar_url_as(static_format="png") + else: + user_text = f"`{user_id}`" + thumbnail = None + + # The infraction's actor + actor_id = int(new_infraction["actor"]["user_id"]) + actor = ctx.guild.get_member(actor_id) or f"`{actor_id}`" + + await self.mod_log.send_log_message( + icon_url=Icons.pencil, + colour=Colour.blurple(), + title="Infraction edited", + thumbnail=thumbnail, + text=textwrap.dedent(f""" + Member: {user_text} + Actor: {actor} + Edited by: {ctx.message.author} + Previous reason: {prev_infraction['reason']} + New reason: {new_infraction['reason']} + """) + ) + # endregion # region: Search infractions @@ -609,6 +798,7 @@ class Moderation: member: Member = guild.get_member(user_id) if member: # remove the mute role + self.mod_log.ignore(Event.member_update, member.id) await member.remove_roles(self._muted_role) else: log.warning(f"Failed to un-mute user: {user_id} (not found)") @@ -663,7 +853,7 @@ def parse_rfc1123(time_str): def _silent_exception(future): try: future.exception() - except Exception: + except Exception: # noqa: S110 pass diff --git a/bot/cogs/modlog.py b/bot/cogs/modlog.py index 87cea2b5a..2f72d92fc 100644 --- a/bot/cogs/modlog.py +++ b/bot/cogs/modlog.py @@ -3,6 +3,7 @@ import datetime import logging from typing import List, Optional, Union +from aiohttp import ClientResponseError from dateutil.relativedelta import relativedelta from deepdiff import DeepDiff from discord import ( @@ -13,16 +14,12 @@ from discord import ( from discord.abc import GuildChannel from discord.ext.commands import Bot -from bot.constants import Channels, Emojis, Icons +from bot.constants import Channels, Colours, Emojis, Event, Icons, Keys, Roles, URLs from bot.constants import Guild as GuildConstant -from bot.utils.time import humanize - +from bot.utils.time import humanize_delta log = logging.getLogger(__name__) -BULLET_POINT = "\u2022" -COLOUR_RED = Colour(0xcd6d6d) -COLOUR_GREEN = Colour(0x68c290) GUILD_CHANNEL = Union[CategoryChannel, TextChannel, VoiceChannel] CHANNEL_CHANGES_UNSUPPORTED = ("permissions",) @@ -38,15 +35,69 @@ class ModLog: def __init__(self, bot: Bot): self.bot = bot - self._ignored_deletions = [] + self.headers = {"X-API-KEY": Keys.site_api} + self._ignored = {event: [] for event in Event} self._cached_deletes = [] self._cached_edits = [] - def ignore_message_deletion(self, *message_ids: int): - for message_id in message_ids: - if message_id not in self._ignored_deletions: - self._ignored_deletions.append(message_id) + async def upload_log(self, messages: List[Message]) -> Optional[str]: + """ + Uploads the log data to the database via + an API endpoint for uploading logs. + + Used in several mod log embeds. + + Returns a URL that can be used to view the log. + """ + + log_data = [] + + for message in messages: + author = f"{message.author.name}#{message.author.discriminator}" + + # message.author may return either a User or a Member. Users don't have roles. + if type(message.author) is User: + role_id = Roles.developer + else: + role_id = message.author.top_role.id + + content = message.content + embeds = [embed.to_dict() for embed in message.embeds] + attachments = ["<Attachment>" for _ in message.attachments] + + log_data.append({ + "content": content, + "author": author, + "user_id": str(message.author.id), + "role_id": str(role_id), + "timestamp": message.created_at.strftime("%D %H:%M"), + "attachments": attachments, + "embeds": embeds, + }) + + response = await self.bot.http_session.post( + URLs.site_logs_api, + headers=self.headers, + json={"log_data": log_data} + ) + + try: + data = await response.json() + log_id = data["log_id"] + except (KeyError, ClientResponseError): + log.debug( + "API returned an unexpected result:\n" + f"{response.text}" + ) + return + + return f"{URLs.site_logs_view}/{log_id}" + + def ignore(self, event: Event, *items: int): + for item in items: + if item not in self._ignored[event]: + self._ignored[event].append(item) async def send_log_message( self, icon_url: Optional[str], colour: Colour, title: Optional[str], text: str, thumbnail: str = None, @@ -92,7 +143,7 @@ class ModLog: else: message = f"{channel.name} (`{channel.id}`)" - await self.send_log_message(Icons.hash_green, COLOUR_GREEN, title, message) + await self.send_log_message(Icons.hash_green, Colour(Colours.soft_green), title, message) async def on_guild_channel_delete(self, channel: GUILD_CHANNEL): if channel.guild.id != GuildConstant.id: @@ -111,7 +162,7 @@ class ModLog: message = f"{channel.name} (`{channel.id}`)" await self.send_log_message( - Icons.hash_red, COLOUR_RED, + Icons.hash_red, Colour(Colours.soft_red), title, message ) @@ -157,7 +208,7 @@ class ModLog: message = "" for item in sorted(changes): - message += f"{BULLET_POINT} {item}\n" + message += f"{Emojis.bullet} {item}\n" if after.category: message = f"**{after.category}/#{after.name} (`{after.id}`)**\n{message}" @@ -174,7 +225,7 @@ class ModLog: return await self.send_log_message( - Icons.crown_green, COLOUR_GREEN, + Icons.crown_green, Colour(Colours.soft_green), "Role created", f"`{role.id}`" ) @@ -183,7 +234,7 @@ class ModLog: return await self.send_log_message( - Icons.crown_red, COLOUR_RED, + Icons.crown_red, Colour(Colours.soft_red), "Role removed", f"{role.name} (`{role.id}`)" ) @@ -229,7 +280,7 @@ class ModLog: message = "" for item in sorted(changes): - message += f"{BULLET_POINT} {item}\n" + message += f"{Emojis.bullet} {item}\n" message = f"**{after.name}** (`{after.id}`)\n{message}" @@ -277,7 +328,7 @@ class ModLog: message = "" for item in sorted(changes): - message += f"{BULLET_POINT} {item}\n" + message += f"{Emojis.bullet} {item}\n" message = f"**{after.name}** (`{after.id}`)\n{message}" @@ -291,8 +342,12 @@ class ModLog: if guild.id != GuildConstant.id: return + if member.id in self._ignored[Event.member_ban]: + self._ignored[Event.member_ban].remove(member.id) + return + await self.send_log_message( - Icons.user_ban, COLOUR_RED, + Icons.user_ban, Colour(Colours.soft_red), "User banned", f"{member.name}#{member.discriminator} (`{member.id}`)", thumbnail=member.avatar_url_as(static_format="png") ) @@ -302,17 +357,16 @@ class ModLog: return message = f"{member.name}#{member.discriminator} (`{member.id}`)" - now = datetime.datetime.utcnow() difference = abs(relativedelta(now, member.created_at)) - message += "\n\n**Account age:** " + humanize(difference) + message += "\n\n**Account age:** " + humanize_delta(member.created_at) if difference.days < 1 and difference.months < 1 and difference.years < 1: # New user account! message = f"{Emojis.new} {message}" await self.send_log_message( - Icons.sign_in, COLOUR_GREEN, + Icons.sign_in, Colour(Colours.soft_green), "User joined", message, thumbnail=member.avatar_url_as(static_format="png") ) @@ -321,8 +375,12 @@ class ModLog: if member.guild.id != GuildConstant.id: return + if member.id in self._ignored[Event.member_remove]: + self._ignored[Event.member_remove].remove(member.id) + return + await self.send_log_message( - Icons.sign_out, COLOUR_RED, + Icons.sign_out, Colour(Colours.soft_red), "User left", f"{member.name}#{member.discriminator} (`{member.id}`)", thumbnail=member.avatar_url_as(static_format="png") ) @@ -331,6 +389,10 @@ class ModLog: if guild.id != GuildConstant.id: return + if member.id in self._ignored[Event.member_unban]: + self._ignored[Event.member_unban].remove(member.id) + return + await self.send_log_message( Icons.user_unban, Colour.blurple(), "User unbanned", f"{member.name}#{member.discriminator} (`{member.id}`)", @@ -341,6 +403,10 @@ class ModLog: if before.guild.id != GuildConstant.id: return + if before.id in self._ignored[Event.member_update]: + self._ignored[Event.member_update].remove(before.id) + return + diff = DeepDiff(before, after) changes = [] done = [] @@ -410,7 +476,7 @@ class ModLog: message = "" for item in sorted(changes): - message += f"{BULLET_POINT} {item}\n" + message += f"{Emojis.bullet} {item}\n" message = f"**{after.name}#{after.discriminator}** (`{after.id}`)\n{message}" @@ -430,8 +496,8 @@ class ModLog: ignored_messages = 0 for message_id in event.message_ids: - if message_id in self._ignored_deletions: - self._ignored_deletions.remove(message_id) + if message_id in self._ignored[Event.message_delete]: + self._ignored[Event.message_delete].remove(message_id) ignored_messages += 1 if ignored_messages >= len(event.message_ids): @@ -460,8 +526,8 @@ class ModLog: self._cached_deletes.append(message.id) - if message.id in self._ignored_deletions: - self._ignored_deletions.remove(message.id) + if message.id in self._ignored[Event.message_delete]: + self._ignored[Event.message_delete].remove(message.id) return if author.bot: @@ -473,7 +539,6 @@ class ModLog: f"**Channel:** {channel.category}/#{channel.name} (`{channel.id}`)\n" f"**Message ID:** `{message.id}`\n" "\n" - f"{message.clean_content}" ) else: response = ( @@ -481,15 +546,23 @@ class ModLog: f"**Channel:** #{channel.name} (`{channel.id}`)\n" f"**Message ID:** `{message.id}`\n" "\n" - f"{message.clean_content}" ) + # Shorten the message content if necessary + content = message.clean_content + remaining_chars = 2040 - len(response) + + if len(content) > remaining_chars: + content = content[:remaining_chars] + "..." + + response += f"{content}" + if message.attachments: # Prepend the message metadata with the number of attachments response = f"**Attachments:** {len(message.attachments)}\n" + response await self.send_log_message( - Icons.message_delete, COLOUR_RED, + Icons.message_delete, Colours.soft_red, "Message deleted", response, channel_id=Channels.message_log @@ -506,8 +579,8 @@ class ModLog: self._cached_deletes.remove(event.message_id) return - if event.message_id in self._ignored_deletions: - self._ignored_deletions.remove(event.message_id) + if event.message_id in self._ignored[Event.message_delete]: + self._ignored[Event.message_delete].remove(event.message_id) return channel = self.bot.get_channel(event.channel_id) @@ -528,7 +601,7 @@ class ModLog: ) await self.send_log_message( - Icons.message_delete, COLOUR_RED, + Icons.message_delete, Colour(Colours.soft_red), "Message deleted", response, channel_id=Channels.message_log diff --git a/bot/cogs/off_topic_names.py b/bot/cogs/off_topic_names.py index f089e0b5a..ac2e1269c 100644 --- a/bot/cogs/off_topic_names.py +++ b/bot/cogs/off_topic_names.py @@ -19,19 +19,22 @@ class OffTopicName(Converter): @staticmethod async def convert(ctx: Context, argument: str): + allowed_characters = ("-", "’", "'", "`") + if not (2 <= len(argument) <= 96): raise BadArgument("Channel name must be between 2 and 96 chars long") - elif not all(c.isalnum() or c == '-' for c in argument): + elif not all(c.isalnum() or c in allowed_characters for c in argument): raise BadArgument( - "Channel name must only consist of" - " alphanumeric characters or minus signs" + "Channel name must only consist of " + "alphanumeric characters, minus signs or apostrophes." ) elif not argument.islower(): raise BadArgument("Channel name must be lowercase") - return argument + # Replace some unusable apostrophe-like characters with "’". + return argument.replace("'", "’").replace("`", "’") async def update_names(bot: Bot, headers: dict): @@ -111,6 +114,32 @@ class OffTopicNames: error_reason = response.get('message', "No reason provided.") await ctx.send(f":warning: got non-200 from the API: {error_reason}") + @otname_group.command(name='delete', aliases=('remove', 'rm', 'del', 'd')) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def delete_command(self, ctx, name: OffTopicName): + """Removes a off-topic name from the rotation.""" + + result = await self.bot.http_session.delete( + URLs.site_off_topic_names_api, + headers=self.headers, + params={'name': name} + ) + + response = await result.json() + + if result.status == 200: + if response['deleted'] == 0: + await ctx.send(f":warning: No name matching `{name}` was found in the database.") + else: + log.info( + f"{ctx.author.name}#{ctx.author.discriminator}" + f" deleted the off-topic channel name '{name}" + ) + await ctx.send(":ok_hand:") + else: + error_reason = response.get('message', "No reason provided.") + await ctx.send(f":warning: got non-200 from the API: {error_reason}") + @otname_group.command(name='list', aliases=('l',)) @with_role(Roles.owner, Roles.admin, Roles.moderator) async def list_command(self, ctx): diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py new file mode 100644 index 000000000..952fa4682 --- /dev/null +++ b/bot/cogs/reddit.py @@ -0,0 +1,291 @@ +import asyncio +import logging +import random +import textwrap +from datetime import datetime, timedelta + +from discord import Colour, Embed, TextChannel +from discord.ext.commands import Bot, Context, group + +from bot.constants import Channels, ERROR_REPLIES, Reddit as RedditConfig, Roles +from bot.converters import Subreddit +from bot.decorators import with_role +from bot.pagination import LinePaginator + +log = logging.getLogger(__name__) + + +class Reddit: + """ + Track subreddit posts and show detailed statistics about them. + """ + + HEADERS = {"User-Agent": "Discord Bot: PythonDiscord (https://pythondiscord.com/)"} + URL = "https://www.reddit.com" + + def __init__(self, bot: Bot): + self.bot = bot + + self.reddit_channel = None + + self.prev_lengths = {} + self.last_ids = {} + + async def fetch_posts(self, route: str, *, amount: int = 25, params=None): + """ + A helper method to fetch a certain amount of Reddit posts at a given route. + """ + + # Reddit's JSON responses only provide 25 posts at most. + if not 25 >= amount > 0: + raise ValueError("Invalid amount of subreddit posts requested.") + + if params is None: + params = {} + + response = await self.bot.http_session.get( + url=f"{self.URL}/{route}.json", + headers=self.HEADERS, + params=params + ) + + content = await response.json() + posts = content["data"]["children"] + + return posts[:amount] + + async def send_top_posts(self, channel: TextChannel, subreddit: Subreddit, content=None, time="all"): + """ + Create an embed for the top posts, then send it in a given TextChannel. + """ + + # Create the new spicy embed. + embed = Embed() + embed.description = "" + + # Get the posts + posts = await self.fetch_posts( + route=f"{subreddit}/top", + amount=5, + params={ + "t": time + } + ) + + if not posts: + embed.title = random.choice(ERROR_REPLIES) + embed.colour = Colour.red() + embed.description = ( + "Sorry! We couldn't find any posts from that subreddit. " + "If this problem persists, please let us know." + ) + + return await channel.send( + embed=embed + ) + + for post in posts: + data = post["data"] + + text = data["selftext"] + if text: + text = textwrap.shorten(text, width=128, placeholder="...") + text += "\n" # Add newline to separate embed info + + ups = data["ups"] + comments = data["num_comments"] + author = data["author"] + + title = textwrap.shorten(data["title"], width=64, placeholder="...") + link = self.URL + data["permalink"] + + embed.description += ( + f"[**{title}**]({link})\n" + f"{text}" + f"| {ups} upvotes | {comments} comments | u/{author} | {subreddit} |\n\n" + ) + + embed.colour = Colour.blurple() + + return await channel.send( + content=content, + embed=embed + ) + + async def poll_new_posts(self): + """ + Periodically search for new subreddit posts. + """ + + while True: + await asyncio.sleep(RedditConfig.request_delay) + + for subreddit in RedditConfig.subreddits: + # Make a HEAD request to the subreddit + head_response = await self.bot.http_session.head( + url=f"{self.URL}/{subreddit}/new.rss", + headers=self.HEADERS + ) + + content_length = head_response.headers["content-length"] + + # If the content is the same size as before, assume there's no new posts. + if content_length == self.prev_lengths.get(subreddit, None): + continue + + self.prev_lengths[subreddit] = content_length + + # Now we can actually fetch the new data + posts = await self.fetch_posts(f"{subreddit}/new") + new_posts = [] + + # Only show new posts if we've checked before. + if subreddit in self.last_ids: + for post in posts: + data = post["data"] + + # Convert the ID to an integer for easy comparison. + int_id = int(data["id"], 36) + + # If we've already seen this post, finish checking + if int_id <= self.last_ids[subreddit]: + break + + embed_data = { + "title": textwrap.shorten(data["title"], width=64, placeholder="..."), + "text": textwrap.shorten(data["selftext"], width=128, placeholder="..."), + "url": self.URL + data["permalink"], + "author": data["author"] + } + + new_posts.append(embed_data) + + self.last_ids[subreddit] = int(posts[0]["data"]["id"], 36) + + # Send all of the new posts as spicy embeds + for data in new_posts: + embed = Embed() + + embed.title = data["title"] + embed.url = data["url"] + embed.description = data["text"] + embed.set_footer(text=f"Posted by u/{data['author']} in {subreddit}") + embed.colour = Colour.blurple() + + await self.reddit_channel.send(embed=embed) + + log.trace(f"Sent {len(new_posts)} new {subreddit} posts to channel {self.reddit_channel.id}.") + + async def poll_top_weekly_posts(self): + """ + Post a summary of the top posts every week. + """ + + while True: + now = datetime.utcnow() + + # Calculate the amount of seconds until midnight next monday. + monday = now + timedelta(days=7 - now.weekday()) + monday = monday.replace(hour=0, minute=0, second=0) + until_monday = (monday - now).total_seconds() + + await asyncio.sleep(until_monday) + + for subreddit in RedditConfig.subreddits: + # Send and pin the new weekly posts. + message = await self.send_top_posts( + channel=self.reddit_channel, + subreddit=subreddit, + content=f"This week's top {subreddit} posts have arrived!", + time="week" + ) + + if subreddit.lower() == "r/python": + # Remove the oldest pins so that only 5 remain at most. + pins = await self.reddit_channel.pins() + + while len(pins) >= 5: + await pins[-1].unpin() + del pins[-1] + + await message.pin() + + @group(name="reddit", invoke_without_command=True) + async def reddit_group(self, ctx: Context): + """ + View the top posts from various subreddits. + """ + + await ctx.invoke(self.bot.get_command("help"), "reddit") + + @reddit_group.command(name="top") + async def top_command(self, ctx: Context, subreddit: Subreddit = "r/Python"): + """ + Send the top posts of all time from a given subreddit. + """ + + await self.send_top_posts( + channel=ctx.channel, + subreddit=subreddit, + content=f"Here are the top {subreddit} posts of all time!", + time="all" + ) + + @reddit_group.command(name="daily") + async def daily_command(self, ctx: Context, subreddit: Subreddit = "r/Python"): + """ + Send the top posts of today from a given subreddit. + """ + + await self.send_top_posts( + channel=ctx.channel, + subreddit=subreddit, + content=f"Here are today's top {subreddit} posts!", + time="day" + ) + + @reddit_group.command(name="weekly") + async def weekly_command(self, ctx: Context, subreddit: Subreddit = "r/Python"): + """ + Send the top posts of this week from a given subreddit. + """ + + await self.send_top_posts( + channel=ctx.channel, + subreddit=subreddit, + content=f"Here are this week's top {subreddit} posts!", + time="week" + ) + + @with_role(Roles.owner, Roles.admin, Roles.moderator, Roles.helpers) + @reddit_group.command(name="subreddits", aliases=("subs",)) + async def subreddits_command(self, ctx: Context): + """ + Send a paginated embed of all the subreddits we're relaying. + """ + + embed = Embed() + embed.title = "Relayed subreddits." + embed.colour = Colour.blurple() + + await LinePaginator.paginate( + RedditConfig.subreddits, + ctx, embed, + footer_text="Use the reddit commands along with these to view their posts.", + empty=False, + max_lines=15 + ) + + async def on_ready(self): + self.reddit_channel = self.bot.get_channel(Channels.reddit) + + if self.reddit_channel is not None: + self.bot.loop.create_task(self.poll_new_posts()) + self.bot.loop.create_task(self.poll_top_weekly_posts()) + else: + log.warning("Couldn't locate a channel for subreddit relaying.") + + +def setup(bot): + bot.add_cog(Reddit(bot)) + log.info("Cog loaded: Reddit") diff --git a/bot/cogs/security.py b/bot/cogs/security.py index 443dd08e8..f4a843fbf 100644 --- a/bot/cogs/security.py +++ b/bot/cogs/security.py @@ -13,10 +13,14 @@ class Security: def __init__(self, bot: Bot): self.bot = bot self.bot.check(self.check_not_bot) # Global commands check - no bots can run any commands at all + self.bot.check(self.check_on_guild) # Global commands check - commands can't be run in a DM def check_not_bot(self, ctx: Context): return not ctx.author.bot + def check_on_guild(self, ctx: Context): + return ctx.guild is not None + def setup(bot): bot.add_cog(Security(bot)) diff --git a/bot/cogs/site.py b/bot/cogs/site.py new file mode 100644 index 000000000..e5fd645fb --- /dev/null +++ b/bot/cogs/site.py @@ -0,0 +1,98 @@ +import logging + +from discord import Colour, Embed +from discord.ext.commands import Bot, Context, group + +from bot.constants import URLs + +log = logging.getLogger(__name__) + +INFO_URL = f"{URLs.site_schema}{URLs.site}/info" + + +class Site: + """Commands for linking to different parts of the site.""" + + def __init__(self, bot: Bot): + self.bot = bot + + @group(name="site", aliases=("s",), invoke_without_command=True) + async def site_group(self, ctx): + """Commands for getting info about our website.""" + + await ctx.invoke(self.bot.get_command("help"), "site") + + @site_group.command(name="home", aliases=("about",)) + async def site_main(self, ctx: Context): + """Info about the website itself.""" + + url = f"{URLs.site_schema}{URLs.site}/" + + embed = Embed(title="Python Discord website") + embed.set_footer(text=url) + embed.colour = Colour.blurple() + embed.description = ( + f"[Our official website]({url}) is an open-source community project " + "created with Python and Flask. It contains information about the server " + "itself, lets you sign up for upcoming events, has its own wiki, contains " + "a list of valuable learning resources, and much more." + ) + + await ctx.send(embed=embed) + + @site_group.command(name="resources") + async def site_resources(self, ctx: Context): + """Info about the site's Resources page.""" + + url = f"{INFO_URL}/resources" + + embed = Embed(title="Resources") + embed.set_footer(text=url) + embed.colour = Colour.blurple() + embed.description = ( + f"The [Resources page]({url}) on our website contains a " + "list of hand-selected goodies that we regularly recommend " + "to both beginners and experts." + ) + + await ctx.send(embed=embed) + + @site_group.command(name="help") + async def site_help(self, ctx: Context): + """Info about the site's Getting Help page.""" + + url = f"{INFO_URL}/help" + + embed = Embed(title="Getting Help") + embed.set_footer(text=url) + embed.colour = Colour.blurple() + embed.description = ( + "Asking the right question about something that's new to you can sometimes be tricky. " + f"To help with this, we've created a [guide to asking good questions]({url}) on our website. " + "It contains everything you need to get the very best help from our community." + ) + + await ctx.send(embed=embed) + + @site_group.command(name="faq") + async def site_faq(self, ctx: Context): + """Info about the site's FAQ page.""" + + url = f"{INFO_URL}/faq" + + embed = Embed(title="FAQ") + embed.set_footer(text=url) + embed.colour = Colour.blurple() + embed.description = ( + "As the largest Python community on Discord, we get hundreds of questions every day. " + "Many of these questions have been asked before. We've compiled a list of the most " + "frequently asked questions along with their answers, which can be found on " + f"our [FAQ page]({url})." + ) + + await ctx.send(embed=embed) + + +def setup(bot): + bot.add_cog(Site(bot)) + log.info("Cog loaded: Site") diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 17acf757b..fb9164194 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -2,6 +2,7 @@ import datetime import logging import random import re +import textwrap from discord import Colour, Embed from discord.ext.commands import ( @@ -25,12 +26,29 @@ venv_file = "/snekbox/.venv/bin/activate_this.py" exec(open(venv_file).read(), dict(__file__=venv_file)) try: - {CODE} +{CODE} except Exception as e: print(e) """ ESCAPE_REGEX = re.compile("[`\u202E\u200B]{3,}") +FORMATTED_CODE_REGEX = re.compile( + r"^\s*" # any leading whitespace from the beginning of the string + r"(?P<delim>(?P<block>```)|``?)" # code delimiter: 1-3 backticks; (?P=block) only matches if it's a block + r"(?(block)(?:(?P<lang>[a-z]+)\n)?)" # if we're in a block, match optional language (only letters plus newline) + r"(?:[ \t]*\n)*" # any blank (empty or tabs/spaces only) lines before the code + r"(?P<code>.*?)" # extract all code inside the markup + r"\s*" # any more whitespace before the end of the code markup + r"(?P=delim)" # match the exact same delimiter from the start again + r"\s*$", # any trailing whitespace until the end of the string + re.DOTALL | re.IGNORECASE # "." also matches newlines, case insensitive +) +RAW_CODE_REGEX = re.compile( + r"^(?:[ \t]*\n)*" # any blank (empty or tabs/spaces only) lines before the code + r"(?P<code>.*?)" # extract all the rest as code + r"\s*$", # any trailing whitespace until the end of the string + re.DOTALL # "." also matches newlines +) BYPASS_ROLES = (Roles.owner, Roles.admin, Roles.moderator, Roles.helpers) WHITELISTED_CHANNELS = (Channels.bot,) WHITELISTED_CHANNELS_STRING = ', '.join(f"<#{channel_id}>" for channel_id in WHITELISTED_CHANNELS) @@ -53,8 +71,6 @@ class Snekbox: Safe evaluation using Snekbox """ - jobs = None # type: dict - def __init__(self, bot: Bot): self.bot = bot self.jobs = {} @@ -66,32 +82,40 @@ class Snekbox: @command(name='eval', aliases=('e',)) @guild_only() @check(channel_is_whitelisted_or_author_can_bypass) - async def eval_command(self, ctx: Context, *, code: str): + async def eval_command(self, ctx: Context, *, code: str = None): """ Run some code. get the result back. We've done our best to make this safe, but do let us know if you manage to find an issue with it! + + This command supports multiple lines of code, including code wrapped inside a formatted code block. """ if ctx.author.id in self.jobs: await ctx.send(f"{ctx.author.mention} You've already got a job running - please wait for it to finish!") return + if not code: # None or empty string + return await ctx.invoke(self.bot.get_command("help"), "eval") + log.info(f"Received code from {ctx.author.name}#{ctx.author.discriminator} for evaluation:\n{code}") self.jobs[ctx.author.id] = datetime.datetime.now() - while code.startswith("\n"): - code = code[1:] - - if code.startswith("```") and code.endswith("```"): - code = code[3:-3] - - if code.startswith("python"): - code = code[6:] - elif code.startswith("py"): - code = code[2:] + # Strip whitespace and inline or block code markdown and extract the code and some formatting info + match = FORMATTED_CODE_REGEX.fullmatch(code) + if match: + code, block, lang, delim = match.group("code", "block", "lang", "delim") + code = textwrap.dedent(code) + if block: + info = (f"'{lang}' highlighted" if lang else "plain") + " code block" + else: + info = f"{delim}-enclosed inline code" + log.trace(f"Extracted {info} for evaluation:\n{code}") + else: + code = textwrap.dedent(RAW_CODE_REGEX.fullmatch(code).group("code")) + log.trace(f"Eval message contains not or badly formatted code, stripping whitespace only:\n{code}") - code = [f" {line.strip()}" for line in code.split("\n")] - code = CODE_TEMPLATE.replace("{CODE}", "\n".join(code)) + code = textwrap.indent(code, " ") + code = CODE_TEMPLATE.replace("{CODE}", code) try: await self.rmq.send_json( diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index c8621118b..8277513a7 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -5,12 +5,12 @@ import re import struct from datetime import datetime -from discord import Message +from discord import Colour, Message from discord.ext.commands import Bot from discord.utils import snowflake_time -from bot.constants import Channels - +from bot.cogs.modlog import ModLog +from bot.constants import Channels, Colours, Event, Icons log = logging.getLogger(__name__) @@ -26,12 +26,12 @@ DISCORD_EPOCH_TIMESTAMP = datetime(2017, 1, 1) TOKEN_EPOCH = 1_293_840_000 TOKEN_RE = re.compile( r"(?<=(\"|'))" # Lookbehind: Only match if there's a double or single quote in front - r"[^\W\.]+" # Matches token part 1: The user ID string, encoded as base64 - r"\." # Matches a literal dot between the token parts - r"[^\W\.]+" # Matches token part 2: The creation timestamp, as an integer - r"\." # Matches a literal dot between the token parts - r"[^\W\.]+" # Matches token part 3: The HMAC, unused by us, but check that it isn't empty - r"(?=(\"|'))" # Lookahead: Only match if there's a double or single quote after + r"[^\s\.]+" # Matches token part 1: The user ID string, encoded as base64 + r"\." # Matches a literal dot between the token parts + r"[^\s\.]+" # Matches token part 2: The creation timestamp, as an integer + r"\." # Matches a literal dot between the token parts + r"[^\s\.]+" # Matches token part 3: The HMAC, unused by us, but check that it isn't empty + r"(?=(\"|'))" # Lookahead: Only match if there's a double or single quote after ) @@ -40,10 +40,10 @@ class TokenRemover: def __init__(self, bot: Bot): self.bot = bot - self.modlog = None - async def on_ready(self): - self.modlog = self.bot.get_channel(Channels.modlog) + @property + def mod_log(self) -> ModLog: + return self.bot.get_cog("ModLog") async def on_message(self, msg: Message): if msg.author.bot: @@ -59,13 +59,26 @@ class TokenRemover: return if self.is_valid_user_id(user_id) and self.is_valid_timestamp(creation_timestamp): + self.mod_log.ignore(Event.message_delete, msg.id) await msg.delete() await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention)) - await self.modlog.send( - ":key2::mute: censored a seemingly valid token sent by " + + message = ( + "Censored a seemingly valid token sent by " f"{msg.author} (`{msg.author.id}`) in {msg.channel.mention}, token was " f"`{user_id}.{creation_timestamp}.{'x' * len(hmac)}`" ) + log.debug(message) + + # Send pretty mod log embed to mod-alerts + await self.mod_log.send_log_message( + icon_url=Icons.token_removed, + colour=Colour(Colours.soft_red), + title="Token removed!", + text=message, + thumbnail=msg.author.avatar_url_as(static_format="png"), + channel_id=Channels.mod_alerts, + ) @staticmethod def is_valid_user_id(b64_content: str) -> bool: diff --git a/bot/cogs/utils.py b/bot/cogs/utils.py index 22e0cfbe7..b101b8816 100644 --- a/bot/cogs/utils.py +++ b/bot/cogs/utils.py @@ -30,11 +30,16 @@ class Utils: Fetches information about a PEP and sends it to the channel. """ + if pep_number.isdigit(): + pep_number = int(pep_number) + else: + return await ctx.invoke(self.bot.get_command("help"), "pep") + # Newer PEPs are written in RST instead of txt - if int(pep_number) > 542: - pep_url = f"{self.base_github_pep_url}{pep_number.zfill(4)}.rst" + if pep_number > 542: + pep_url = f"{self.base_github_pep_url}{pep_number:04}.rst" else: - pep_url = f"{self.base_github_pep_url}{pep_number.zfill(4)}.txt" + pep_url = f"{self.base_github_pep_url}{pep_number:04}.txt" # Attempt to fetch the PEP log.trace(f"Requesting PEP {pep_number} with {pep_url}") @@ -51,7 +56,7 @@ class Utils: # Assemble the embed pep_embed = Embed( title=f"**PEP {pep_number} - {pep_header['Title']}**", - description=f"[Link]({self.base_pep_url}{pep_number.zfill(4)})", + description=f"[Link]({self.base_pep_url}{pep_number:04})", ) pep_embed.set_thumbnail(url="https://www.python.org/static/opengraph-icon-200x200.png") diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index b0667fdd0..8d29a4bee 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -4,7 +4,7 @@ from discord import Message, NotFound, Object from discord.ext.commands import Bot, Context, command from bot.cogs.modlog import ModLog -from bot.constants import Channels, Roles +from bot.constants import Channels, Event, Roles from bot.decorators import in_channel, without_role log = logging.getLogger(__name__) @@ -37,7 +37,7 @@ class Verification: self.bot = bot @property - def modlog(self) -> ModLog: + def mod_log(self) -> ModLog: return self.bot.get_cog("ModLog") async def on_message(self, message: Message): @@ -90,7 +90,7 @@ class Verification: log.trace(f"Deleting the message posted by {ctx.author}.") try: - self.modlog.ignore_message_deletion(ctx.message.id) + self.mod_log.ignore(Event.message_delete, ctx.message.id) await ctx.message.delete() except NotFound: log.trace("No message found, it must have been deleted by another bot.") @@ -110,7 +110,7 @@ class Verification: break if has_role: - await ctx.send( + return await ctx.send( f"{ctx.author.mention} You're already subscribed!", ) diff --git a/bot/constants.py b/bot/constants.py index 205b09111..3ade4ac7b 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -13,8 +13,9 @@ their default values from `config-default.yml`. import logging import os from collections.abc import Mapping +from enum import Enum from pathlib import Path -from typing import List +from typing import Dict, List import yaml from yaml.constructor import ConstructorError @@ -98,7 +99,7 @@ yaml.SafeLoader.add_constructor("!JOIN", _join_var_constructor) yaml.SafeLoader.add_constructor("!REQUIRED_ENV", _required_env_var_constructor) -with open("config-default.yml") as f: +with open("config-default.yml", encoding="UTF-8") as f: _CONFIG_YAML = yaml.safe_load(f) @@ -123,7 +124,7 @@ def _recursive_update(original, new): if Path("config.yml").exists(): log.info("Found `config.yml` file, loading constants from it.") - with open("config.yml") as f: + with open("config.yml", encoding="UTF-8") as f: user_config = yaml.safe_load(f) _recursive_update(_CONFIG_YAML, user_config) @@ -191,6 +192,26 @@ class Bot(metaclass=YAMLGetter): token: str +class Filter(metaclass=YAMLGetter): + section = "filter" + + filter_zalgo: bool + filter_invites: bool + filter_domains: bool + watch_words: bool + watch_tokens: bool + + ping_everyone: bool + guild_invite_whitelist: List[str] + vanity_url_whitelist: List[str] + domain_blacklist: List[str] + word_watchlist: List[str] + token_watchlist: List[str] + + channel_whitelist: List[int] + role_whitelist: List[int] + + class Cooldowns(metaclass=YAMLGetter): section = "bot" subsection = "cooldowns" @@ -198,8 +219,16 @@ class Cooldowns(metaclass=YAMLGetter): tags: int +class Colours(metaclass=YAMLGetter): + section = "style" + subsection = "colours" + + soft_red: int + soft_green: int + + class Emojis(metaclass=YAMLGetter): - section = "bot" + section = "style" subsection = "emojis" defcon_disabled: str # noqa: E704 @@ -209,23 +238,33 @@ class Emojis(metaclass=YAMLGetter): green_chevron: str red_chevron: str white_chevron: str + lemoneye2: str + + status_online: str + status_offline: str + status_idle: str + status_dnd: str + bullet: str new: str pencil: str + cross_mark: str class Icons(metaclass=YAMLGetter): - section = "bot" + section = "style" subsection = "icons" crown_blurple: str crown_green: str crown_red: str - defcon_denied: str # noqa: E704 + defcon_denied: str # noqa: E704 defcon_disabled: str # noqa: E704 - defcon_enabled: str # noqa: E704 - defcon_updated: str # noqa: E704 + defcon_enabled: str # noqa: E704 + defcon_updated: str # noqa: E704 + + filtering: str guild_update: str @@ -240,10 +279,24 @@ class Icons(metaclass=YAMLGetter): sign_in: str sign_out: str + token_removed: str + user_ban: str user_unban: str user_update: str + user_mute: str + user_unmute: str + + pencil: str + + +class CleanMessages(metaclass=YAMLGetter): + section = "bot" + subsection = "clean" + + message_limit: int + class Channels(metaclass=YAMLGetter): section = "guild" @@ -265,11 +318,13 @@ class Channels(metaclass=YAMLGetter): help_5: int helpers: int message_log: int + mod_alerts: int modlog: int off_topic_1: int off_topic_2: int off_topic_3: int python: int + reddit: int verification: int @@ -281,9 +336,11 @@ class Roles(metaclass=YAMLGetter): announcements: int champion: int contributor: int + developer: int devops: int jammer: int moderator: int + muted: int owner: int verified: int muted: int @@ -306,14 +363,6 @@ class Keys(metaclass=YAMLGetter): youtube: str -class ClickUp(metaclass=YAMLGetter): - section = "clickup" - - key: str - space: int - team: int - - class RabbitMQ(metaclass=YAMLGetter): section = "rabbitmq" @@ -331,9 +380,13 @@ class URLs(metaclass=YAMLGetter): gitlab_bot_repo: str omdb: str site: str + site_api: str site_facts_api: str + site_clean_api: str site_hiphopify_api: str site_idioms_api: str + site_logs_api: str + site_logs_view: str site_names_api: str site_quiz_api: str site_schema: str @@ -345,12 +398,30 @@ class URLs(metaclass=YAMLGetter): site_infractions: str site_infractions_user: str site_infractions_type: str + site_infractions_by_id: str site_infractions_user_type_current: str site_infractions_user_type: str status: str paste_service: str +class Reddit(metaclass=YAMLGetter): + section = "reddit" + + request_delay: int + subreddits: list + + +class AntiSpam(metaclass=YAMLGetter): + section = 'anti_spam' + + clean_offending: bool + ping_everyone: bool + + punishment: Dict[str, Dict[str, int]] + rules: Dict[str, Dict[str, int]] + + # Debug mode DEBUG_MODE = True if 'local' in os.environ.get("SITE_URL", "local") else False @@ -408,3 +479,27 @@ ERROR_REPLIES = [ "Are you trying to kill me?", "Noooooo!!" ] + + +class Event(Enum): + """ + Event names. This does not include every event (for example, raw + events aren't here), but only events used in ModLog for now. + """ + + guild_channel_create = "guild_channel_create" + guild_channel_delete = "guild_channel_delete" + guild_channel_update = "guild_channel_update" + guild_role_create = "guild_role_create" + guild_role_delete = "guild_role_delete" + guild_role_update = "guild_role_update" + guild_update = "guild_update" + + member_join = "member_join" + member_remove = "member_remove" + member_ban = "member_ban" + member_unban = "member_unban" + member_update = "member_update" + + message_delete = "message_delete" + message_edit = "message_edit" diff --git a/bot/converters.py b/bot/converters.py index f18b2f6c7..3def4b07a 100644 --- a/bot/converters.py +++ b/bot/converters.py @@ -172,3 +172,29 @@ class InfractionSearchQuery(Converter): except Exception: return arg return user or arg + + +class Subreddit(Converter): + """ + Forces a string to begin with "r/" and checks if it's a valid subreddit. + """ + + @staticmethod + async def convert(ctx, sub: str): + sub = sub.lower() + + if not sub.startswith("r/"): + sub = f"r/{sub}" + + resp = await ctx.bot.http_session.get( + "https://www.reddit.com/subreddits/search.json", + params={"q": sub} + ) + + json = await resp.json() + if not json["data"]["children"]: + raise BadArgument( + f"The subreddit `{sub}` either doesn't exist, or it has no posts." + ) + + return sub diff --git a/bot/rules/__init__.py b/bot/rules/__init__.py new file mode 100644 index 000000000..a01ceae73 --- /dev/null +++ b/bot/rules/__init__.py @@ -0,0 +1,12 @@ +# flake8: noqa + +from .attachments import apply as apply_attachments +from .burst import apply as apply_burst +from .burst_shared import apply as apply_burst_shared +from .chars import apply as apply_chars +from .discord_emojis import apply as apply_discord_emojis +from .duplicates import apply as apply_duplicates +from .links import apply as apply_links +from .mentions import apply as apply_mentions +from .newlines import apply as apply_newlines +from .role_mentions import apply as apply_role_mentions diff --git a/bot/rules/attachments.py b/bot/rules/attachments.py new file mode 100644 index 000000000..47b927101 --- /dev/null +++ b/bot/rules/attachments.py @@ -0,0 +1,30 @@ +"""Detects total attachments exceeding the limit sent by a single user.""" + +from typing import Dict, Iterable, List, Optional, Tuple + +from discord import Member, Message + + +async def apply( + last_message: Message, + recent_messages: List[Message], + config: Dict[str, int] +) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: + + relevant_messages = tuple( + msg + for msg in recent_messages + if ( + msg.author == last_message.author + and len(msg.attachments) > 0 + ) + ) + total_recent_attachments = sum(len(msg.attachments) for msg in relevant_messages) + + if total_recent_attachments > config['max']: + return ( + f"sent {total_recent_attachments} attachments in {config['max']}s", + (last_message.author,), + relevant_messages + ) + return None diff --git a/bot/rules/burst.py b/bot/rules/burst.py new file mode 100644 index 000000000..80c79be60 --- /dev/null +++ b/bot/rules/burst.py @@ -0,0 +1,27 @@ +"""Detects repeated messages sent by a single user.""" + +from typing import Dict, Iterable, List, Optional, Tuple + +from discord import Member, Message + + +async def apply( + last_message: Message, + recent_messages: List[Message], + config: Dict[str, int] +) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: + + relevant_messages = tuple( + msg + for msg in recent_messages + if msg.author == last_message.author + ) + total_relevant = len(relevant_messages) + + if total_relevant > config['max']: + return ( + f"sent {total_relevant} messages in {config['interval']}s", + (last_message.author,), + relevant_messages + ) + return None diff --git a/bot/rules/burst_shared.py b/bot/rules/burst_shared.py new file mode 100644 index 000000000..2cb7b5200 --- /dev/null +++ b/bot/rules/burst_shared.py @@ -0,0 +1,22 @@ +"""Detects repeated messages sent by multiple users.""" + +from typing import Dict, Iterable, List, Optional, Tuple + +from discord import Member, Message + + +async def apply( + last_message: Message, + recent_messages: List[Message], + config: Dict[str, int] +) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: + + total_recent = len(recent_messages) + + if total_recent > config['max']: + return ( + f"sent {total_recent} messages in {config['interval']}s", + set(msg.author for msg in recent_messages), + recent_messages + ) + return None diff --git a/bot/rules/chars.py b/bot/rules/chars.py new file mode 100644 index 000000000..d05e3cd83 --- /dev/null +++ b/bot/rules/chars.py @@ -0,0 +1,28 @@ +"""Detects total message char count exceeding the limit sent by a single user.""" + +from typing import Dict, Iterable, List, Optional, Tuple + +from discord import Member, Message + + +async def apply( + last_message: Message, + recent_messages: List[Message], + config: Dict[str, int] +) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: + + relevant_messages = tuple( + msg + for msg in recent_messages + if msg.author == last_message.author + ) + + total_recent_chars = sum(len(msg.content) for msg in relevant_messages) + + if total_recent_chars > config['max']: + return ( + f"sent {total_recent_chars} characters in {config['interval']}s", + (last_message.author,), + relevant_messages + ) + return None diff --git a/bot/rules/discord_emojis.py b/bot/rules/discord_emojis.py new file mode 100644 index 000000000..e4f957ddb --- /dev/null +++ b/bot/rules/discord_emojis.py @@ -0,0 +1,35 @@ +"""Detects total Discord emojis (excluding Unicode emojis) exceeding the limit sent by a single user.""" + +import re +from typing import Dict, Iterable, List, Optional, Tuple + +from discord import Member, Message + + +DISCORD_EMOJI_RE = re.compile(r"<:\w+:\d+>") + + +async def apply( + last_message: Message, + recent_messages: List[Message], + config: Dict[str, int] +) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: + + relevant_messages = tuple( + msg + for msg in recent_messages + if msg.author == last_message.author + ) + + total_emojis = sum( + len(DISCORD_EMOJI_RE.findall(msg.content)) + for msg in relevant_messages + ) + + if total_emojis > config['max']: + return ( + f"sent {total_emojis} emojis in {config['interval']}s", + (last_message.author,), + relevant_messages + ) + return None diff --git a/bot/rules/duplicates.py b/bot/rules/duplicates.py new file mode 100644 index 000000000..763fc9983 --- /dev/null +++ b/bot/rules/duplicates.py @@ -0,0 +1,31 @@ +"""Detects duplicated messages sent by a single user.""" + +from typing import Dict, Iterable, List, Optional, Tuple + +from discord import Member, Message + + +async def apply( + last_message: Message, + recent_messages: List[Message], + config: Dict[str, int] +) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: + + relevant_messages = tuple( + msg + for msg in recent_messages + if ( + msg.author == last_message.author + and msg.content == last_message.content + ) + ) + + total_duplicated = len(relevant_messages) + + if total_duplicated > config['max']: + return ( + f"sent {total_duplicated} duplicated messages in {config['interval']}s", + (last_message.author,), + relevant_messages + ) + return None diff --git a/bot/rules/links.py b/bot/rules/links.py new file mode 100644 index 000000000..dfeb38c61 --- /dev/null +++ b/bot/rules/links.py @@ -0,0 +1,31 @@ +"""Detects total links exceeding the limit sent by a single user.""" + +import re +from typing import Dict, Iterable, List, Optional, Tuple + +from discord import Member, Message + + +LINK_RE = re.compile(r"(https?://[^\s]+)") + + +async def apply( + last_message: Message, + recent_messages: List[Message], + config: Dict[str, int] +) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: + + relevant_messages = tuple( + msg + for msg in recent_messages + if msg.author == last_message.author + ) + total_links = sum(len(LINK_RE.findall(msg.content)) for msg in relevant_messages) + + if total_links > config['max']: + return ( + f"sent {total_links} links in {config['interval']}s", + (last_message.author,), + relevant_messages + ) + return None diff --git a/bot/rules/mentions.py b/bot/rules/mentions.py new file mode 100644 index 000000000..45c47b6ba --- /dev/null +++ b/bot/rules/mentions.py @@ -0,0 +1,28 @@ +"""Detects total mentions exceeding the limit sent by a single user.""" + +from typing import Dict, Iterable, List, Optional, Tuple + +from discord import Member, Message + + +async def apply( + last_message: Message, + recent_messages: List[Message], + config: Dict[str, int] +) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: + + relevant_messages = tuple( + msg + for msg in recent_messages + if msg.author == last_message.author + ) + + total_recent_mentions = sum(len(msg.mentions) for msg in relevant_messages) + + if total_recent_mentions > config['max']: + return ( + f"sent {total_recent_mentions} mentions in {config['interval']}s", + (last_message.author,), + relevant_messages + ) + return None diff --git a/bot/rules/newlines.py b/bot/rules/newlines.py new file mode 100644 index 000000000..a6a1a52d0 --- /dev/null +++ b/bot/rules/newlines.py @@ -0,0 +1,28 @@ +"""Detects total newlines exceeding the set limit sent by a single user.""" + +from typing import Dict, Iterable, List, Optional, Tuple + +from discord import Member, Message + + +async def apply( + last_message: Message, + recent_messages: List[Message], + config: Dict[str, int] +) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: + + relevant_messages = tuple( + msg + for msg in recent_messages + if msg.author == last_message.author + ) + + total_recent_newlines = sum(msg.content.count('\n') for msg in relevant_messages) + + if total_recent_newlines > config['max']: + return ( + f"sent {total_recent_newlines} newlines in {config['interval']}s", + (last_message.author,), + relevant_messages + ) + return None diff --git a/bot/rules/role_mentions.py b/bot/rules/role_mentions.py new file mode 100644 index 000000000..2177a73b5 --- /dev/null +++ b/bot/rules/role_mentions.py @@ -0,0 +1,28 @@ +"""Detects total role mentions exceeding the limit sent by a single user.""" + +from typing import Dict, Iterable, List, Optional, Tuple + +from discord import Member, Message + + +async def apply( + last_message: Message, + recent_messages: List[Message], + config: Dict[str, int] +) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: + + relevant_messages = tuple( + msg + for msg in recent_messages + if msg.author == last_message.author + ) + + total_recent_mentions = sum(len(msg.role_mentions) for msg in relevant_messages) + + if total_recent_mentions > config['max']: + return ( + f"sent {total_recent_mentions} role mentions in {config['interval']}s", + (last_message.author,), + relevant_messages + ) + return None diff --git a/bot/utils/time.py b/bot/utils/time.py index b3f55932c..77cef4670 100644 --- a/bot/utils/time.py +++ b/bot/utils/time.py @@ -1,59 +1,91 @@ +import datetime + from dateutil.relativedelta import relativedelta -def _plural_timestring(value: int, unit: str) -> str: +def _stringify_time_unit(value: int, unit: str): """ - Takes a value and a unit type, - such as 24 and "hours". - - Returns a string that takes - the correct plural into account. + Returns a string to represent a value and time unit, + ensuring that it uses the right plural form of the unit. - >>> _plural_timestring(1, "seconds") + >>> _stringify_time_unit(1, "seconds") "1 second" - >>> _plural_timestring(24, "hours") + >>> _stringify_time_unit(24, "hours") "24 hours" + >>> _stringify_time_unit(0, "minutes") + "less than a minute" """ if value == 1: return f"{value} {unit[:-1]}" + elif value == 0: + return f"less than a {unit[:-1]}" else: return f"{value} {unit}" -def humanize(delta: relativedelta, accuracy: str = "seconds") -> str: +def humanize_delta(delta: relativedelta, precision: str = "seconds", max_units: int = 6): """ - This takes a relativedelta and - returns a nice human readable string. + Returns a human-readable version of the relativedelta. - "4 days, 12 hours and 1 second" + :param delta: A dateutil.relativedelta.relativedelta object + :param precision: The smallest unit that should be included. + :param max_units: The maximum number of time-units to return. - :param delta: A dateutils.relativedelta.relativedelta object - :param accuracy: The smallest unit that should be included. - :return: A humanized string. + :return: A string like `4 days, 12 hours and 1 second`, + `1 minute`, or `less than a minute`. """ - units = { - "years": delta.years, - "months": delta.months, - "days": delta.days, - "hours": delta.hours, - "minutes": delta.minutes, - "seconds": delta.seconds - } + units = ( + ("years", delta.years), + ("months", delta.months), + ("days", delta.days), + ("hours", delta.hours), + ("minutes", delta.minutes), + ("seconds", delta.seconds), + ) - # Add the time units that are >0, but stop at accuracy. + # Add the time units that are >0, but stop at accuracy or max_units. time_strings = [] - for unit, value in units.items(): + unit_count = 0 + for unit, value in units: if value: - time_strings.append(_plural_timestring(value, unit)) + time_strings.append(_stringify_time_unit(value, unit)) + unit_count += 1 - if unit == accuracy: + if unit == precision or unit_count >= max_units: break - # Add the 'and' between the last two units + # Add the 'and' between the last two units, if necessary if len(time_strings) > 1: time_strings[-1] = f"{time_strings[-2]} and {time_strings[-1]}" del time_strings[-2] - return ", ".join(time_strings) + # If nothing has been found, just make the value 0 precision, e.g. `0 days`. + if not time_strings: + humanized = _stringify_time_unit(0, precision) + else: + humanized = ", ".join(time_strings) + + return humanized + + +def time_since(past_datetime: datetime.datetime, precision: str = "seconds", max_units: int = 6): + """ + Takes a datetime and returns a human-readable string that + describes how long ago that datetime was. + + :param past_datetime: A datetime.datetime object + :param precision: The smallest unit that should be included. + :param max_units: The maximum number of time-units to return. + + :return: A string like `4 days, 12 hours and 1 second ago`, + `1 minute ago`, or `less than a minute ago`. + """ + + now = datetime.datetime.utcnow() + delta = abs(relativedelta(now, past_datetime)) + + humanized = humanize_delta(delta, precision, max_units) + + return f"{humanized} ago" diff --git a/config-default.yml b/config-default.yml index ee3e6a74e..b621c5b90 100644 --- a/config-default.yml +++ b/config-default.yml @@ -6,6 +6,16 @@ bot: # Per channel, per tag. tags: 60 + clean: + # Maximum number of messages to traverse for clean commands + message_limit: 10000 + + +style: + colours: + soft_red: 0xcd6d6d + soft_green: 0x68c290 + emojis: defcon_disabled: "<:defcondisabled:470326273952972810>" defcon_enabled: "<:defconenabled:470326274213150730>" @@ -16,8 +26,15 @@ bot: white_chevron: "<:whitechevron:418110396973711363>" lemoneye2: "<:lemoneye2:435193765582340098>" - pencil: "\u270F" - new: "\U0001F195" + status_online: "<:status_online:470326272351010816>" + status_idle: "<:status_idle:470326266625785866>" + status_dnd: "<:status_dnd:470326272082313216>" + status_offline: "<:status_offline:470326266537705472>" + + bullet: "\u2022" + pencil: "\u270F" + new: "\U0001F195" + cross_mark: "\u274C" icons: crown_blurple: "https://cdn.discordapp.com/emojis/469964153289965568.png" @@ -29,6 +46,8 @@ bot: defcon_enabled: "https://cdn.discordapp.com/emojis/470326274213150730.png" defcon_updated: "https://cdn.discordapp.com/emojis/472472638342561793.png" + filtering: "https://cdn.discordapp.com/emojis/472472638594482195.png" + guild_update: "https://cdn.discordapp.com/emojis/469954765141442561.png" hash_blurple: "https://cdn.discordapp.com/emojis/469950142942806017.png" @@ -42,52 +61,133 @@ bot: sign_in: "https://cdn.discordapp.com/emojis/469952898181234698.png" sign_out: "https://cdn.discordapp.com/emojis/469952898089091082.png" + token_removed: "https://cdn.discordapp.com/emojis/470326273298792469.png" + user_ban: "https://cdn.discordapp.com/emojis/469952898026045441.png" user_unban: "https://cdn.discordapp.com/emojis/469952898692808704.png" user_update: "https://cdn.discordapp.com/emojis/469952898684551168.png" + user_mute: "https://cdn.discordapp.com/emojis/472472640100106250.png" + user_unmute: "https://cdn.discordapp.com/emojis/472472639206719508.png" + + pencil: "https://cdn.discordapp.com/emojis/470326272401211415.png" + guild: id: 267624335836053506 channels: - admins: &ADMINS 365960823622991872 - announcements: 354619224620138496 - big_brother_logs: 468507907357409333 - bot: 267659945086812160 - checkpoint_test: 422077681434099723 - devalerts: 460181980097675264 - devlog: 409308876241108992 - devtest: 414574275865870337 - help_0: 303906576991780866 - help_1: 303906556754395136 - help_2: 303906514266226689 - help_3: 439702951246692352 - help_4: 451312046647148554 - help_5: 454941769734422538 - helpers: 385474242440986624 - message_log: &MESSAGE_LOG 467752170159079424 - modlog: &MODLOG 282638479504965634 - off_topic_0: 291284109232308226 - off_topic_1: 463035241142026251 - off_topic_2: 463035268514185226 - python: 267624335836053506 - verification: 352442727016693763 + admins: &ADMINS 365960823622991872 + announcements: 354619224620138496 + big_brother_logs: &BBLOGS 468507907357409333 + bot: 267659945086812160 + checkpoint_test: 422077681434099723 + devalerts: 460181980097675264 + devlog: &DEVLOG 409308876241108992 + devtest: &DEVTEST 414574275865870337 + help_0: 303906576991780866 + help_1: 303906556754395136 + help_2: 303906514266226689 + help_3: 439702951246692352 + help_4: 451312046647148554 + help_5: 454941769734422538 + helpers: 385474242440986624 + message_log: &MESSAGE_LOG 467752170159079424 + mod_alerts: 473092532147060736 + modlog: &MODLOG 282638479504965634 + off_topic_0: 291284109232308226 + off_topic_1: 463035241142026251 + off_topic_2: 463035268514185226 + python: 267624335836053506 + reddit: 458224812528238616 + staff_lounge: &STAFF_LOUNGE 464905259261755392 + verification: 352442727016693763 ignored: [*ADMINS, *MESSAGE_LOG, *MODLOG] roles: - admin: 267628507062992896 - announcements: 463658397560995840 - champion: 430492892331769857 - contributor: 295488872404484098 - devops: 409416496733880320 - jammer: 423054537079783434 - moderator: 267629731250176001 - owner: 267627879762755584 - verified: 352427296948486144 - helpers: 267630620367257601 - muted: 277914926603829249 + admin: &ADMIN_ROLE 267628507062992896 + announcements: 463658397560995840 + champion: 430492892331769857 + contributor: 295488872404484098 + developer: 352427296948486144 + devops: &DEVOPS_ROLE 409416496733880320 + jammer: 423054537079783434 + moderator: &MOD_ROLE 267629731250176001 + muted: &MUTED_ROLE 277914926603829249 + owner: &OWNER_ROLE 267627879762755584 + verified: 352427296948486144 + helpers: 267630620367257601 + rockstars: &ROCKSTARS_ROLE 458226413825294336 + + +filter: + + # What do we filter? + filter_zalgo: true + filter_invites: true + filter_domains: true + watch_words: true + watch_tokens: true + + # Filter configuration + ping_everyone: true # Ping @everyone when we send a mod-alert? + + guild_invite_whitelist: + - kWJYurV # Functional Programming + - XBGetGp # STEM + + vanity_url_whitelist: + - python # Python Discord + + domain_blacklist: + - pornhub.com + - liveleak.com + + word_watchlist: + - goo+ks* + - ky+s+ + - gh?[ae]+y+s* + - ki+ke+s* + - beaner+s? + - coo+ns* + - nig+lets* + - slant-eyes* + - towe?l-?head+s* + - chi*n+k+s* + - spick*s* + - kill* +(?:yo)?urself+ + - jew+s* + - suicide + - rape + - (re+)tar+(d+|t+)(ed)? + - ta+r+d+ + - cunts* + + token_watchlist: + - fa+g+s* + - 卐 + - 卍 + - cuck(?!oo+) + - nigg+(?:e*r+|a+h*?|u+h+)s? + - fag+o+t+s* + + # Censor doesn't apply to these + channel_whitelist: + - *ADMINS + - *MODLOG + - *MESSAGE_LOG + - *DEVLOG + - *BBLOGS + - *STAFF_LOUNGE + - *DEVTEST + + role_whitelist: + - *ADMIN_ROLE + - *MOD_ROLE + - *OWNER_ROLE + - *DEVOPS_ROLE + - *ROCKSTARS_ROLE keys: @@ -98,12 +198,6 @@ keys: youtube: !ENV "YOUTUBE_API_KEY" -clickup: - key: !ENV "CLICKUP_KEY" - space: 757069 - team: 754996 - - rabbitmq: host: "pdrmq" password: !ENV ["RABBITMQ_DEFAULT_PASS", "guest"] @@ -113,27 +207,30 @@ rabbitmq: urls: # PyDis site vars - site: &DOMAIN "api.pythondiscord.com" - site_schema: &SCHEMA "https://" - - site_bigbrother_api: !JOIN [*SCHEMA, *DOMAIN, "/bot/bigbrother"] - site_docs_api: !JOIN [*SCHEMA, *DOMAIN, "/bot/docs"] - site_facts_api: !JOIN [*SCHEMA, *DOMAIN, "/bot/snake_facts"] - site_hiphopify_api: !JOIN [*SCHEMA, *DOMAIN, "/bot/hiphopify"] - site_idioms_api: !JOIN [*SCHEMA, *DOMAIN, "/bot/snake_idioms"] - site_names_api: !JOIN [*SCHEMA, *DOMAIN, "/bot/snake_names"] - site_off_topic_names_api: !JOIN [*SCHEMA, *DOMAIN, "/bot/off-topic-names"] - site_quiz_api: !JOIN [*SCHEMA, *DOMAIN, "/bot/snake_quiz"] - site_settings_api: !JOIN [*SCHEMA, *DOMAIN, "/bot/settings"] - site_special_api: !JOIN [*SCHEMA, *DOMAIN, "/bot/special_snakes"] - site_tags_api: !JOIN [*SCHEMA, *DOMAIN, "/bot/tags"] - site_user_api: !JOIN [*SCHEMA, *DOMAIN, "/bot/users"] - site_user_complete_api: !JOIN [*SCHEMA, *DOMAIN, "/bot/users/complete"] - site_infractions: !JOIN [*SCHEMA, *DOMAIN, "/bot/infractions"] - site_infractions_user: !JOIN [*SCHEMA, *DOMAIN, "/bot/infractions/user/{user_id}"] - site_infractions_type: !JOIN [*SCHEMA, *DOMAIN, "/bot/infractions/type/{infraction_type}"] - site_infractions_by_id: !JOIN [*SCHEMA, *DOMAIN, "/bot/infractions/id/{infraction_id}"] - site_infractions_user_type_current: !JOIN [*SCHEMA, *DOMAIN, "/bot/infractions/user/{user_id}/{infraction_type}/current"] + site: &DOMAIN "pythondiscord.com" + site_api: &API !JOIN ["api.", *DOMAIN] + site_schema: &SCHEMA "https://" + + site_bigbrother_api: !JOIN [*SCHEMA, *API, "/bot/bigbrother"] + site_docs_api: !JOIN [*SCHEMA, *API, "/bot/docs"] + site_facts_api: !JOIN [*SCHEMA, *API, "/bot/snake_facts"] + site_hiphopify_api: !JOIN [*SCHEMA, *API, "/bot/hiphopify"] + site_idioms_api: !JOIN [*SCHEMA, *API, "/bot/snake_idioms"] + site_infractions: !JOIN [*SCHEMA, *API, "/bot/infractions"] + site_infractions_user: !JOIN [*SCHEMA, *API, "/bot/infractions/user/{user_id}"] + site_infractions_type: !JOIN [*SCHEMA, *API, "/bot/infractions/type/{infraction_type}"] + site_infractions_by_id: !JOIN [*SCHEMA, *API, "/bot/infractions/id/{infraction_id}"] + site_infractions_user_type_current: !JOIN [*SCHEMA, *API, "/bot/infractions/user/{user_id}/{infraction_type}/current"] + site_logs_api: !JOIN [*SCHEMA, *API, "/bot/logs"] + site_logs_view: !JOIN [*SCHEMA, *DOMAIN, "/bot/logs"] + site_names_api: !JOIN [*SCHEMA, *API, "/bot/snake_names"] + site_off_topic_names_api: !JOIN [*SCHEMA, *API, "/bot/off-topic-names"] + site_quiz_api: !JOIN [*SCHEMA, *API, "/bot/snake_quiz"] + site_settings_api: !JOIN [*SCHEMA, *API, "/bot/settings"] + site_special_api: !JOIN [*SCHEMA, *API, "/bot/special_snakes"] + site_tags_api: !JOIN [*SCHEMA, *API, "/bot/tags"] + site_user_api: !JOIN [*SCHEMA, *API, "/bot/users"] + site_user_complete_api: !JOIN [*SCHEMA, *API, "/bot/users/complete"] # Env vars deploy: !ENV "DEPLOY_URL" @@ -144,3 +241,60 @@ urls: gitlab_bot_repo: "https://gitlab.com/python-discord/projects/bot" omdb: "http://omdbapi.com" paste_service: "https://paste.pydis.com/{key}" + + +anti_spam: + # Clean messages that violate a rule. + clean_offending: true + ping_everyone: true + + punishment: + role_id: *MUTED_ROLE + remove_after: 600 + + rules: + attachments: + interval: 10 + max: 3 + + burst: + interval: 10 + max: 7 + + burst_shared: + interval: 10 + max: 20 + + chars: + interval: 5 + max: 3_000 + + duplicates: + interval: 10 + max: 3 + + discord_emojis: + interval: 10 + max: 20 + + links: + interval: 10 + max: 20 + + mentions: + interval: 10 + max: 5 + + newlines: + interval: 10 + max: 100 + + role_mentions: + interval: 10 + max: 3 + + +reddit: + request_delay: 60 + subreddits: + - 'r/Python' diff --git a/docker/Dockerfile.base b/docker/base.Dockerfile index 2f6929e0d..de2c68c13 100644 --- a/docker/Dockerfile.base +++ b/docker/base.Dockerfile @@ -22,6 +22,6 @@ ENV PIPENV_IGNORE_VIRTUALENVS=1 ENV PIPENV_NOSPIN=1 ENV PIPENV_HIDE_EMOJIS=1 -RUN pipenv install +RUN pipenv install --deploy --system # usage: FROM pythondiscord/bot-base:latest diff --git a/docker/Dockerfile b/docker/bot.Dockerfile index 350e38ec0..4713e1f0e 100644 --- a/docker/Dockerfile +++ b/docker/bot.Dockerfile @@ -5,13 +5,10 @@ ENV PIPENV_IGNORE_VIRTUALENVS=1 ENV PIPENV_NOSPIN=1 ENV PIPENV_HIDE_EMOJIS=1 -RUN pip install pipenv - COPY . /bot WORKDIR /bot -RUN pipenv clean -RUN pipenv sync +RUN pipenv install --deploy --system ENTRYPOINT ["/sbin/tini", "--"] -CMD ["pipenv", "run", "start"] +CMD ["python", "-m", "bot"] diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 50ec87f59..070d0ec26 100644 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -5,22 +5,22 @@ if [[ $CI_COMMIT_REF_SLUG == 'master' ]]; then echo "Connecting to docker hub" echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - changed_lines=$(git diff HEAD~1 HEAD docker/Dockerfile.base | wc -l) + changed_lines=$(git diff HEAD~1 HEAD docker/base.Dockerfile | wc -l) if [ $changed_lines != '0' ]; then - echo "Dockerfile.base was changed" + echo "base.Dockerfile was changed" echo "Building bot base" - docker build -t pythondiscord/bot-base:latest -f docker/Dockerfile.base . + docker build -t pythondiscord/bot-base:latest -f docker/base.Dockerfile . echo "Pushing image to Docker Hub" docker push pythondiscord/bot-base:latest else - echo "Dockerfile.base was not changed, not building" + echo "base.Dockerfile was not changed, not building" fi echo "Building image" - docker build -t pythondiscord/bot:latest -f docker/Dockerfile . + docker build -t pythondiscord/bot:latest -f docker/bot.Dockerfile . echo "Pushing image" docker push pythondiscord/bot:latest |