Merge branch 'feature/add-anti-spam-cog' into 'master'

Add the `AntiSpam` cog.

See merge request python-discord/projects/bot!35

15 files changed, 545 insertions, 2 deletions

diff --git a/bot/__main__.py b/bot/__main__.py
index 9e5806690..9a2ef9319 100644
--- a/bot/__main__.py
+++ b/bot/__main__.py
@@ -43,6 +43,7 @@ bot.load_extension("bot.cogs.security")
 bot.load_extension("bot.cogs.events")
 
 # Commands, etc
+bot.load_extension("bot.cogs.antispam")
 bot.load_extension("bot.cogs.bigbrother")
 bot.load_extension("bot.cogs.bot")
 bot.load_extension("bot.cogs.clean")
diff --git a/bot/cogs/antispam.py b/bot/cogs/antispam.py
new file mode 100644
index 000000000..04b030889
--- /dev/null
+++ b/bot/cogs/antispam.py
@@ -0,0 +1,161 @@
+import asyncio
+import logging
+import textwrap
+from datetime import datetime, timedelta
+from typing import Dict, List
+
+from dateutil.relativedelta import relativedelta
+from discord import Member, Message, Object, TextChannel
+from discord.ext.commands import Bot
+
+from bot import rules
+from bot.constants import AntiSpam as AntiSpamConfig, Channels, Colours, Guild as GuildConfig, Icons
+from bot.utils.time import humanize as humanize_delta
+
+
+log = logging.getLogger(__name__)
+
+RULE_FUNCTION_MAPPING = {
+    'attachments': rules.apply_attachments,
+    'burst': rules.apply_burst,
+    'burst_shared': rules.apply_burst_shared,
+    'chars': rules.apply_chars,
+    'discord_emojis': rules.apply_discord_emojis,
+    'duplicates': rules.apply_duplicates,
+    'links': rules.apply_links,
+    'mentions': rules.apply_mentions,
+    'newlines': rules.apply_newlines,
+    'role_mentions': rules.apply_role_mentions
+}
+
+
+class AntiSpam:
+    def __init__(self, bot: Bot):
+        self.bot = bot
+        self.muted_role = None
+
+    async def on_ready(self):
+        role_id = AntiSpamConfig.punishment['role_id']
+        self.muted_role = Object(role_id)
+
+    async def on_message(self, message: Message):
+        if message.guild.id != GuildConfig.id or message.author.bot:
+            return
+
+        # Fetch the rule configuration with the highest rule interval.
+        max_interval_config = max(
+            AntiSpamConfig.rules.values(),
+            key=lambda config: config['interval']
+        )
+        max_interval = max_interval_config['interval']
+
+        # Store history messages since `interval` seconds ago in a list to prevent unnecessary API calls.
+        earliest_relevant_at = datetime.utcnow() - timedelta(seconds=max_interval)
+        relevant_messages = [
+            msg async for msg in message.channel.history(after=earliest_relevant_at, reverse=False)
+        ]
+
+        for rule_name in AntiSpamConfig.rules:
+            rule_config = AntiSpamConfig.rules[rule_name]
+            rule_function = RULE_FUNCTION_MAPPING[rule_name]
+
+            # Create a list of messages that were sent in the interval that the rule cares about.
+            latest_interesting_stamp = datetime.utcnow() - timedelta(seconds=rule_config['interval'])
+            messages_for_rule = [
+                msg for msg in relevant_messages if msg.created_at > latest_interesting_stamp
+            ]
+            result = await rule_function(message, messages_for_rule, rule_config)
+
+            # If the rule returns `None`, that means the message didn't violate it.
+            # If it doesn't, it returns a tuple in the form `(str, Iterable[discord.Member])`
+            # which contains the reason for why the message violated the rule and
+            # an iterable of all members that violated the rule.
+            if result is not None:
+                reason, members, relevant_messages = result
+                full_reason = f"`{rule_name}` rule: {reason}"
+                for member in members:
+
+                    # Fire it off as a background task to ensure
+                    # that the sleep doesn't block further tasks
+                    self.bot.loop.create_task(
+                        self.punish(message, member, rule_config, full_reason)
+                    )
+
+                await self.maybe_delete_messages(message.channel, relevant_messages)
+                break
+
+    async def punish(self, msg: Message, member: Member, rule_config: Dict[str, int], reason: str):
+        # Sanity check to ensure we're not lagging behind
+        if self.muted_role not in member.roles:
+            remove_role_after = AntiSpamConfig.punishment['remove_after']
+            duration_delta = relativedelta(seconds=remove_role_after)
+            human_duration = humanize_delta(duration_delta)
+
+            mod_alert_channel = self.bot.get_channel(Channels.mod_alerts)
+            if mod_alert_channel is not None:
+                await mod_alert_channel.send(
+                    f"<:messagefiltered:473092874289020929> Spam detected in {msg.channel.mention}. "
+                    f"See the message and mod log for further details."
+                )
+            else:
+                log.warning(
+                    "Tried logging spam event to the mod-alerts channel, but it could not be found."
+                )
+
+            await member.add_roles(self.muted_role, reason=reason)
+            description = textwrap.dedent(f"""
+            **Channel**: {msg.channel.mention}
+            **User**: {msg.author.mention} (`{msg.author.id}`)
+            **Reason**: {reason}
+            Role will be removed after {human_duration}.
+            """)
+
+            modlog = self.bot.get_cog('ModLog')
+            await modlog.send_log_message(
+                icon_url=Icons.user_mute, colour=Colours.soft_red,
+                title="User muted", text=description
+            )
+
+            await asyncio.sleep(remove_role_after)
+            await member.remove_roles(self.muted_role, reason="AntiSpam mute expired")
+
+            await modlog.send_log_message(
+                icon_url=Icons.user_mute, colour=Colours.soft_green,
+                title="User unmuted",
+                text=f"Was muted by `AntiSpam` cog for {human_duration}."
+            )
+
+    async def maybe_delete_messages(self, channel: TextChannel, messages: List[Message]):
+        # Is deletion of offending messages actually enabled?
+        if AntiSpamConfig.clean_offending:
+
+            # If we have more than one message, we can use bulk delete.
+            if len(messages) > 1:
+                await channel.delete_messages(messages)
+
+            # Otherwise, the bulk delete endpoint will throw up.
+            # Delete the message directly instead.
+            else:
+                await messages[0].delete()
+
+
+def validate_config():
+    for name, config in AntiSpamConfig.rules.items():
+        if name not in RULE_FUNCTION_MAPPING:
+            raise ValueError(
+                f"Unrecognized antispam rule `{name}`. "
+                f"Valid rules are: {', '.join(RULE_FUNCTION_MAPPING)}"
+            )
+
+        for required_key in ('interval', 'max'):
+            if required_key not in config:
+                raise ValueError(
+                    f"`{required_key}` is required but was not "
+                    f"set in rule `{name}`'s configuration."
+                )
+
+
+def setup(bot: Bot):
+    validate_config()
+    bot.add_cog(AntiSpam(bot))
+    log.info("Cog loaded: AntiSpam")
diff --git a/bot/constants.py b/bot/constants.py
index 58bf62b15..ad6236cd9 100644
--- a/bot/constants.py
+++ b/bot/constants.py
@@ -14,7 +14,7 @@ import logging
 import os
 from collections.abc import Mapping
 from pathlib import Path
-from typing import List
+from typing import Dict, List
 
 import yaml
 from yaml.constructor import ConstructorError
@@ -370,6 +370,13 @@ class URLs(metaclass=YAMLGetter):
     paste_service: str
 
 
+class AntiSpam(metaclass=YAMLGetter):
+    section = 'anti_spam'
+
+    punishment: Dict[str, Dict[str, int]]
+    rules: Dict[str, Dict[str, int]]
+
+
 # Debug mode
 DEBUG_MODE = True if 'local' in os.environ.get("SITE_URL", "local") else False
 
diff --git a/bot/rules/__init__.py b/bot/rules/__init__.py
new file mode 100644
index 000000000..a01ceae73
--- /dev/null
+++ b/bot/rules/__init__.py
@@ -0,0 +1,12 @@
+# flake8: noqa
+
+from .attachments import apply as apply_attachments
+from .burst import apply as apply_burst
+from .burst_shared import apply as apply_burst_shared
+from .chars import apply as apply_chars
+from .discord_emojis import apply as apply_discord_emojis
+from .duplicates import apply as apply_duplicates
+from .links import apply as apply_links
+from .mentions import apply as apply_mentions
+from .newlines import apply as apply_newlines
+from .role_mentions import apply as apply_role_mentions
diff --git a/bot/rules/attachments.py b/bot/rules/attachments.py
new file mode 100644
index 000000000..47b927101
--- /dev/null
+++ b/bot/rules/attachments.py
@@ -0,0 +1,30 @@
+"""Detects total attachments exceeding the limit sent by a single user."""
+
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from discord import Member, Message
+
+
+async def apply(
+    last_message: Message,
+    recent_messages: List[Message],
+    config: Dict[str, int]
+) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]:
+
+    relevant_messages = tuple(
+        msg
+        for msg in recent_messages
+        if (
+            msg.author == last_message.author
+            and len(msg.attachments) > 0
+        )
+    )
+    total_recent_attachments = sum(len(msg.attachments) for msg in relevant_messages)
+
+    if total_recent_attachments > config['max']:
+        return (
+            f"sent {total_recent_attachments} attachments in {config['max']}s",
+            (last_message.author,),
+            relevant_messages
+        )
+    return None
diff --git a/bot/rules/burst.py b/bot/rules/burst.py
new file mode 100644
index 000000000..80c79be60
--- /dev/null
+++ b/bot/rules/burst.py
@@ -0,0 +1,27 @@
+"""Detects repeated messages sent by a single user."""
+
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from discord import Member, Message
+
+
+async def apply(
+    last_message: Message,
+    recent_messages: List[Message],
+    config: Dict[str, int]
+) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]:
+
+    relevant_messages = tuple(
+        msg
+        for msg in recent_messages
+        if msg.author == last_message.author
+    )
+    total_relevant = len(relevant_messages)
+
+    if total_relevant > config['max']:
+        return (
+            f"sent {total_relevant} messages in {config['interval']}s",
+            (last_message.author,),
+            relevant_messages
+        )
+    return None
diff --git a/bot/rules/burst_shared.py b/bot/rules/burst_shared.py
new file mode 100644
index 000000000..2cb7b5200
--- /dev/null
+++ b/bot/rules/burst_shared.py
@@ -0,0 +1,22 @@
+"""Detects repeated messages sent by multiple users."""
+
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from discord import Member, Message
+
+
+async def apply(
+    last_message: Message,
+    recent_messages: List[Message],
+    config: Dict[str, int]
+) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]:
+
+    total_recent = len(recent_messages)
+
+    if total_recent > config['max']:
+        return (
+            f"sent {total_recent} messages in {config['interval']}s",
+            set(msg.author for msg in recent_messages),
+            recent_messages
+        )
+    return None
diff --git a/bot/rules/chars.py b/bot/rules/chars.py
new file mode 100644
index 000000000..d05e3cd83
--- /dev/null
+++ b/bot/rules/chars.py
@@ -0,0 +1,28 @@
+"""Detects total message char count exceeding the limit sent by a single user."""
+
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from discord import Member, Message
+
+
+async def apply(
+    last_message: Message,
+    recent_messages: List[Message],
+    config: Dict[str, int]
+) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]:
+
+    relevant_messages = tuple(
+        msg
+        for msg in recent_messages
+        if msg.author == last_message.author
+    )
+
+    total_recent_chars = sum(len(msg.content) for msg in relevant_messages)
+
+    if total_recent_chars > config['max']:
+        return (
+            f"sent {total_recent_chars} characters in {config['interval']}s",
+            (last_message.author,),
+            relevant_messages
+        )
+    return None
diff --git a/bot/rules/discord_emojis.py b/bot/rules/discord_emojis.py
new file mode 100644
index 000000000..e4f957ddb
--- /dev/null
+++ b/bot/rules/discord_emojis.py
@@ -0,0 +1,35 @@
+"""Detects total Discord emojis (excluding Unicode emojis) exceeding the limit sent by a single user."""
+
+import re
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from discord import Member, Message
+
+
+DISCORD_EMOJI_RE = re.compile(r"<:\w+:\d+>")
+
+
+async def apply(
+    last_message: Message,
+    recent_messages: List[Message],
+    config: Dict[str, int]
+) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]:
+
+    relevant_messages = tuple(
+        msg
+        for msg in recent_messages
+        if msg.author == last_message.author
+    )
+
+    total_emojis = sum(
+        len(DISCORD_EMOJI_RE.findall(msg.content))
+        for msg in relevant_messages
+    )
+
+    if total_emojis > config['max']:
+        return (
+            f"sent {total_emojis} emojis in {config['interval']}s",
+            (last_message.author,),
+            relevant_messages
+        )
+    return None
diff --git a/bot/rules/duplicates.py b/bot/rules/duplicates.py
new file mode 100644
index 000000000..763fc9983
--- /dev/null
+++ b/bot/rules/duplicates.py
@@ -0,0 +1,31 @@
+"""Detects duplicated messages sent by a single user."""
+
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from discord import Member, Message
+
+
+async def apply(
+    last_message: Message,
+    recent_messages: List[Message],
+    config: Dict[str, int]
+) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]:
+
+    relevant_messages = tuple(
+        msg
+        for msg in recent_messages
+        if (
+            msg.author == last_message.author
+            and msg.content == last_message.content
+        )
+    )
+
+    total_duplicated = len(relevant_messages)
+
+    if total_duplicated > config['max']:
+        return (
+            f"sent {total_duplicated} duplicated messages in {config['interval']}s",
+            (last_message.author,),
+            relevant_messages
+        )
+    return None
diff --git a/bot/rules/links.py b/bot/rules/links.py
new file mode 100644
index 000000000..dfeb38c61
--- /dev/null
+++ b/bot/rules/links.py
@@ -0,0 +1,31 @@
+"""Detects total links exceeding the limit sent by a single user."""
+
+import re
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from discord import Member, Message
+
+
+LINK_RE = re.compile(r"(https?://[^\s]+)")
+
+
+async def apply(
+    last_message: Message,
+    recent_messages: List[Message],
+    config: Dict[str, int]
+) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]:
+
+    relevant_messages = tuple(
+        msg
+        for msg in recent_messages
+        if msg.author == last_message.author
+    )
+    total_links = sum(len(LINK_RE.findall(msg.content)) for msg in relevant_messages)
+
+    if total_links > config['max']:
+        return (
+            f"sent {total_links} links in {config['interval']}s",
+            (last_message.author,),
+            relevant_messages
+        )
+    return None
diff --git a/bot/rules/mentions.py b/bot/rules/mentions.py
new file mode 100644
index 000000000..45c47b6ba
--- /dev/null
+++ b/bot/rules/mentions.py
@@ -0,0 +1,28 @@
+"""Detects total mentions exceeding the limit sent by a single user."""
+
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from discord import Member, Message
+
+
+async def apply(
+    last_message: Message,
+    recent_messages: List[Message],
+    config: Dict[str, int]
+) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]:
+
+    relevant_messages = tuple(
+        msg
+        for msg in recent_messages
+        if msg.author == last_message.author
+    )
+
+    total_recent_mentions = sum(len(msg.mentions) for msg in relevant_messages)
+
+    if total_recent_mentions > config['max']:
+        return (
+            f"sent {total_recent_mentions} mentions in {config['interval']}s",
+            (last_message.author,),
+            relevant_messages
+        )
+    return None
diff --git a/bot/rules/newlines.py b/bot/rules/newlines.py
new file mode 100644
index 000000000..a6a1a52d0
--- /dev/null
+++ b/bot/rules/newlines.py
@@ -0,0 +1,28 @@
+"""Detects total newlines exceeding the set limit sent by a single user."""
+
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from discord import Member, Message
+
+
+async def apply(
+    last_message: Message,
+    recent_messages: List[Message],
+    config: Dict[str, int]
+) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]:
+
+    relevant_messages = tuple(
+        msg
+        for msg in recent_messages
+        if msg.author == last_message.author
+    )
+
+    total_recent_newlines = sum(msg.content.count('\n') for msg in relevant_messages)
+
+    if total_recent_newlines > config['max']:
+        return (
+            f"sent {total_recent_newlines} newlines in {config['interval']}s",
+            (last_message.author,),
+            relevant_messages
+        )
+    return None
diff --git a/bot/rules/role_mentions.py b/bot/rules/role_mentions.py
new file mode 100644
index 000000000..2177a73b5
--- /dev/null
+++ b/bot/rules/role_mentions.py
@@ -0,0 +1,28 @@
+"""Detects total role mentions exceeding the limit sent by a single user."""
+
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from discord import Member, Message
+
+
+async def apply(
+    last_message: Message,
+    recent_messages: List[Message],
+    config: Dict[str, int]
+) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]:
+
+    relevant_messages = tuple(
+        msg
+        for msg in recent_messages
+        if msg.author == last_message.author
+    )
+
+    total_recent_mentions = sum(len(msg.role_mentions) for msg in relevant_messages)
+
+    if total_recent_mentions > config['max']:
+        return (
+            f"sent {total_recent_mentions} role mentions in {config['interval']}s",
+            (last_message.author,),
+            relevant_messages
+        )
+    return None
diff --git a/config-default.yml b/config-default.yml
index 8ef74f6c3..9a4f3d4fc 100644
--- a/config-default.yml
+++ b/config-default.yml
@@ -57,6 +57,9 @@ style:
         user_unban:  "https://cdn.discordapp.com/emojis/469952898692808704.png"
         user_update: "https://cdn.discordapp.com/emojis/469952898684551168.png"
 
+        user_mute:   "https://cdn.discordapp.com/emojis/472472640100106250.png"
+        user_unmute: "https://cdn.discordapp.com/emojis/472472640100106250.png"
+
 
 guild:
     id: 267624335836053506
@@ -78,6 +81,7 @@ guild:
         help_5:                       454941769734422538
         helpers:                      385474242440986624
         message_log:     &MESSAGE_LOG 467752170159079424
+        mod_alerts:                   473092793431097354
         modlog:          &MODLOG      282638479504965634
         off_topic_0:                  291284109232308226
         off_topic_1:                  463035241142026251
@@ -99,7 +103,7 @@ guild:
         owner: 267627879762755584
         verified: 352427296948486144
         helpers: 267630620367257601
-        muted: 277914926603829249
+        muted: &MUTED_ROLE 277914926603829249
 
 
 keys:
@@ -158,3 +162,73 @@ urls:
     gitlab_bot_repo: "https://gitlab.com/python-discord/projects/bot"
     omdb:            "http://omdbapi.com"
     paste_service:   "https://paste.pydis.com/{key}"
+
+
+anti_spam:
+    # Clean messages that violate a rule.
+    clean_offending: true
+
+    punishment:
+        role_id: *MUTED_ROLE
+        remove_after: 600
+
+    rules:
+        attachments:
+            interval: 10
+            max: 3
+
+        burst:
+            interval: 10
+            max: 7
+
+        burst_shared:
+            interval: 10
+            max: 20
+
+        chars:
+            interval: 5
+            max: 3_000
+
+        duplicates:
+            interval: 10
+            max: 3
+
+        discord_emojis:
+            interval: 10
+            max: 6
+
+        links:
+            interval: 10
+            max: 4
+
+        mentions:
+            interval: 10
+            max: 5
+
+        newlines:
+            interval: 10
+            max: 100
+
+        role_mentions:
+            interval: 10
+            max: 3
+
+        discord_emojis:
+            interval: 10
+            max: 6
+
+        links:
+            interval: 10
+            max: 4
+
+        mentions:
+            interval: 10
+            max: 5
+
+        newlines:
+            interval: 10
+            max: 20
+
+        role_mentions:
+            interval: 10
+            max: 3