diff options
| author | 2022-10-30 22:51:15 +0200 | |
|---|---|---|
| committer | 2022-10-31 22:01:59 +0200 | |
| commit | 3fc53356b5746533d76d82846b8fd9b9c649eac5 (patch) | |
| tree | f1933bac691dc0d716ad50563c2ab7f42cd66a70 | |
| parent | Handle context message possibly being None (diff) | |
Add antispam filter list and duplicates filter
Adds the antispam filterlist, which dispatches a set of messages sent in the last X seconds to its filters.
Unlike other filter lists, it doesn't just return the actions to be taken, but rather splits them in two: any actions unrelated to alerting mods are returned as usual, while actions related to alerting are used in a delayed coroutine, from which the alert is sent.
- Fixes `infraction_channel` not using the context channel when it's None.
- Moves the `upload_log` function outside the ModLog cog, as it only needed it for the bot instance.
- Any auto-deleted message with attachments will now have its attachments logged.
| -rw-r--r-- | bot/exts/filtering/_filter_context.py | 14 | ||||
| -rw-r--r-- | bot/exts/filtering/_filter_lists/antispam.py | 179 | ||||
| -rw-r--r-- | bot/exts/filtering/_filter_lists/filter_list.py | 47 | ||||
| -rw-r--r-- | bot/exts/filtering/_filter_lists/unique.py | 41 | ||||
| -rw-r--r-- | bot/exts/filtering/_filters/antispam/__init__.py | 9 | ||||
| -rw-r--r-- | bot/exts/filtering/_filters/antispam/duplicates.py | 44 | ||||
| -rw-r--r-- | bot/exts/filtering/_settings_types/actions/delete_messages.py | 49 | ||||
| -rw-r--r-- | bot/exts/filtering/_settings_types/actions/infraction_and_notification.py | 25 | ||||
| -rw-r--r-- | bot/exts/filtering/_ui/ui.py | 61 | ||||
| -rw-r--r-- | bot/exts/filtering/_utils.py | 2 | ||||
| -rw-r--r-- | bot/exts/filtering/filtering.py | 46 | ||||
| -rw-r--r-- | bot/exts/moderation/clean.py | 3 | ||||
| -rw-r--r-- | bot/exts/moderation/modlog.py | 51 | ||||
| -rw-r--r-- | bot/utils/messages.py | 49 |
14 files changed, 479 insertions, 141 deletions
diff --git a/bot/exts/filtering/_filter_context.py b/bot/exts/filtering/_filter_context.py index 22950d5df..3227b333a 100644 --- a/bot/exts/filtering/_filter_context.py +++ b/bot/exts/filtering/_filter_context.py @@ -1,11 +1,15 @@ from __future__ import annotations -from collections.abc import Callable, Coroutine +import typing +from collections.abc import Callable, Coroutine, Iterable from dataclasses import dataclass, field, replace from enum import Enum, auto from discord import DMChannel, Member, Message, TextChannel, Thread, User +if typing.TYPE_CHECKING: + from bot.exts.filtering._filters.filter import Filter + class Event(Enum): """Types of events that can trigger filtering. Note this does not have to align with gateway event types.""" @@ -22,7 +26,7 @@ class FilterContext: event: Event # The type of event author: User | Member | None # Who triggered the event channel: TextChannel | Thread | DMChannel # The channel involved - content: str | set # What actually needs filtering + content: str | Iterable # What actually needs filtering message: Message | None # The message involved embeds: list = field(default_factory=list) # Any embeds involved # Output context @@ -32,10 +36,14 @@ class FilterContext: alert_content: str = field(default_factory=str) # The content of the alert alert_embeds: list = field(default_factory=list) # Any embeds to add to the alert action_descriptions: list = field(default_factory=list) # What actions were taken - matches: list = field(default_factory=list) # What exactly was found + matches: list[str] = field(default_factory=list) # What exactly was found notification_domain: str = field(default_factory=str) # A domain to send the user for context + filter_info: dict['Filter', str] = field(default_factory=dict) # Additional info from a filter. # Additional actions to perform additional_actions: list[Callable[[FilterContext], Coroutine]] = field(default_factory=list) + related_messages: set[Message] = field(default_factory=set) + related_channels: set[TextChannel | Thread | DMChannel] = field(default_factory=set) + attachments: dict[int, list[str]] = field(default_factory=dict) # Message ID to attachment URLs. def replace(self, **changes) -> FilterContext: """Return a new context object assigning new values to the specified fields.""" diff --git a/bot/exts/filtering/_filter_lists/antispam.py b/bot/exts/filtering/_filter_lists/antispam.py new file mode 100644 index 000000000..2dab54ce6 --- /dev/null +++ b/bot/exts/filtering/_filter_lists/antispam.py @@ -0,0 +1,179 @@ +import asyncio +import typing +from collections.abc import Callable, Coroutine +from dataclasses import dataclass, field +from datetime import timedelta +from functools import reduce +from itertools import takewhile +from operator import add, or_ + +import arrow +from botcore.utils import scheduling +from botcore.utils.logging import get_logger +from discord import HTTPException, Member + +import bot +from bot.constants import Webhooks +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._filter_lists.filter_list import ListType, SubscribingAtomicList, UniquesListBase +from bot.exts.filtering._filters.antispam import antispam_filter_types +from bot.exts.filtering._filters.filter import UniqueFilter +from bot.exts.filtering._settings import ActionSettings +from bot.exts.filtering._settings_types.actions.infraction_and_notification import Infraction +from bot.exts.filtering._ui.ui import build_mod_alert + +if typing.TYPE_CHECKING: + from bot.exts.filtering.filtering import Filtering + +log = get_logger(__name__) + +ALERT_DELAY = 6 + + +class AntispamList(UniquesListBase): + """ + A list of anti-spam rules. + + Messages from the last X seconds is passed to each rule, which decide whether it triggers across those messages. + """ + + name = "antispam" + + def __init__(self, filtering_cog: 'Filtering'): + super().__init__(filtering_cog) + self.message_deletion_queue: dict[Member, DeletionContext] = dict() + + def get_filter_type(self, content: str) -> type[UniqueFilter] | None: + """Get a subclass of filter matching the filter list and the filter's content.""" + try: + return antispam_filter_types[content] + except KeyError: + if content not in self._already_warned: + log.warn(f"An antispam filter named {content} was supplied, but no matching implementation found.") + self._already_warned.add(content) + return None + + async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: + """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" + if not ctx.message: + return None, [] + + sublist: SubscribingAtomicList = self[ListType.DENY] + potential_filters = [sublist.filters[id_] for id_ in sublist.subscriptions[ctx.event]] + max_interval = max(filter_.extra_fields.interval for filter_ in potential_filters) + + earliest_relevant_at = arrow.utcnow() - timedelta(seconds=max_interval) + relevant_messages = list( + takewhile(lambda msg: msg.created_at > earliest_relevant_at, self.filtering_cog.message_cache) + ) + new_ctx = ctx.replace(content=relevant_messages) + triggers = sublist.filter_list_result(new_ctx) + if not triggers: + return None, [] + + if ctx.author not in self.message_deletion_queue: + self.message_deletion_queue[ctx.author] = DeletionContext() + ctx.additional_actions.append(self._create_deletion_context_handler(ctx.author)) + ctx.related_channels |= {msg.channel for msg in ctx.related_messages} + else: # The additional messages found are already part of the deletion context + ctx.related_messages = set() + current_infraction = self.message_deletion_queue[ctx.author].current_infraction + self.message_deletion_queue[ctx.author].add(ctx, triggers) + + current_actions = sublist.merge_actions(triggers) if triggers else None + # Don't alert yet. + current_actions.pop("ping", None) + current_actions.pop("send_alert", None) + new_infraction = current_actions["infraction_and_notification"].copy() + # Smaller infraction value = higher in hierarchy. + if not current_infraction or new_infraction.infraction_type.value < current_infraction.value: + # Pick the first triggered filter for the reason, there's no good way to decide between them. + new_infraction.infraction_reason = f"{triggers[0].name} spam - {ctx.filter_info[triggers[0]]}" + current_actions["infraction_and_notification"] = new_infraction + self.message_deletion_queue[ctx.author].current_infraction = new_infraction.infraction_type + else: + current_actions.pop("infraction_and_notification", None) + + # Provide some message in case another filter list wants there to be an alert. + return current_actions, ["Handling spam event..."] + + def _create_deletion_context_handler(self, context_id: Member) -> Callable[[FilterContext], Coroutine]: + async def schedule_processing(ctx: FilterContext) -> None: + """ + Schedule a coroutine to process the deletion context. + + It cannot be awaited directly, as it waits ALERT_DELAY seconds, and actioning a filtering context depends on + all actions finishing. + + This is async and takes a context to adhere to the type of ctx.additional_actions. + """ + async def process_deletion_context() -> None: + """Processes the Deletion Context queue.""" + log.trace("Sleeping before processing message deletion queue.") + await asyncio.sleep(ALERT_DELAY) + + if context_id not in self.message_deletion_queue: + log.error(f"Started processing deletion queue for context `{context_id}`, but it was not found!") + return + + deletion_context = self.message_deletion_queue.pop(context_id) + await deletion_context.send_alert(self) + + scheduling.create_task(process_deletion_context()) + + return schedule_processing + + +@dataclass +class DeletionContext: + """Represents a Deletion Context for a single spam event.""" + + contexts: list[FilterContext] = field(default_factory=list) + rules: set[UniqueFilter] = field(default_factory=set) + current_infraction: Infraction | None = None + + def add(self, ctx: FilterContext, rules: list[UniqueFilter]) -> None: + """Adds new rule violation events to the deletion context.""" + self.contexts.append(ctx) + self.rules.update(rules) + + async def send_alert(self, antispam_list: AntispamList) -> None: + """Post the mod alert.""" + if not self.contexts or not self.rules: + return + try: + webhook = await bot.instance.fetch_webhook(Webhooks.filters) + except HTTPException: + return + + ctx, *other_contexts = self.contexts + new_ctx = FilterContext(ctx.event, ctx.author, ctx.channel, ctx.content, ctx.message) + new_ctx.action_descriptions = reduce( + add, (other_ctx.action_descriptions for other_ctx in other_contexts), ctx.action_descriptions + ) + # It shouldn't ever come to this, but just in case. + if descriptions_num := len(new_ctx.action_descriptions) > 20: + new_ctx.action_descriptions = new_ctx.action_descriptions[:20] + new_ctx.action_descriptions[-1] += f" (+{descriptions_num - 20} other actions)" + new_ctx.related_messages = reduce( + or_, (other_ctx.related_messages for other_ctx in other_contexts), ctx.related_messages + ) + new_ctx.related_channels = reduce( + or_, (other_ctx.related_channels for other_ctx in other_contexts), ctx.related_channels + ) + new_ctx.attachments = reduce(or_, (other_ctx.attachments for other_ctx in other_contexts), ctx.attachments) + + rules = list(self.rules) + actions = antispam_list[ListType.DENY].merge_actions(rules) + for action in list(actions): + if action not in ("ping", "send_alert"): + actions.pop(action, None) + await actions.action(new_ctx) + + messages = antispam_list[ListType.DENY].format_messages(rules) + embed = await build_mod_alert(new_ctx, {antispam_list: messages}) + if other_contexts: + embed.set_footer( + text="The list of actions taken includes actions from additional contexts after deletion began." + ) + await webhook.send(username="Anti-Spam", content=ctx.alert_content, embeds=[embed]) diff --git a/bot/exts/filtering/_filter_lists/filter_list.py b/bot/exts/filtering/_filter_lists/filter_list.py index 0bb0dc7f8..f9db54a21 100644 --- a/bot/exts/filtering/_filter_lists/filter_list.py +++ b/bot/exts/filtering/_filter_lists/filter_list.py @@ -17,6 +17,9 @@ from bot.exts.filtering._settings import ActionSettings, Defaults, create_settin from bot.exts.filtering._utils import FieldRequiring, past_tense from bot.log import get_logger +if typing.TYPE_CHECKING: + from bot.exts.filtering.filtering import Filtering + log = get_logger(__name__) @@ -223,3 +226,47 @@ class SubscribingAtomicList(AtomicList): """Sift through the list of filters, and return only the ones which apply to the given context.""" event_filters = [self.filters[id_] for id_ in self.subscriptions[ctx.event]] return self._create_filter_list_result(ctx, self.defaults, event_filters) + + +class UniquesListBase(FilterList[UniqueFilter]): + """ + A list of unique filters. + + Unique filters are ones that should only be run once in a given context. + Each unique filter subscribes to a subset of events to respond to. + """ + + _already_warned = set() + + def __init__(self, filtering_cog: 'Filtering'): + super().__init__() + self.filtering_cog = filtering_cog + self.loaded_types: dict[str, type[UniqueFilter]] = {} + + def add_list(self, list_data: dict) -> SubscribingAtomicList: + """Add a new type of list (such as a whitelist or a blacklist) this filter list.""" + actions, validations = create_settings(list_data["settings"], keep_empty=True) + list_type = ListType(list_data["list_type"]) + defaults = Defaults(actions, validations) + new_list = SubscribingAtomicList(list_data["id"], self.name, list_type, defaults, {}) + self[list_type] = new_list + + filters = {} + events = set() + for filter_data in list_data["filters"]: + new_filter = self._create_filter(filter_data, defaults) + if new_filter: + new_list.subscribe(new_filter, *new_filter.events) + filters[filter_data["id"]] = new_filter + self.loaded_types[new_filter.name] = type(new_filter) + events.update(new_filter.events) + + new_list.filters.update(filters) + if hasattr(self.filtering_cog, "subscribe"): # Subscribe the filter list to any new events found. + self.filtering_cog.subscribe(self, *events) + return new_list + + @property + def filter_types(self) -> set[type[UniqueFilter]]: + """Return the types of filters used by this list.""" + return set(self.loaded_types.values()) diff --git a/bot/exts/filtering/_filter_lists/unique.py b/bot/exts/filtering/_filter_lists/unique.py index 63caa7d36..5204065f9 100644 --- a/bot/exts/filtering/_filter_lists/unique.py +++ b/bot/exts/filtering/_filter_lists/unique.py @@ -1,16 +1,15 @@ from botcore.utils.logging import get_logger -from discord.ext.commands import Cog from bot.exts.filtering._filter_context import FilterContext -from bot.exts.filtering._filter_lists.filter_list import FilterList, ListType, SubscribingAtomicList +from bot.exts.filtering._filter_lists.filter_list import ListType, UniquesListBase from bot.exts.filtering._filters.filter import UniqueFilter from bot.exts.filtering._filters.unique import unique_filter_types -from bot.exts.filtering._settings import ActionSettings, Defaults, create_settings +from bot.exts.filtering._settings import ActionSettings log = get_logger(__name__) -class UniquesList(FilterList[UniqueFilter]): +class UniquesList(UniquesListBase): """ A list of unique filters. @@ -19,35 +18,6 @@ class UniquesList(FilterList[UniqueFilter]): """ name = "unique" - _already_warned = set() - - def __init__(self, filtering_cog: Cog): - super().__init__() - self.filtering_cog = filtering_cog # This is typed as a Cog to avoid a circular import. - self.loaded_types: dict[str, type[UniqueFilter]] = {} - - def add_list(self, list_data: dict) -> SubscribingAtomicList: - """Add a new type of list (such as a whitelist or a blacklist) this filter list.""" - actions, validations = create_settings(list_data["settings"], keep_empty=True) - list_type = ListType(list_data["list_type"]) - defaults = Defaults(actions, validations) - new_list = SubscribingAtomicList(list_data["id"], self.name, list_type, defaults, {}) - self[list_type] = new_list - - filters = {} - events = set() - for filter_data in list_data["filters"]: - new_filter = self._create_filter(filter_data, defaults) - if new_filter: - new_list.subscribe(new_filter, *new_filter.events) - filters[filter_data["id"]] = new_filter - self.loaded_types[new_filter.name] = type(new_filter) - events.update(new_filter.events) - - new_list.filters.update(filters) - if hasattr(self.filtering_cog, "subscribe"): # Subscribe the filter list to any new events found. - self.filtering_cog.subscribe(self, *events) - return new_list def get_filter_type(self, content: str) -> type[UniqueFilter] | None: """Get a subclass of filter matching the filter list and the filter's content.""" @@ -59,11 +29,6 @@ class UniquesList(FilterList[UniqueFilter]): self._already_warned.add(content) return None - @property - def filter_types(self) -> set[type[UniqueFilter]]: - """Return the types of filters used by this list.""" - return set(self.loaded_types.values()) - async def actions_for(self, ctx: FilterContext) -> tuple[ActionSettings | None, list[str]]: """Dispatch the given event to the list's filters, and return actions to take and messages to relay to mods.""" triggers = self[ListType.DENY].filter_list_result(ctx) diff --git a/bot/exts/filtering/_filters/antispam/__init__.py b/bot/exts/filtering/_filters/antispam/__init__.py new file mode 100644 index 000000000..637bcd410 --- /dev/null +++ b/bot/exts/filtering/_filters/antispam/__init__.py @@ -0,0 +1,9 @@ +from os.path import dirname + +from bot.exts.filtering._filters.filter import UniqueFilter +from bot.exts.filtering._utils import subclasses_in_package + +antispam_filter_types = subclasses_in_package(dirname(__file__), f"{__name__}.", UniqueFilter) +antispam_filter_types = {filter_.name: filter_ for filter_ in antispam_filter_types} + +__all__ = [antispam_filter_types] diff --git a/bot/exts/filtering/_filters/antispam/duplicates.py b/bot/exts/filtering/_filters/antispam/duplicates.py new file mode 100644 index 000000000..5df2bb5c0 --- /dev/null +++ b/bot/exts/filtering/_filters/antispam/duplicates.py @@ -0,0 +1,44 @@ +from datetime import timedelta +from itertools import takewhile +from typing import ClassVar + +import arrow +from pydantic import BaseModel + +from bot.exts.filtering._filter_context import Event, FilterContext +from bot.exts.filtering._filters.filter import UniqueFilter + + +class ExtraDuplicatesSettings(BaseModel): + """Extra settings for when to trigger the antispam rule.""" + + interval_description: ClassVar[str] = ( + "Look for rule violations in messages from the last `interval` number of seconds." + ) + threshold_description: ClassVar[str] = "Number of duplicate messages required to trigger the filter." + + interval: int = 10 + threshold: int = 3 + + +class DuplicatesFilter(UniqueFilter): + """Detects duplicated messages sent by a single user.""" + + name = "duplicates" + events = (Event.MESSAGE,) + extra_fields_type = ExtraDuplicatesSettings + + def triggered_on(self, ctx: FilterContext) -> bool: + """Search for the filter's content within a given context.""" + earliest_relevant_at = arrow.utcnow() - timedelta(seconds=self.extra_fields.interval) + relevant_messages = list(takewhile(lambda msg: msg.created_at > earliest_relevant_at, ctx.content)) + + detected_messages = { + msg for msg in relevant_messages + if msg.author == ctx.author and msg.content == ctx.message.content and msg.content + } + if len(detected_messages) > self.extra_fields.threshold: + ctx.related_messages |= detected_messages + ctx.filter_info[self] = f"sent {len(detected_messages)} duplicate messages" + return True + return False diff --git a/bot/exts/filtering/_settings_types/actions/delete_messages.py b/bot/exts/filtering/_settings_types/actions/delete_messages.py index 2f851ef04..19c0beb95 100644 --- a/bot/exts/filtering/_settings_types/actions/delete_messages.py +++ b/bot/exts/filtering/_settings_types/actions/delete_messages.py @@ -1,9 +1,24 @@ +from collections import defaultdict from typing import ClassVar -from discord.errors import NotFound +from botcore.utils import scheduling +from discord import Message +from discord.errors import HTTPException +from bot.constants import Channels from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry +from bot.utils.messages import send_attachments + + +async def upload_messages_attachments(ctx: FilterContext, messages: list[Message]) -> None: + """Re-upload the messages' attachments for future logging.""" + if not messages: + return + destination = messages[0].guild.get_channel(Channels.attachment_log) + for message in messages: + if message.attachments and message.id not in ctx.attachments: + ctx.attachments[message.id] = await send_attachments(message, destination, link_large=False) class DeleteMessages(ActionEntry): @@ -24,12 +39,34 @@ class DeleteMessages(ActionEntry): if not ctx.message.guild: return - try: - await ctx.message.delete() - except NotFound: - ctx.action_descriptions.append("failed to delete") + channel_messages = defaultdict(set) # Duplicates will cause batch deletion to fail. + for message in {ctx.message} | ctx.related_messages: + channel_messages[message.channel].add(message) + + success = fail = 0 + deleted = list() + for channel, messages in channel_messages.items(): + try: + await channel.delete_messages(messages) + except HTTPException: + fail += len(messages) + else: + success += len(messages) + deleted.extend(messages) + scheduling.create_task(upload_messages_attachments(ctx, deleted)) + + if not fail: + if success == 1: + ctx.action_descriptions.append("deleted") + else: + ctx.action_descriptions.append("deleted all") + elif not success: + if fail == 1: + ctx.action_descriptions.append("failed to delete") + else: + ctx.action_descriptions.append("all failed to delete") else: - ctx.action_descriptions.append("deleted") + ctx.action_descriptions.append(f"{success} deleted, {fail} failed to delete") def __or__(self, other: ActionEntry): """Combines two actions of the same type. Each type of action is executed once per filter.""" diff --git a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py index 922101d6d..fb679855a 100644 --- a/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py +++ b/bot/exts/filtering/_settings_types/actions/infraction_and_notification.py @@ -5,6 +5,7 @@ from typing import ClassVar import arrow import discord.abc +from botcore.utils.logging import get_logger from discord import Colour, Embed, Member, User from discord.errors import Forbidden from pydantic import validator @@ -14,6 +15,8 @@ from bot.constants import Channels, Guild from bot.exts.filtering._filter_context import FilterContext from bot.exts.filtering._settings_types.settings_entry import ActionEntry +log = get_logger(__name__) + @dataclass class FakeContext: @@ -65,17 +68,12 @@ class Infraction(Enum): async def invoke( self, user: Member | User, - channel: int | None, + channel: discord.abc.Messageable, + alerts_channel: discord.TextChannel, duration: float | None, reason: str | None ) -> None: """Invokes the command matching the infraction name.""" - alerts_channel = bot_module.instance.get_channel(Channels.mod_alerts) - if not channel: - channel = alerts_channel - else: - channel = bot_module.instance.get_channel(channel) - command_name = self.name.lower() command = bot_module.instance.get_command(command_name) if not command: @@ -108,7 +106,8 @@ class InfractionAndNotification(ActionEntry): "infraction_reason": "The reason delivered with the infraction.", "infraction_channel": ( "The channel ID in which to invoke the infraction (and send the confirmation message). " - "If blank, the infraction will be sent in the context channel." + "If blank, the infraction will be sent in the context channel. If the ID fails to resolve, it will default " + "to the mod-alerts channel." ), "dm_content": "The contents of a message to be DMed to the offending user.", "dm_embed": "The contents of the embed to be DMed to the offending user." @@ -152,8 +151,16 @@ class InfractionAndNotification(ActionEntry): ctx.action_descriptions.append("failed to notify") if self.infraction_type is not None: + alerts_channel = bot_module.instance.get_channel(Channels.mod_alerts) + if self.infraction_channel: + channel = bot_module.instance.get_channel(self.infraction_channel) + if not channel: + log.info(f"Could not find a channel with ID {self.infraction_channel}, infracting in mod-alerts.") + channel = alerts_channel + else: + channel = ctx.channel await self.infraction_type.invoke( - ctx.author, self.infraction_channel, self.infraction_duration, self.infraction_reason + ctx.author, channel, alerts_channel, self.infraction_duration, self.infraction_reason ) ctx.action_descriptions.append(self.infraction_type.name.lower()) diff --git a/bot/exts/filtering/_ui/ui.py b/bot/exts/filtering/_ui/ui.py index 6a261bc46..9fc15410e 100644 --- a/bot/exts/filtering/_ui/ui.py +++ b/bot/exts/filtering/_ui/ui.py @@ -13,6 +13,13 @@ from botcore.utils.logging import get_logger from discord import Embed, Interaction from discord.ext.commands import Context from discord.ui.select import MISSING as SELECT_MISSING, SelectOption +from discord.utils import escape_markdown + +import bot +from bot.constants import Colours +from bot.exts.filtering._filter_context import FilterContext +from bot.exts.filtering._filter_lists import FilterList +from bot.utils.messages import format_channel, format_user, upload_log log = get_logger(__name__) @@ -31,7 +38,7 @@ DELETION_TIMEOUT = 60 MAX_MODAL_TITLE_LENGTH = 45 # Max number of items in a select MAX_SELECT_ITEMS = 25 -MAX_EMBED_DESCRIPTION = 4000 +MAX_EMBED_DESCRIPTION = 4080 SETTINGS_DELIMITER = re.compile(r"\s+(?=\S+=\S+)") SINGLE_SETTING_PATTERN = re.compile(r"[\w/]+=.+") @@ -42,6 +49,58 @@ MISSING = object() T = TypeVar('T') +async def _build_alert_message_content(ctx: FilterContext, current_message_length: int) -> str: + """Build the content section of the alert.""" + # For multiple messages and those with attachments or excessive newlines, use the logs API + if any(( + ctx.related_messages, + len(ctx.attachments) > 0, + ctx.content.count('\n') > 15 + )): + url = await upload_log(ctx.related_messages, bot.instance.user.id, ctx.attachments) + alert_content = f"A complete log of the offending messages can be found [here]({url})" + else: + alert_content = escape_markdown(ctx.content) + remaining_chars = MAX_EMBED_DESCRIPTION - current_message_length + + if len(alert_content) > remaining_chars: + url = await upload_log([ctx.message], bot.instance.user.id, ctx.attachments) + log_site_msg = f"The full message can be found [here]({url})" + # 7 because that's the length of "[...]\n\n" + alert_content = alert_content[:remaining_chars - (7 + len(log_site_msg))] + "[...]\n\n" + log_site_msg + + return alert_content + + +async def build_mod_alert(ctx: FilterContext, triggered_filters: dict[FilterList, list[str]]) -> Embed: + """Build an alert message from the filter context.""" + embed = Embed(color=Colours.soft_orange) + embed.set_thumbnail(url=ctx.author.display_avatar.url) + triggered_by = f"**Triggered by:** {format_user(ctx.author)}" + if ctx.channel.guild: + triggered_in = f"**Triggered in:** {format_channel(ctx.channel)}\n" + else: + triggered_in = "**Triggered in:** :warning:**DM**:warning:\n" + if len(ctx.related_channels) > 1: + triggered_in += f"**Channels:** {', '.join(channel.mention for channel in ctx.related_channels)}\n" + + filters = [] + for filter_list, list_message in triggered_filters.items(): + if list_message: + filters.append(f"**{filter_list.name.title()} Filters:** {', '.join(list_message)}") + filters = "\n".join(filters) + + matches = "**Matches:** " + ", ".join(repr(match) for match in ctx.matches) if ctx.matches else "" + actions = "\n**Actions Taken:** " + (", ".join(ctx.action_descriptions) if ctx.action_descriptions else "-") + + mod_alert_message = "\n".join(part for part in (triggered_by, triggered_in, filters, matches, actions) if part) + mod_alert_message += f"\n**[Original Content]({ctx.message.jump_url})**:\n" + mod_alert_message += await _build_alert_message_content(ctx, len(mod_alert_message)) + + embed.description = mod_alert_message + return embed + + def populate_embed_from_dict(embed: Embed, data: dict) -> None: """Populate a Discord embed by populating fields from the given dict.""" for setting, value in data.items(): diff --git a/bot/exts/filtering/_utils.py b/bot/exts/filtering/_utils.py index d5dfbfc83..86b6ab101 100644 --- a/bot/exts/filtering/_utils.py +++ b/bot/exts/filtering/_utils.py @@ -31,7 +31,7 @@ def subclasses_in_package(package: str, prefix: str, parent: T) -> set[T]: # Find all classes in each module... for _, class_ in inspect.getmembers(module, inspect.isclass): # That are a subclass of the given class. - if parent in class_.__bases__: + if parent in class_.__mro__: subclasses.add(class_) return subclasses diff --git a/bot/exts/filtering/filtering.py b/bot/exts/filtering/filtering.py index 44eaa5ea7..514ef39e1 100644 --- a/bot/exts/filtering/filtering.py +++ b/bot/exts/filtering/filtering.py @@ -8,16 +8,15 @@ from typing import Literal, Optional, get_type_hints import discord from botcore.site_api import ResponseCodeError -from discord import Colour, Embed, HTTPException, Message +from discord import Colour, Embed, HTTPException, Message, MessageType from discord.ext import commands from discord.ext.commands import BadArgument, Cog, Context, has_any_role -from discord.utils import escape_markdown import bot import bot.exts.filtering._ui.filter as filters_ui from bot import constants from bot.bot import Bot -from bot.constants import Channels, Colours, MODERATION_ROLES, Roles, Webhooks +from bot.constants import Channels, MODERATION_ROLES, Roles, Webhooks from bot.exts.filtering._filter_context import Event, FilterContext from bot.exts.filtering._filter_lists import FilterList, ListType, filter_list_types, list_type_converter from bot.exts.filtering._filter_lists.filter_list import AtomicList @@ -28,14 +27,18 @@ from bot.exts.filtering._ui.filter import ( ) from bot.exts.filtering._ui.filter_list import FilterListAddView, FilterListEditView, settings_converter from bot.exts.filtering._ui.search import SearchEditView, search_criteria_converter -from bot.exts.filtering._ui.ui import ArgumentCompletionView, DeleteConfirmationView, format_response_error +from bot.exts.filtering._ui.ui import ( + ArgumentCompletionView, DeleteConfirmationView, build_mod_alert, format_response_error +) from bot.exts.filtering._utils import past_tense, repr_equals, starting_value, to_serializable from bot.log import get_logger from bot.pagination import LinePaginator -from bot.utils.messages import format_channel, format_user +from bot.utils.message_cache import MessageCache log = get_logger(__name__) +CACHE_SIZE = 100 + class Filtering(Cog): """Filtering and alerting for content posted on the server.""" @@ -55,6 +58,8 @@ class Filtering(Cog): self.loaded_filters = {} self.loaded_filter_settings = {} + self.message_cache = MessageCache(CACHE_SIZE, newest_first=True) + async def cog_load(self) -> None: """ Fetch the filter data from the API, parse it, and load it to the appropriate data structures. @@ -165,8 +170,9 @@ class Filtering(Cog): @Cog.listener() async def on_message(self, msg: Message) -> None: """Filter the contents of a sent message.""" - if msg.author.bot or msg.webhook_id: + if msg.author.bot or msg.webhook_id or msg.type == MessageType.auto_moderation_action: return + self.message_cache.append(msg) ctx = FilterContext(Event.MESSAGE, msg.author, msg.channel, msg.content, msg, msg.embeds) @@ -766,32 +772,8 @@ class Filtering(Cog): return name = f"{ctx.event.name.replace('_', ' ').title()} Filter" - - embed = Embed(color=Colours.soft_orange) - embed.set_thumbnail(url=ctx.author.display_avatar.url) - triggered_by = f"**Triggered by:** {format_user(ctx.author)}" - if ctx.channel.guild: - triggered_in = f"**Triggered in:** {format_channel(ctx.channel)}\n" - else: - triggered_in = "**Triggered in:** :warning:**DM**:warning:\n" - - filters = [] - for filter_list, list_message in triggered_filters.items(): - if list_message: - filters.append(f"**{filter_list.name.title()} Filters:** {', '.join(list_message)}") - filters = "\n".join(filters) - - matches = "**Matches:** " + ", ".join(repr(match) for match in ctx.matches) - actions = "\n**Actions Taken:** " + (", ".join(ctx.action_descriptions) if ctx.action_descriptions else "-") - content = f"**[Original Content]({ctx.message.jump_url})**:\n{escape_markdown(ctx.content)}" - - embed_content = "\n".join( - part for part in (triggered_by, triggered_in, filters, matches, actions, content) if part - ) - if len(embed_content) > 4000: - embed_content = embed_content[:4000] + " [...]" - embed.description = embed_content - + embed = await build_mod_alert(ctx, triggered_filters) + # There shouldn't be more than 10, but if there are it's not very useful to send them all. await self.webhook.send(username=name, content=ctx.alert_content, embeds=[embed, *ctx.alert_embeds][:10]) async def _resolve_list_type_and_name( diff --git a/bot/exts/moderation/clean.py b/bot/exts/moderation/clean.py index fd9404b1a..aee751345 100644 --- a/bot/exts/moderation/clean.py +++ b/bot/exts/moderation/clean.py @@ -19,6 +19,7 @@ from bot.converters import Age, ISODateTime from bot.exts.moderation.modlog import ModLog from bot.log import get_logger from bot.utils.channel import is_mod_channel +from bot.utils.messages import upload_log log = get_logger(__name__) @@ -351,7 +352,7 @@ class Clean(Cog): # Reverse the list to have reverse chronological order log_messages = reversed(messages) - log_url = await self.mod_log.upload_log(log_messages, ctx.author.id) + log_url = await upload_log(log_messages, ctx.author.id) # Build the embed and send it if channels == "*": diff --git a/bot/exts/moderation/modlog.py b/bot/exts/moderation/modlog.py index efa87ce25..511f05c50 100644 --- a/bot/exts/moderation/modlog.py +++ b/bot/exts/moderation/modlog.py @@ -3,23 +3,20 @@ import difflib import itertools import typing as t from datetime import datetime, timezone -from itertools import zip_longest import discord -from botcore.site_api import ResponseCodeError from dateutil.relativedelta import relativedelta from deepdiff import DeepDiff from discord import Colour, Message, Thread from discord.abc import GuildChannel from discord.ext.commands import Cog, Context from discord.utils import escape_markdown, format_dt, snowflake_time -from sentry_sdk import add_breadcrumb from bot.bot import Bot -from bot.constants import Categories, Channels, Colours, Emojis, Event, Guild as GuildConstant, Icons, Roles, URLs +from bot.constants import Categories, Channels, Colours, Emojis, Event, Guild as GuildConstant, Icons, Roles from bot.log import get_logger from bot.utils import time -from bot.utils.messages import format_user +from bot.utils.messages import format_user, upload_log log = get_logger(__name__) @@ -45,48 +42,6 @@ class ModLog(Cog, name="ModLog"): self._cached_edits = [] - async def upload_log( - self, - messages: t.Iterable[discord.Message], - actor_id: int, - attachments: t.Iterable[t.List[str]] = None - ) -> str: - """Upload message logs to the database and return a URL to a page for viewing the logs.""" - if attachments is None: - attachments = [] - - deletedmessage_set = [ - { - "id": message.id, - "author": message.author.id, - "channel_id": message.channel.id, - "content": message.content.replace("\0", ""), # Null chars cause 400. - "embeds": [embed.to_dict() for embed in message.embeds], - "attachments": attachment, - } - for message, attachment in zip_longest(messages, attachments, fillvalue=[]) - ] - - try: - response = await self.bot.api_client.post( - "bot/deleted-messages", - json={ - "actor": actor_id, - "creation": datetime.now(timezone.utc).isoformat(), - "deletedmessage_set": deletedmessage_set, - } - ) - except ResponseCodeError as e: - add_breadcrumb( - category="api_error", - message=str(e), - level="error", - data=deletedmessage_set, - ) - raise - - return f"{URLs.site_logs_view}/{response['id']}" - def ignore(self, event: Event, *items: int) -> None: """Add event to ignored events to suppress log emission.""" for item in items: @@ -609,7 +564,7 @@ class ModLog(Cog, name="ModLog"): remaining_chars = 4090 - len(response) if len(content) > remaining_chars: - botlog_url = await self.upload_log(messages=[message], actor_id=message.author.id) + botlog_url = await upload_log(messages=[message], actor_id=message.author.id) ending = f"\n\nMessage truncated, [full message here]({botlog_url})." truncation_point = remaining_chars - len(ending) content = f"{content[:truncation_point]}...{ending}" diff --git a/bot/utils/messages.py b/bot/utils/messages.py index 63929cd0b..c5f6dc41a 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -1,16 +1,21 @@ import asyncio import random import re +from collections.abc import Iterable +from datetime import datetime, timezone from functools import partial from io import BytesIO from typing import Callable, List, Optional, Sequence, Union import discord +from botcore.site_api import ResponseCodeError from botcore.utils import scheduling +from discord import Message from discord.ext.commands import Context +from sentry_sdk import add_breadcrumb import bot -from bot.constants import Emojis, MODERATION_ROLES, NEGATIVE_REPLIES +from bot.constants import Emojis, MODERATION_ROLES, NEGATIVE_REPLIES, URLs from bot.log import get_logger log = get_logger(__name__) @@ -235,7 +240,7 @@ async def send_denial(ctx: Context, reason: str) -> discord.Message: return await ctx.send(embed=embed) -def format_user(user: discord.abc.User) -> str: +def format_user(user: discord.User | discord.Member) -> str: """Return a string for `user` which has their mention and ID.""" return f"{user.mention} (`{user.id}`)" @@ -247,3 +252,43 @@ def format_channel(channel: discord.abc.Messageable) -> str: formatted += f"/{channel.parent}" formatted += ")" return formatted + + +async def upload_log(messages: Iterable[Message], actor_id: int, attachments: dict[int, list[str]] = None) -> str: + """Upload message logs to the database and return a URL to a page for viewing the logs.""" + if attachments is None: + attachments = [] + else: + attachments = [attachments.get(message.id, []) for message in messages] + + deletedmessage_set = [ + { + "id": message.id, + "author": message.author.id, + "channel_id": message.channel.id, + "content": message.content.replace("\0", ""), # Null chars cause 400. + "embeds": [embed.to_dict() for embed in message.embeds], + "attachments": attachment, + } + for message, attachment in zip(messages, attachments) + ] + + try: + response = await bot.instance.api_client.post( + "bot/deleted-messages", + json={ + "actor": actor_id, + "creation": datetime.now(timezone.utc).isoformat(), + "deletedmessage_set": deletedmessage_set, + } + ) + except ResponseCodeError as e: + add_breadcrumb( + category="api_error", + message=str(e), + level="error", + data=deletedmessage_set, + ) + raise + + return f"{URLs.site_logs_view}/{response['id']}" |