Merge pull request #1446 from Shivansh-007/feature/enhance-incidents

Enhancing information within the incidents channel.

4 files changed, 342 insertions, 17 deletions

diff --git a/bot/constants.py b/bot/constants.py
index e3846fb3d..2dfdd51e2 100644
--- a/bot/constants.py
+++ b/bot/constants.py
@@ -476,6 +476,7 @@ class Webhooks(metaclass=YAMLGetter):
     big_brother: int
     dev_log: int
     duck_pond: int
+    incidents: int
     incidents_archive: int
 
 
diff --git a/bot/exts/moderation/incidents.py b/bot/exts/moderation/incidents.py
index e265e29d3..77dfad255 100644
--- a/bot/exts/moderation/incidents.py
+++ b/bot/exts/moderation/incidents.py
@@ -1,16 +1,18 @@
 import asyncio
-import typing as t
+import re
 from datetime import datetime
 from enum import Enum
+from typing import Optional
 
 import discord
-from discord.ext.commands import Cog
+from async_rediscache import RedisCache
+from discord.ext.commands import Cog, Context, MessageConverter, MessageNotFound
 
 from bot.bot import Bot
-from bot.constants import Channels, Colours, Emojis, Guild, Webhooks
+from bot.constants import Channels, Colours, Emojis, Guild, Roles, Webhooks
 from bot.log import get_logger
 from bot.utils import scheduling
-from bot.utils.messages import sub_clyde
+from bot.utils.messages import format_user, sub_clyde
 
 log = get_logger(__name__)
 
@@ -22,6 +24,12 @@ CRAWL_LIMIT = 50
 # Seconds for `crawl_task` to sleep after adding reactions to a message
 CRAWL_SLEEP = 2
 
+DISCORD_MESSAGE_LINK_RE = re.compile(
+    r"(https?:\/\/(?:(ptb|canary|www)\.)?discord(?:app)?\.com\/channels\/"
+    r"[0-9]{15,20}"
+    r"\/[0-9]{15,20}\/[0-9]{15,20})"
+)
+
 
 class Signal(Enum):
     """
@@ -37,17 +45,17 @@ class Signal(Enum):
 
 
 # Reactions from non-mod roles will be removed
-ALLOWED_ROLES: t.Set[int] = set(Guild.moderation_roles)
+ALLOWED_ROLES: set[int] = set(Guild.moderation_roles)
 
 # Message must have all of these emoji to pass the `has_signals` check
-ALL_SIGNALS: t.Set[str] = {signal.value for signal in Signal}
+ALL_SIGNALS: set[str] = {signal.value for signal in Signal}
 
 # An embed coupled with an optional file to be dispatched
 # If the file is not None, the embed attempts to show it in its body
-FileEmbed = t.Tuple[discord.Embed, t.Optional[discord.File]]
+FileEmbed = tuple[discord.Embed, Optional[discord.File]]
 
 
-async def download_file(attachment: discord.Attachment) -> t.Optional[discord.File]:
+async def download_file(attachment: discord.Attachment) -> Optional[discord.File]:
     """
     Download & return `attachment` file.
 
@@ -121,7 +129,7 @@ def is_incident(message: discord.Message) -> bool:
     return all(conditions)
 
 
-def own_reactions(message: discord.Message) -> t.Set[str]:
+def own_reactions(message: discord.Message) -> set[str]:
     """Get the set of reactions placed on `message` by the bot itself."""
     return {str(reaction.emoji) for reaction in message.reactions if reaction.me}
 
@@ -131,6 +139,108 @@ def has_signals(message: discord.Message) -> bool:
     return ALL_SIGNALS.issubset(own_reactions(message))
 
 
+def shorten_text(text: str) -> str:
+    """
+    Truncate the text if there are over 3 lines or 300 characters, or if it is a single word.
+
+    The maximum length of the string would be 303 characters across 3 lines at maximum.
+    """
+    original_length = len(text)
+    # Truncate text to a maximum of 300 characters
+    if len(text) > 300:
+        text = text[:300]
+
+    # Limit to a maximum of three lines
+    text = "\n".join(text.split("\n", maxsplit=3)[:3])
+
+    # If it is a single word, then truncate it to 50 characters
+    if text.find(" ") == -1:
+        text = text[:50]
+
+    # Remove extra whitespaces from the `text`
+    text = text.strip()
+
+    # Add placeholder if the text was shortened
+    if len(text) < original_length:
+        text = f"{text}..."
+
+    return text
+
+
+async def make_message_link_embed(ctx: Context, message_link: str) -> Optional[discord.Embed]:
+    """
+    Create an embedded representation of the discord message link contained in the incident report.
+
+    The Embed would contain the following information -->
+        Author: @Jason Terror ♦ (736234578745884682)
+        Channel: Special/#bot-commands (814190307980607493)
+        Content: This is a very important message!
+    """
+    embed = None
+
+    try:
+        message: discord.Message = await MessageConverter().convert(ctx, message_link)
+    except MessageNotFound:
+        mod_logs_channel = ctx.bot.get_channel(Channels.mod_log)
+
+        last_100_logs: list[discord.Message] = await mod_logs_channel.history(limit=100).flatten()
+
+        for log_entry in last_100_logs:
+            if not log_entry.embeds:
+                continue
+
+            log_embed: discord.Embed = log_entry.embeds[0]
+            if (
+                    log_embed.author.name == "Message deleted"
+                    and f"[Jump to message]({message_link})" in log_embed.description
+            ):
+                embed = discord.Embed(
+                    colour=discord.Colour.dark_gold(),
+                    title="Deleted Message Link",
+                    description=(
+                        f"Found <#{Channels.mod_log}> entry for deleted message: "
+                        f"[Jump to message]({log_entry.jump_url})."
+                    )
+                )
+        if not embed:
+            embed = discord.Embed(
+                colour=discord.Colour.red(),
+                title="Bad Message Link",
+                description=f"Message {message_link} not found."
+            )
+    except discord.DiscordException as e:
+        log.exception(f"Failed to make message link embed for '{message_link}', raised exception: {e}")
+    else:
+        channel = message.channel
+        if not channel.permissions_for(channel.guild.get_role(Roles.helpers)).view_channel:
+            log.info(
+                f"Helpers don't have read permissions in #{channel.name},"
+                f" not sending message link embed for {message_link}"
+            )
+            return
+
+        embed = discord.Embed(
+            colour=discord.Colour.gold(),
+            description=(
+                f"**Author:** {format_user(message.author)}\n"
+                f"**Channel:** {channel.mention} ({channel.category}"
+                f"{f'/#{channel.parent.name} - ' if isinstance(channel, discord.Thread) else '/#'}"
+                f"{channel.name})\n"
+            ),
+            timestamp=message.created_at
+        )
+        embed.add_field(
+            name="Content",
+            value=shorten_text(message.content) if message.content else "[No Message Content]"
+        )
+        embed.set_footer(text=f"Message ID: {message.id}")
+
+        if message.attachments:
+            embed.set_image(url=message.attachments[0].url)
+
+    return embed
+
+
 async def add_signals(incident: discord.Message) -> None:
     """
     Add `Signal` member emoji to `incident` as reactions.
@@ -168,6 +278,7 @@ class Incidents(Cog):
         * See: `crawl_incidents`
 
     On message:
+        * Run message through `extract_message_links` and send them into the channel
         * Add `Signal` member emoji if message qualifies as an incident
         * Ignore messages starting with #
             * Use this if verbal communication is necessary
@@ -181,18 +292,35 @@ class Incidents(Cog):
         * If `Signal.ACTIONED` or `Signal.NOT_ACTIONED` were chosen, attempt to
           relay the incident message to #incidents-archive
         * If relay successful, delete original message
+        * Delete quotation message if cached
         * See: `on_raw_reaction_add`
 
     Please refer to function docstrings for implementation details.
     """
 
+    # This dictionary maps an incident report message to the message link embed's ID
+    # RedisCache[discord.Message.id, discord.Message.id]
+    message_link_embeds_cache = RedisCache()
+
     def __init__(self, bot: Bot) -> None:
         """Prepare `event_lock` and schedule `crawl_task` on start-up."""
         self.bot = bot
+        self.incidents_webhook = None
+
+        scheduling.create_task(self.fetch_webhook(), event_loop=self.bot.loop)
 
         self.event_lock = asyncio.Lock()
         self.crawl_task = scheduling.create_task(self.crawl_incidents(), event_loop=self.bot.loop)
 
+    async def fetch_webhook(self) -> None:
+        """Fetch the incidents webhook object, so we can post message link embeds to it."""
+        await self.bot.wait_until_guild_available()
+
+        try:
+            self.incidents_webhook = await self.bot.fetch_webhook(Webhooks.incidents)
+        except discord.HTTPException:
+            log.error(f"Failed to fetch incidents webhook with id `{Webhooks.incidents}`.")
+
     async def crawl_incidents(self) -> None:
         """
         Crawl #incidents and add missing emoji where necessary.
@@ -292,8 +420,11 @@ class Incidents(Cog):
         This ensures that if there is a racing event awaiting the lock, it will fail to find the
         message, and will abort. There is a `timeout` to ensure that this doesn't hold the lock
         forever should something go wrong.
+
+        Deletes cache value (`message_link_embeds_cache`) of `incident` if it exists. It then removes the
+        webhook message for that particular link from the channel.
         """
-        members_roles: t.Set[int] = {role.id for role in member.roles}
+        members_roles: set[int] = {role.id for role in member.roles}
         if not members_roles & ALLOWED_ROLES:  # Intersection is truthy on at least 1 common element
             log.debug(f"Removing invalid reaction: user {member} is not permitted to send signals")
             try:
@@ -340,7 +471,11 @@ class Incidents(Cog):
         else:
             log.trace("Deletion was confirmed")
 
-    async def resolve_message(self, message_id: int) -> t.Optional[discord.Message]:
+        if self.incidents_webhook:
+            # Deletes the message link embeds found in cache from the channel and cache.
+            await self.delete_msg_link_embed(incident.id)
+
+    async def resolve_message(self, message_id: int) -> Optional[discord.Message]:
         """
         Get `discord.Message` for `message_id` from cache, or API.
 
@@ -355,7 +490,7 @@ class Incidents(Cog):
         """
         await self.bot.wait_until_guild_available()  # First make sure that the cache is ready
         log.trace(f"Resolving message for: {message_id=}")
-        message: t.Optional[discord.Message] = self.bot._connection._get_message(message_id)
+        message: Optional[discord.Message] = self.bot._connection._get_message(message_id)
 
         if message is not None:
             log.trace("Message was found in cache")
@@ -419,9 +554,107 @@ class Incidents(Cog):
 
     @Cog.listener()
     async def on_message(self, message: discord.Message) -> None:
-        """Pass `message` to `add_signals` if and only if it satisfies `is_incident`."""
-        if is_incident(message):
-            await add_signals(message)
+        """
+        Pass `message` to `add_signals` and `extract_message_links` if it satisfies `is_incident`.
+
+        If `message` is an incident report, then run it through `extract_message_links` to get all
+        the message link embeds (embeds which contain information about that particular link).
+        These message link embeds are then sent into the channel.
+
+        Also passes the message into `add_signals` if the message is an incident.
+        """
+        if not is_incident(message):
+            return
+
+        await add_signals(message)
+
+        # Only use this feature if incidents webhook embed is found
+        if self.incidents_webhook:
+            if embed_list := await self.extract_message_links(message):
+                await self.send_message_link_embeds(embed_list, message, self.incidents_webhook)
+
+    @Cog.listener()
+    async def on_raw_message_delete(self, payload: discord.RawMessageDeleteEvent) -> None:
+        """
+        Delete message link embeds for `payload.message_id`.
+
+        Search through the cache for message, if found delete it from cache and channel.
+        """
+        if self.incidents_webhook:
+            await self.delete_msg_link_embed(payload.message_id)
+
+    async def extract_message_links(self, message: discord.Message) -> Optional[list[discord.Embed]]:
+        """
+        Check if there's any message links in the text content.
+
+        Then pass the message_link into `make_message_link_embed` to format an
+        embed for it containing information about the link.
+
+        As Discord only allows a max of 10 embeds in a single webhook, just send the
+        first 10 embeds and don't care about the rest.
+
+        If no links are found for the message, just log a trace statement.
+        """
+        message_links = DISCORD_MESSAGE_LINK_RE.findall(message.content)
+        if not message_links:
+            log.trace(
+                f"No message links detected on incident message with id {message.id}."
+            )
+            return
+
+        embeds = []
+        for message_link in message_links[:10]:
+            ctx = await self.bot.get_context(message)
+            embed = await make_message_link_embed(ctx, message_link[0])
+            if embed:
+                embeds.append(embed)
+
+        return embeds
+
+    async def send_message_link_embeds(
+            self,
+            webhook_embed_list: list,
+            message: discord.Message,
+            webhook: discord.Webhook,
+    ) -> Optional[int]:
+        """
+        Send message link embeds to #incidents channel.
+
+        Using the `webhook` passed in as a parameter to send
+        the embeds in the `webhook_embed_list` parameter.
+
+        After sending each embed it maps the `message.id`
+        to the `webhook_msg_ids` IDs in the async redis-cache.
+        """
+        try:
+            webhook_msg = await webhook.send(
+                embeds=[embed for embed in webhook_embed_list if embed],
+                username=sub_clyde(message.author.name),
+                avatar_url=message.author.display_avatar.url,
+                wait=True,
+            )
+        except discord.DiscordException:
+            log.exception(
+                f"Failed to send message link embed {message.id} to #incidents."
+            )
+        else:
+            await self.message_link_embeds_cache.set(message.id, webhook_msg.id)
+            log.trace("Message link embeds sent successfully to #incidents!")
+            return webhook_msg.id
+
+    async def delete_msg_link_embed(self, message_id: int) -> None:
+        """Delete the Discord message link message found in cache for `message_id`."""
+        log.trace("Deleting Discord message link's webhook message.")
+        webhook_msg_id = await self.message_link_embeds_cache.get(int(message_id))
+
+        if webhook_msg_id:
+            try:
+                await self.incidents_webhook.delete_message(webhook_msg_id)
+            except discord.errors.NotFound:
+                log.trace(f"Incidents message link embed (`{webhook_msg_id}`) has already been deleted, skipping.")
+
+        await self.message_link_embeds_cache.delete(message_id)
+        log.trace("Successfully deleted discord links webhook message.")
 
 
 def setup(bot: Bot) -> None:
diff --git a/config-default.yml b/config-default.yml
index 4a85ccc56..ed3c3a638 100644
--- a/config-default.yml
+++ b/config-default.yml
@@ -307,6 +307,7 @@ guild:
         big_brother:                        569133704568373283
         dev_log:                            680501655111729222
         duck_pond:                          637821475327311927
+        incidents:                          816650601844572212
         incidents_archive:                  720671599790915702
         python_news:        &PYNEWS_WEBHOOK 704381182279942324
 
diff --git a/tests/bot/exts/moderation/test_incidents.py b/tests/bot/exts/moderation/test_incidents.py
index ccc842050..cfe0c4b03 100644
--- a/tests/bot/exts/moderation/test_incidents.py
+++ b/tests/bot/exts/moderation/test_incidents.py
@@ -3,13 +3,16 @@ import enum
 import logging
 import typing as t
 import unittest
+from unittest import mock
 from unittest.mock import AsyncMock, MagicMock, Mock, call, patch
 
 import aiohttp
 import discord
+from async_rediscache import RedisSession
 
 from bot.constants import Colours
 from bot.exts.moderation import incidents
+from bot.utils.messages import format_user
 from tests.helpers import (
     MockAsyncWebhook, MockAttachment, MockBot, MockMember, MockMessage, MockReaction, MockRole, MockTextChannel,
     MockUser
@@ -276,6 +279,22 @@ class TestIncidents(unittest.IsolatedAsyncioTestCase):
     the instance as they wish.
     """
 
+    session = None
+
+    async def flush(self):
+        """Flush everything from the database to prevent carry-overs between tests."""
+        with await self.session.pool as connection:
+            await connection.flushall()
+
+    async def asyncSetUp(self):  # noqa: N802
+        self.session = RedisSession(use_fakeredis=True)
+        await self.session.connect()
+        await self.flush()
+
+    async def asyncTearDown(self):  # noqa: N802
+        if self.session:
+            await self.session.close()
+
     def setUp(self):
         """
         Prepare a fresh `Incidents` instance for each test.
@@ -506,7 +525,7 @@ class TestProcessEvent(TestIncidents):
         with patch("bot.exts.moderation.incidents.Incidents.make_confirmation_task", mock_task):
             await self.cog_instance.process_event(
                 reaction=incidents.Signal.ACTIONED.value,
-                incident=MockMessage(),
+                incident=MockMessage(id=123),
                 member=MockMember(roles=[MockRole(id=1)])
             )
 
@@ -526,7 +545,7 @@ class TestProcessEvent(TestIncidents):
             with patch("bot.exts.moderation.incidents.Incidents.make_confirmation_task", mock_task):
                 await self.cog_instance.process_event(
                     reaction=incidents.Signal.ACTIONED.value,
-                    incident=MockMessage(),
+                    incident=MockMessage(id=123),
                     member=MockMember(roles=[MockRole(id=1)])
                 )
         except asyncio.TimeoutError:
@@ -761,3 +780,74 @@ class TestOnMessage(TestIncidents):
             await self.cog_instance.on_message(MockMessage())
 
         mock_add_signals.assert_not_called()
+
+
+class TestMessageLinkEmbeds(TestIncidents):
+    """Tests for `extract_message_links` coroutine."""
+
+    async def test_shorten_text(self):
+        """Test all cases of text shortening by mocking messages."""
+        tests = {
+            "thisisasingleword"*10: "thisisasinglewordthisisasinglewordthisisasinglewor...",
+
+            "\n".join("Lets make a new line test".split()): "Lets\nmake\na...",
+
+            'Hello, World!' * 300: (
+                "Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!"
+                "Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!"
+                "Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!Hello, World!"
+                "Hello, World!Hello, World!H..."
+            )
+        }
+
+        for content, expected_conversion in tests.items():
+            with self.subTest(content=content, expected_conversion=expected_conversion):
+                conversion = incidents.shorten_text(content)
+                self.assertEqual(conversion, expected_conversion)
+
+    async def extract_and_form_message_link_embeds(self):
+        """
+        Extract message links from a mocked message and form the message link embed.
+
+        Considers all types of message links, discord supports.
+        """
+        self.guild_id_patcher = mock.patch("bot.exts.backend.sync._cog.constants.Guild.id", 5)
+        self.guild_id = self.guild_id_patcher.start()
+
+        msg = MockMessage(id=555, content="Hello, World!" * 3000)
+        msg.channel.mention = "#lemonade-stand"
+
+        msg_links = [
+            # Valid Message links
+            f"https://discord.com/channels/{self.guild_id}/{msg.channel.discord_id}/{msg.discord_id}",
+            f"http://canary.discord.com/channels/{self.guild_id}/{msg.channel.discord_id}/{msg.discord_id}",
+
+            # Invalid Message links
+            f"https://discord.com/channels/{msg.channel.discord_id}/{msg.discord_id}",
+            f"https://discord.com/channels/{self.guild_id}/{msg.channel.discord_id}000/{msg.discord_id}",
+        ]
+
+        incident_msg = MockMessage(
+            id=777,
+            content=(
+                f"I would like to report the following messages, "
+                f"as they break our rules: \n{', '.join(msg_links)}"
+            )
+        )
+
+        with patch(
+                "bot.exts.moderation.incidents.Incidents.extract_message_links", AsyncMock()
+        ) as mock_extract_message_links:
+            embeds = mock_extract_message_links(incident_msg)
+            description = (
+                f"**Author:** {format_user(msg.author)}\n"
+                f"**Channel:** {msg.channel.mention} ({msg.channel.category}/#{msg.channel.name})\n"
+                f"**Content:** {('Hello, World!' * 3000)[:300] + '...'}\n"
+            )
+
+            # Check number of embeds returned with number of valid links
+            self.assertEqual(len(embeds), 2)
+
+            # Check for the embed descriptions
+            for embed in embeds:
+                self.assertEqual(embed.description, description)