Merge pull request #1293 from ks129/emojis-filter

Include Unicode emojis to emojis filter

4 files changed, 61 insertions, 26 deletions

diff --git a/Pipfile b/Pipfile
index 103ce84cf..ae80ae2ae 100644
--- a/Pipfile
+++ b/Pipfile
@@ -26,6 +26,7 @@ requests = "~=2.22"
 sentry-sdk = "~=0.14"
 sphinx = "~=2.2"
 statsd = "~=3.3"
+emoji = "~=0.6"
 
 [dev-packages]
 coverage = "~=5.0"
diff --git a/Pipfile.lock b/Pipfile.lock
index 6a6a1aaf6..541db1627 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "ca6b100f7ee2e6e01eec413a754fc11be064e965a255b2c4927d4a2dd1c451ec"
+            "sha256": "3ccb368599709d2970f839fc3721cfeebcd5a2700fed7231b2ce38a080828325"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -222,6 +222,13 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==0.16"
         },
+        "emoji": {
+            "hashes": [
+                "sha256:e42da4f8d648f8ef10691bc246f682a1ec6b18373abfd9be10ec0b398823bd11"
+            ],
+            "index": "pypi",
+            "version": "==0.6.0"
+        },
         "fakeredis": {
             "hashes": [
                 "sha256:8070b7fce16f828beaef2c757a4354af91698685d5232404f1aeeb233529c7a5",
@@ -548,16 +555,18 @@
         },
         "pyyaml": {
             "hashes": [
-                "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97",
+                "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf",
+                "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a",
+                "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c",
                 "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76",
-                "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2",
+                "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e",
                 "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648",
-                "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf",
+                "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d",
+                "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97",
                 "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f",
-                "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2",
+                "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2",
                 "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee",
-                "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d",
-                "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c",
+                "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2",
                 "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"
             ],
             "index": "pypi",
@@ -581,11 +590,11 @@
         },
         "sentry-sdk": {
             "hashes": [
-                "sha256:81d7a5d8ca0b13a16666e8280127b004565aa988bfeec6481e98a8601804b215",
-                "sha256:fd48f627945511c140546939b4d73815be4860cd1d2b9149577d7f6563e7bd60"
+                "sha256:1052f0ed084e532f66cb3e4ba617960d820152aee8b93fc6c05bd53861768c1c",
+                "sha256:4c42910a55a6b1fe694d5e4790d5188d105d77b5a6346c1c64cbea8c06c0e8b7"
             ],
             "index": "pypi",
-            "version": "==0.19.3"
+            "version": "==0.19.4"
         },
         "six": {
             "hashes": [
@@ -793,11 +802,11 @@
         },
         "coveralls": {
             "hashes": [
-                "sha256:4430b862baabb3cf090d36d84d331966615e4288d8a8c5957e0fd456d0dd8bd6",
-                "sha256:b3b60c17b03a0dee61952a91aed6f131e0b2ac8bd5da909389c53137811409e1"
+                "sha256:2301a19500b06649d2ec4f2858f9c69638d7699a4c63027c5d53daba666147cc",
+                "sha256:b990ba1f7bc4288e63340be0433698c1efe8217f78c689d254c2540af3d38617"
             ],
             "index": "pypi",
-            "version": "==2.1.2"
+            "version": "==2.2.0"
         },
         "distlib": {
             "hashes": [
@@ -961,16 +970,18 @@
         },
         "pyyaml": {
             "hashes": [
-                "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97",
+                "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf",
+                "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a",
+                "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c",
                 "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76",
-                "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2",
+                "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e",
                 "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648",
-                "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf",
+                "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d",
+                "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97",
                 "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f",
-                "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2",
+                "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2",
                 "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee",
-                "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d",
-                "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c",
+                "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2",
                 "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"
             ],
             "index": "pypi",
diff --git a/bot/rules/discord_emojis.py b/bot/rules/discord_emojis.py
index 6e47f0197..41faf7ee8 100644
--- a/bot/rules/discord_emojis.py
+++ b/bot/rules/discord_emojis.py
@@ -2,16 +2,17 @@ import re
 from typing import Dict, Iterable, List, Optional, Tuple
 
 from discord import Member, Message
+from emoji import demojize
 
 
-DISCORD_EMOJI_RE = re.compile(r"<:\w+:\d+>")
+DISCORD_EMOJI_RE = re.compile(r"<:\w+:\d+>|:\w+:")
 CODE_BLOCK_RE = re.compile(r"```.*?```", flags=re.DOTALL)
 
 
 async def apply(
     last_message: Message, recent_messages: List[Message], config: Dict[str, int]
 ) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]:
-    """Detects total Discord emojis (excluding Unicode emojis) exceeding the limit sent by a single user."""
+    """Detects total Discord emojis exceeding the limit sent by a single user."""
     relevant_messages = tuple(
         msg
         for msg in recent_messages
@@ -19,8 +20,9 @@ async def apply(
     )
 
     # Get rid of code blocks in the message before searching for emojis.
+    # Convert Unicode emojis to :emoji: format to get their count.
     total_emojis = sum(
-        len(DISCORD_EMOJI_RE.findall(CODE_BLOCK_RE.sub("", msg.content)))
+        len(DISCORD_EMOJI_RE.findall(demojize(CODE_BLOCK_RE.sub("", msg.content))))
         for msg in relevant_messages
     )
 
diff --git a/tests/bot/rules/test_discord_emojis.py b/tests/bot/rules/test_discord_emojis.py
index 9a72723e2..66c2d9f92 100644
--- a/tests/bot/rules/test_discord_emojis.py
+++ b/tests/bot/rules/test_discord_emojis.py
@@ -5,11 +5,12 @@ from tests.bot.rules import DisallowedCase, RuleTest
 from tests.helpers import MockMessage
 
 discord_emoji = "<:abcd:1234>"  # Discord emojis follow the format <:name:id>
+unicode_emoji = "🧪"
 
 
-def make_msg(author: str, n_emojis: int) -> MockMessage:
+def make_msg(author: str, n_emojis: int, emoji: str = discord_emoji) -> MockMessage:
     """Build a MockMessage instance with content containing `n_emojis` arbitrary emojis."""
-    return MockMessage(author=author, content=discord_emoji * n_emojis)
+    return MockMessage(author=author, content=emoji * n_emojis)
 
 
 class DiscordEmojisRuleTests(RuleTest):
@@ -20,16 +21,22 @@ class DiscordEmojisRuleTests(RuleTest):
         self.config = {"max": 2, "interval": 10}
 
     async def test_allows_messages_within_limit(self):
-        """Cases with a total amount of discord emojis within limit."""
+        """Cases with a total amount of discord and unicode emojis within limit."""
         cases = (
             [make_msg("bob", 2)],
             [make_msg("alice", 1), make_msg("bob", 2), make_msg("alice", 1)],
+            [make_msg("bob", 2, unicode_emoji)],
+            [
+                make_msg("alice", 1, unicode_emoji),
+                make_msg("bob", 2, unicode_emoji),
+                make_msg("alice", 1, unicode_emoji)
+            ],
         )
 
         await self.run_allowed(cases)
 
     async def test_disallows_messages_beyond_limit(self):
-        """Cases with more than the allowed amount of discord emojis."""
+        """Cases with more than the allowed amount of discord and unicode emojis."""
         cases = (
             DisallowedCase(
                 [make_msg("bob", 3)],
@@ -41,6 +48,20 @@ class DiscordEmojisRuleTests(RuleTest):
                 ("alice",),
                 4,
             ),
+            DisallowedCase(
+                [make_msg("bob", 3, unicode_emoji)],
+                ("bob",),
+                3,
+            ),
+            DisallowedCase(
+                [
+                    make_msg("alice", 2, unicode_emoji),
+                    make_msg("bob", 2, unicode_emoji),
+                    make_msg("alice", 2, unicode_emoji)
+                ],
+                ("alice",),
+                4
+            )
         )
 
         await self.run_disallowed(cases)