StatsD integration

15 files changed, 145 insertions, 34 deletions

diff --git a/Pipfile b/Pipfile
index 04cc98427..e7fb61957 100644
--- a/Pipfile
+++ b/Pipfile
@@ -19,7 +19,8 @@ requests = "~=2.22"
 more_itertools = "~=8.2"
 sentry-sdk = "~=0.14"
 coloredlogs = "~=14.0"
-colorama = {version = "~=0.4.3", sys_platform = "== 'win32'"}
+colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"}
+statsd = "~=3.3"
 
 [dev-packages]
 coverage = "~=5.0"
diff --git a/Pipfile.lock b/Pipfile.lock
index ad9a3173a..19e03bda4 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "2d3ba484e8467a115126b2ba39fa5f36f103ea455477813dd658797875c79cc9"
+            "sha256": "10636aef5a07f17bd00608df2cc5214fcbfe3de4745cdeea7a076b871754620a"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -87,18 +87,18 @@
         },
         "beautifulsoup4": {
             "hashes": [
-                "sha256:05fd825eb01c290877657a56df4c6e4c311b3965bda790c613a3d6fb01a5462a",
-                "sha256:9fbb4d6e48ecd30bcacc5b63b94088192dcda178513b2ae3c394229f8911b887",
-                "sha256:e1505eeed31b0f4ce2dbb3bc8eb256c04cc2b3b72af7d551a4ab6efd5cbe5dae"
+                "sha256:594ca51a10d2b3443cbac41214e12dbb2a1cd57e1a7344659849e2e20ba6a8d8",
+                "sha256:a4bbe77fd30670455c5296242967a123ec28c37e9702a8a81bd2f20a4baf0368",
+                "sha256:d4e96ac9b0c3a6d3f0caae2e4124e6055c5dcafde8e2f831ff194c104f0775a0"
             ],
-            "version": "==4.8.2"
+            "version": "==4.9.0"
         },
         "certifi": {
             "hashes": [
-                "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3",
-                "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"
+                "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304",
+                "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519"
             ],
-            "version": "==2019.11.28"
+            "version": "==2020.4.5.1"
         },
         "cffi": {
             "hashes": [
@@ -167,10 +167,10 @@
         },
         "discord-py": {
             "hashes": [
-                "sha256:7424be26b07b37ecad4404d9383d685995a0e0b3df3f9c645bdd3a4d977b83b4"
+                "sha256:406871b06d86c3dc49fba63238519f28628dac946fef8a0e22988ff58ec05580"
             ],
             "index": "pypi",
-            "version": "==1.3.2"
+            "version": "==1.3.3"
         },
         "docutils": {
             "hashes": [
@@ -393,17 +393,10 @@
         },
         "pyparsing": {
             "hashes": [
-                "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f",
-                "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"
+                "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
+                "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
             ],
-            "version": "==2.4.6"
-        },
-        "pyreadline": {
-            "hashes": [
-                "sha256:4530592fc2e85b25b1a9f79664433da09237c1a270e4d78ea5aa3a2c7229e2d1"
-            ],
-            "markers": "sys_platform == 'win32'",
-            "version": "==2.1"
+            "version": "==2.4.7"
         },
         "python-dateutil": {
             "hashes": [
@@ -524,6 +517,14 @@
             ],
             "version": "==1.1.4"
         },
+        "statsd": {
+            "hashes": [
+                "sha256:c610fb80347fca0ef62666d241bce64184bd7cc1efe582f9690e045c25535eaa",
+                "sha256:e3e6db4c246f7c59003e51c9720a51a7f39a396541cb9b147ff4b14d15b5dd1f"
+            ],
+            "index": "pypi",
+            "version": "==3.3.0"
+        },
         "urllib3": {
             "hashes": [
                 "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc",
@@ -717,11 +718,11 @@
         },
         "flake8-tidy-imports": {
             "hashes": [
-                "sha256:5b6e75cec6d751e66534c522fbdce7dac1c2738b1216b0f6b10453995932e188",
-                "sha256:cf26fbb3ab31a398f265d53b6f711d80006450c19221e41b2b7b0e0b14ac39c5"
+                "sha256:62059ca07d8a4926b561d392cbab7f09ee042350214a25cf12823384a45d27dd",
+                "sha256:c30b40337a2e6802ba3bb611c26611154a27e94c53fc45639e3e282169574fd3"
             ],
             "index": "pypi",
-            "version": "==4.0.1"
+            "version": "==4.1.0"
         },
         "flake8-todo": {
             "hashes": [
@@ -732,10 +733,10 @@
         },
         "identify": {
             "hashes": [
-                "sha256:a7577a1f55cee1d21953a5cf11a3c839ab87f5ef909a4cba6cf52ed72b4c6059",
-                "sha256:ab246293e6585a1c6361a505b68d5b501a0409310932b7de2c2ead667b564d89"
+                "sha256:2bb8760d97d8df4408f4e805883dad26a2d076f04be92a10a3e43f09c6060742",
+                "sha256:faffea0fd8ec86bb146ac538ac350ed0c73908326426d387eded0bcc9d077522"
             ],
-            "version": "==1.4.13"
+            "version": "==1.4.14"
         },
         "mccabe": {
             "hashes": [
@@ -835,10 +836,10 @@
         },
         "virtualenv": {
             "hashes": [
-                "sha256:87831f1070534b636fea2241dd66f3afe37ac9041bcca6d0af3215cdcfbf7d82",
-                "sha256:f3128d882383c503003130389bf892856341c1da12c881ae24d6358c82561b55"
+                "sha256:00cfe8605fb97f5a59d52baab78e6070e72c12ca64f51151695407cc0eb8a431",
+                "sha256:c8364ec469084046c779c9a11ae6340094e8a0bf1d844330fc55c1cefe67c172"
             ],
-            "version": "==20.0.13"
+            "version": "==20.0.17"
         }
     }
 }
diff --git a/bot/__main__.py b/bot/__main__.py
index bf98f2cfd..2125e8590 100644
--- a/bot/__main__.py
+++ b/bot/__main__.py
@@ -57,6 +57,7 @@ bot.load_extension("bot.cogs.reminders")
 bot.load_extension("bot.cogs.site")
 bot.load_extension("bot.cogs.snekbox")
 bot.load_extension("bot.cogs.sync")
+bot.load_extension("bot.cogs.stats")
 bot.load_extension("bot.cogs.tags")
 bot.load_extension("bot.cogs.token_remover")
 bot.load_extension("bot.cogs.utils")
diff --git a/bot/bot.py b/bot/bot.py
index 950ac6751..65081e438 100644
--- a/bot/bot.py
+++ b/bot/bot.py
@@ -6,10 +6,10 @@ from typing import Optional
 
 import aiohttp
 import discord
+import statsd
 from discord.ext import commands
 
-from bot import api
-from bot import constants
+from bot import DEBUG_MODE, api, constants
 
 log = logging.getLogger('bot')
 
@@ -33,6 +33,16 @@ class Bot(commands.Bot):
         self._resolver = None
         self._guild_available = asyncio.Event()
 
+        statsd_url = constants.Bot.statsd_host
+
+        if DEBUG_MODE:
+            # Since statsd is UDP, there are no errors for sending to a down port.
+            # For this reason, setting the statsd host to 127.0.0.1 for development
+            # will effectively disable stats.
+            statsd_url = "127.0.0.1"
+
+        self.stats = statsd.StatsClient(statsd_url, 8125, prefix="bot")
+
     def add_cog(self, cog: commands.Cog) -> None:
         """Adds a "cog" to the bot and logs the operation."""
         super().add_cog(cog)
diff --git a/bot/cogs/antispam.py b/bot/cogs/antispam.py
index baa6b9459..d63acbc4a 100644
--- a/bot/cogs/antispam.py
+++ b/bot/cogs/antispam.py
@@ -182,6 +182,7 @@ class AntiSpam(Cog):
             # which contains the reason for why the message violated the rule and
             # an iterable of all members that violated the rule.
             if result is not None:
+                self.bot.stats.incr(f"mod_alerts.{rule_name}")
                 reason, members, relevant_messages = result
                 full_reason = f"`{rule_name}` rule: {reason}"
 
diff --git a/bot/cogs/defcon.py b/bot/cogs/defcon.py
index cc0f79fe8..80dc6082f 100644
--- a/bot/cogs/defcon.py
+++ b/bot/cogs/defcon.py
@@ -104,6 +104,7 @@ class Defcon(Cog):
                     log.exception(f"Unable to send rejection message to user: {member}")
 
                 await member.kick(reason="DEFCON active, user is too new")
+                self.bot.stats.incr("defcon_leaves")
 
                 message = (
                     f"{member} (`{member.id}`) was denied entry because their account is too new."
diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py
index 6a622d2ce..747ab4a6e 100644
--- a/bot/cogs/error_handler.py
+++ b/bot/cogs/error_handler.py
@@ -236,6 +236,8 @@ class ErrorHandler(Cog):
             f"```{e.__class__.__name__}: {e}```"
         )
 
+        ctx.bot.stats.incr("command_error_count")
+
         with push_scope() as scope:
             scope.user = {
                 "id": ctx.author.id,
diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py
index 3f3dbb853..fa4420be1 100644
--- a/bot/cogs/filtering.py
+++ b/bot/cogs/filtering.py
@@ -207,6 +207,8 @@ class Filtering(Cog):
 
                         log.debug(message)
 
+                        self.bot.stats.incr(f"bot.filters.{filter_name}")
+
                         additional_embeds = None
                         additional_embeds_msg = None
 
diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py
index 697a4d3b7..389a4ad2a 100644
--- a/bot/cogs/help_channels.py
+++ b/bot/cogs/help_channels.py
@@ -367,6 +367,18 @@ class HelpChannels(Scheduler, commands.Cog):
         log.info("Cog is ready!")
         self.ready.set()
 
+        self.report_stats()
+
+    def report_stats(self) -> None:
+        """Report the channel count stats."""
+        total_in_use = len(list(self.get_category_channels(self.in_use_category)))
+        total_available = len(list(self.get_category_channels(self.available_category)))
+        total_dormant = len(list(self.get_category_channels(self.dormant_category)))
+
+        self.bot.stats.gauge("help.total.in_use", total_in_use)
+        self.bot.stats.gauge("help.total.available", total_available)
+        self.bot.stats.gauge("help.total.dormant", total_dormant)
+
     @staticmethod
     def is_dormant_message(message: t.Optional[discord.Message]) -> bool:
         """Return True if the contents of the `message` match `DORMANT_MSG`."""
@@ -432,6 +444,7 @@ class HelpChannels(Scheduler, commands.Cog):
             f"synchronized permissions after moving `{channel}` into it."
         )
         await self.ensure_permissions_synchronization(self.available_category)
+        self.report_stats()
 
     async def move_to_dormant(self, channel: discord.TextChannel) -> None:
         """Make the `channel` dormant."""
@@ -442,7 +455,6 @@ class HelpChannels(Scheduler, commands.Cog):
             category=self.dormant_category,
             sync_permissions=True,
             topic=DORMANT_TOPIC,
-            position=10000,
         )
 
         log.trace(f"Position of #{channel} ({channel.id}) is actually {channel.position}.")
@@ -453,6 +465,7 @@ class HelpChannels(Scheduler, commands.Cog):
 
         log.trace(f"Pushing #{channel} ({channel.id}) into the channel queue.")
         self.channel_queue.put_nowait(channel)
+        self.report_stats()
 
     async def move_to_in_use(self, channel: discord.TextChannel) -> None:
         """Make a channel in-use and schedule it to be made dormant."""
@@ -463,7 +476,6 @@ class HelpChannels(Scheduler, commands.Cog):
             category=self.in_use_category,
             sync_permissions=True,
             topic=IN_USE_TOPIC,
-            position=10000,
         )
 
         timeout = constants.HelpChannels.idle_minutes * 60
@@ -471,6 +483,7 @@ class HelpChannels(Scheduler, commands.Cog):
         log.trace(f"Scheduling #{channel} ({channel.id}) to become dormant in {timeout} sec.")
         data = TaskData(timeout, self.move_idle_channel(channel))
         self.schedule_task(channel.id, data)
+        self.report_stats()
 
     async def notify(self) -> None:
         """
@@ -511,6 +524,8 @@ class HelpChannels(Scheduler, commands.Cog):
                 f"using the `{constants.Bot.prefix}dormant` command within the channels."
             )
 
+            self.bot.stats.incr("help.out_of_channel_alerts")
+
             self.last_notification = message.created_at
         except Exception:
             # Handle it here cause this feature isn't critical for the functionality of the system.
@@ -543,6 +558,8 @@ class HelpChannels(Scheduler, commands.Cog):
             await self.move_to_in_use(channel)
             await self.revoke_send_permissions(message.author)
 
+            self.bot.stats.incr("help.claimed")
+
             log.trace(f"Releasing on_message lock for {message.id}.")
 
         # Move a dormant channel to the Available category to fill in the gap.
diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py
new file mode 100644
index 000000000..b75d29b7e
--- /dev/null
+++ b/bot/cogs/stats.py
@@ -0,0 +1,65 @@
+from discord import Member, Message, Status
+from discord.ext.commands import Bot, Cog, Context
+
+
+class Stats(Cog):
+    """A cog which provides a way to hook onto Discord events and forward to stats."""
+
+    def __init__(self, bot: Bot):
+        self.bot = bot
+
+    @Cog.listener()
+    async def on_message(self, message: Message) -> None:
+        """Report message events in the server to statsd."""
+        if message.guild is None:
+            return
+
+        reformatted_name = message.channel.name.replace('-', '_')
+
+        if reformatted_name.startswith("ot"):
+            # Off-topic channels change names, we don't want this for stats.
+            # This will change 'ot1-lemon-in-the-dishwasher' to just 'ot1'
+            reformatted_name = reformatted_name[:3]
+
+        stat_name = f"channels.{reformatted_name}"
+        self.bot.stats.incr(stat_name)
+
+        # Increment the total message count
+        self.bot.stats.incr("messages")
+
+    @Cog.listener()
+    async def on_command_completion(self, ctx: Context) -> None:
+        """Report completed commands to statsd."""
+        command_name = ctx.command.qualified_name.replace(" ", "_")
+
+        self.bot.stats.incr(f"commands.{command_name}")
+
+    @Cog.listener()
+    async def on_member_join(self, member: Member) -> None:
+        """Update member count stat on member join."""
+        self.bot.stats.gauge(f"guild.total_members", len(member.guild.members))
+
+    @Cog.listener()
+    async def on_member_leave(self, member: Member) -> None:
+        """Update member count stat on member leave."""
+        self.bot.stats.gauge(f"guild.total_members", len(member.guild.members))
+
+    @Cog.listener()
+    async def on_member_update(self, _before: Member, after: Member) -> None:
+        """Update presence estimates on member update."""
+        members = after.guild.members
+
+        online = len([m for m in members if m.status == Status.online])
+        idle = len([m for m in members if m.status == Status.idle])
+        dnd = len([m for m in members if m.status == Status.do_not_disturb])
+        offline = len([m for m in members if m.status == Status.offline])
+
+        self.bot.stats.gauge("guild.status.online", online)
+        self.bot.stats.gauge("guild.status.idle", idle)
+        self.bot.stats.gauge("guild.status.do_not_disturb", dnd)
+        self.bot.stats.gauge("guild.status.offline", offline)
+
+
+def setup(bot: Bot) -> None:
+    """Load the stats cog."""
+    bot.add_cog(Stats(bot))
diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py
index a6e5952ff..b81859db1 100644
--- a/bot/cogs/tags.py
+++ b/bot/cogs/tags.py
@@ -207,6 +207,9 @@ class Tags(Cog):
                         "time": time.time(),
                         "channel": ctx.channel.id
                     }
+
+                self.bot.stats.incr(f"tags.usages.{tag_name.replace('-', '_')}")
+
                 await wait_for_deletion(
                     await ctx.send(embed=Embed.from_dict(tag['embed'])),
                     [ctx.author.id],
diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py
index 421ad23e2..6721f0e02 100644
--- a/bot/cogs/token_remover.py
+++ b/bot/cogs/token_remover.py
@@ -93,6 +93,8 @@ class TokenRemover(Cog):
             channel_id=Channels.mod_alerts,
         )
 
+        self.bot.stats.incr("tokens.removed_tokens")
+
     @classmethod
     def find_token_in_message(cls, msg: Message) -> t.Optional[str]:
         """Return a seemingly valid token found in `msg` or `None` if no token is found."""
diff --git a/bot/cogs/webhook_remover.py b/bot/cogs/webhook_remover.py
index 49692113d..1b5c3f821 100644
--- a/bot/cogs/webhook_remover.py
+++ b/bot/cogs/webhook_remover.py
@@ -54,6 +54,8 @@ class WebhookRemover(Cog):
             channel_id=Channels.mod_alerts
         )
 
+        self.bot.stats.incr("tokens.removed_webhooks")
+
     @Cog.listener()
     async def on_message(self, msg: Message) -> None:
         """Check if a Discord webhook URL is in `message`."""
diff --git a/bot/constants.py b/bot/constants.py
index 60e3c4897..33c1d530d 100644
--- a/bot/constants.py
+++ b/bot/constants.py
@@ -199,6 +199,7 @@ class Bot(metaclass=YAMLGetter):
     prefix: str
     token: str
     sentry_dsn: str
+    statsd_host: str
 
 class Filter(metaclass=YAMLGetter):
     section = "filter"
diff --git a/config-default.yml b/config-default.yml
index 896003973..567caacbf 100644
--- a/config-default.yml
+++ b/config-default.yml
@@ -3,6 +3,8 @@ bot:
     token:       !ENV "BOT_TOKEN"
     sentry_dsn:  !ENV "BOT_SENTRY_DSN"
 
+    statsd_host: "graphite"
+
     cooldowns:
         # Per channel, per tag.
         tags: 60