diff options
author | 2020-04-19 09:50:24 -0700 | |
---|---|---|
committer | 2020-04-19 09:50:24 -0700 | |
commit | 80d8c7e13ecdce31917f9f0083fa17c1f97317eb (patch) | |
tree | 211b3b6f98fe8343740d15762191172a5a30f5d6 | |
parent | Tags: explicitly use UTF-8 to read files (diff) | |
parent | Merge pull request #877 from python-discord/bug/frontend/870/help-channel-dm-... (diff) |
Merge branch 'master' into bug/info/869/tag-encoding
-rw-r--r-- | Pipfile | 3 | ||||
-rw-r--r-- | Pipfile.lock | 59 | ||||
-rw-r--r-- | bot/__init__.py | 2 | ||||
-rw-r--r-- | bot/__main__.py | 3 | ||||
-rw-r--r-- | bot/async_stats.py | 39 | ||||
-rw-r--r-- | bot/bot.py | 20 | ||||
-rw-r--r-- | bot/cogs/antispam.py | 1 | ||||
-rw-r--r-- | bot/cogs/defcon.py | 17 | ||||
-rw-r--r-- | bot/cogs/error_handler.py | 14 | ||||
-rw-r--r-- | bot/cogs/filtering.py | 2 | ||||
-rw-r--r-- | bot/cogs/help_channels.py | 114 | ||||
-rw-r--r-- | bot/cogs/stats.py | 107 | ||||
-rw-r--r-- | bot/cogs/sync/syncers.py | 3 | ||||
-rw-r--r-- | bot/cogs/tags.py | 3 | ||||
-rw-r--r-- | bot/cogs/token_remover.py | 2 | ||||
-rw-r--r-- | bot/cogs/webhook_remover.py | 2 | ||||
-rw-r--r-- | bot/constants.py | 7 | ||||
-rw-r--r-- | config-default.yml | 5 | ||||
-rw-r--r-- | tests/bot/cogs/sync/test_base.py | 3 |
19 files changed, 354 insertions, 52 deletions
@@ -19,7 +19,8 @@ requests = "~=2.22" more_itertools = "~=8.2" sentry-sdk = "~=0.14" coloredlogs = "~=14.0" -colorama = {version = "~=0.4.3", sys_platform = "== 'win32'"} +colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} +statsd = "~=3.3" [dev-packages] coverage = "~=5.0" diff --git a/Pipfile.lock b/Pipfile.lock index ad9a3173a..19e03bda4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "2d3ba484e8467a115126b2ba39fa5f36f103ea455477813dd658797875c79cc9" + "sha256": "10636aef5a07f17bd00608df2cc5214fcbfe3de4745cdeea7a076b871754620a" }, "pipfile-spec": 6, "requires": { @@ -87,18 +87,18 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:05fd825eb01c290877657a56df4c6e4c311b3965bda790c613a3d6fb01a5462a", - "sha256:9fbb4d6e48ecd30bcacc5b63b94088192dcda178513b2ae3c394229f8911b887", - "sha256:e1505eeed31b0f4ce2dbb3bc8eb256c04cc2b3b72af7d551a4ab6efd5cbe5dae" + "sha256:594ca51a10d2b3443cbac41214e12dbb2a1cd57e1a7344659849e2e20ba6a8d8", + "sha256:a4bbe77fd30670455c5296242967a123ec28c37e9702a8a81bd2f20a4baf0368", + "sha256:d4e96ac9b0c3a6d3f0caae2e4124e6055c5dcafde8e2f831ff194c104f0775a0" ], - "version": "==4.8.2" + "version": "==4.9.0" }, "certifi": { "hashes": [ - "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", - "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" + "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304", + "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519" ], - "version": "==2019.11.28" + "version": "==2020.4.5.1" }, "cffi": { "hashes": [ @@ -167,10 +167,10 @@ }, "discord-py": { "hashes": [ - "sha256:7424be26b07b37ecad4404d9383d685995a0e0b3df3f9c645bdd3a4d977b83b4" + "sha256:406871b06d86c3dc49fba63238519f28628dac946fef8a0e22988ff58ec05580" ], "index": "pypi", - "version": "==1.3.2" + "version": "==1.3.3" }, "docutils": { "hashes": [ @@ -393,17 +393,10 @@ }, "pyparsing": { "hashes": [ - "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", - "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "version": "==2.4.6" - }, - "pyreadline": { - "hashes": [ - "sha256:4530592fc2e85b25b1a9f79664433da09237c1a270e4d78ea5aa3a2c7229e2d1" - ], - "markers": "sys_platform == 'win32'", - "version": "==2.1" + "version": "==2.4.7" }, "python-dateutil": { "hashes": [ @@ -524,6 +517,14 @@ ], "version": "==1.1.4" }, + "statsd": { + "hashes": [ + "sha256:c610fb80347fca0ef62666d241bce64184bd7cc1efe582f9690e045c25535eaa", + "sha256:e3e6db4c246f7c59003e51c9720a51a7f39a396541cb9b147ff4b14d15b5dd1f" + ], + "index": "pypi", + "version": "==3.3.0" + }, "urllib3": { "hashes": [ "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", @@ -717,11 +718,11 @@ }, "flake8-tidy-imports": { "hashes": [ - "sha256:5b6e75cec6d751e66534c522fbdce7dac1c2738b1216b0f6b10453995932e188", - "sha256:cf26fbb3ab31a398f265d53b6f711d80006450c19221e41b2b7b0e0b14ac39c5" + "sha256:62059ca07d8a4926b561d392cbab7f09ee042350214a25cf12823384a45d27dd", + "sha256:c30b40337a2e6802ba3bb611c26611154a27e94c53fc45639e3e282169574fd3" ], "index": "pypi", - "version": "==4.0.1" + "version": "==4.1.0" }, "flake8-todo": { "hashes": [ @@ -732,10 +733,10 @@ }, "identify": { "hashes": [ - "sha256:a7577a1f55cee1d21953a5cf11a3c839ab87f5ef909a4cba6cf52ed72b4c6059", - "sha256:ab246293e6585a1c6361a505b68d5b501a0409310932b7de2c2ead667b564d89" + "sha256:2bb8760d97d8df4408f4e805883dad26a2d076f04be92a10a3e43f09c6060742", + "sha256:faffea0fd8ec86bb146ac538ac350ed0c73908326426d387eded0bcc9d077522" ], - "version": "==1.4.13" + "version": "==1.4.14" }, "mccabe": { "hashes": [ @@ -835,10 +836,10 @@ }, "virtualenv": { "hashes": [ - "sha256:87831f1070534b636fea2241dd66f3afe37ac9041bcca6d0af3215cdcfbf7d82", - "sha256:f3128d882383c503003130389bf892856341c1da12c881ae24d6358c82561b55" + "sha256:00cfe8605fb97f5a59d52baab78e6070e72c12ca64f51151695407cc0eb8a431", + "sha256:c8364ec469084046c779c9a11ae6340094e8a0bf1d844330fc55c1cefe67c172" ], - "version": "==20.0.13" + "version": "==20.0.17" } } } diff --git a/bot/__init__.py b/bot/__init__.py index c9dbc3f40..2dd4af225 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -33,7 +33,7 @@ log_format = logging.Formatter(format_string) log_file = Path("logs", "bot.log") log_file.parent.mkdir(exist_ok=True) -file_handler = handlers.RotatingFileHandler(log_file, maxBytes=5242880, backupCount=7) +file_handler = handlers.RotatingFileHandler(log_file, maxBytes=5242880, backupCount=7, encoding="utf8") file_handler.setFormatter(log_format) root_log = logging.getLogger() diff --git a/bot/__main__.py b/bot/__main__.py index bf98f2cfd..3aa36bfc0 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -46,8 +46,8 @@ bot.load_extension("bot.cogs.verification") # Feature cogs bot.load_extension("bot.cogs.alias") bot.load_extension("bot.cogs.defcon") -bot.load_extension("bot.cogs.eval") bot.load_extension("bot.cogs.duck_pond") +bot.load_extension("bot.cogs.eval") bot.load_extension("bot.cogs.information") bot.load_extension("bot.cogs.jams") bot.load_extension("bot.cogs.moderation") @@ -56,6 +56,7 @@ bot.load_extension("bot.cogs.reddit") bot.load_extension("bot.cogs.reminders") bot.load_extension("bot.cogs.site") bot.load_extension("bot.cogs.snekbox") +bot.load_extension("bot.cogs.stats") bot.load_extension("bot.cogs.sync") bot.load_extension("bot.cogs.tags") bot.load_extension("bot.cogs.token_remover") diff --git a/bot/async_stats.py b/bot/async_stats.py new file mode 100644 index 000000000..58a80f528 --- /dev/null +++ b/bot/async_stats.py @@ -0,0 +1,39 @@ +import asyncio +import socket + +from statsd.client.base import StatsClientBase + + +class AsyncStatsClient(StatsClientBase): + """An async transport method for statsd communication.""" + + def __init__( + self, + loop: asyncio.AbstractEventLoop, + host: str = 'localhost', + port: int = 8125, + prefix: str = None + ): + """Create a new client.""" + family, _, _, _, addr = socket.getaddrinfo( + host, port, socket.AF_INET, socket.SOCK_DGRAM)[0] + self._addr = addr + self._prefix = prefix + self._loop = loop + self._transport = None + + async def create_socket(self) -> None: + """Use the loop.create_datagram_endpoint method to create a socket.""" + self._transport, _ = await self._loop.create_datagram_endpoint( + asyncio.DatagramProtocol, + family=socket.AF_INET, + remote_addr=self._addr + ) + + def _send(self, data: str) -> None: + """Start an async task to send data to statsd.""" + self._loop.create_task(self._async_send(data)) + + async def _async_send(self, data: str) -> None: + """Send data to the statsd server using the async transport.""" + self._transport.sendto(data.encode('ascii'), self._addr) diff --git a/bot/bot.py b/bot/bot.py index 950ac6751..6dd5ba896 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -8,8 +8,8 @@ import aiohttp import discord from discord.ext import commands -from bot import api -from bot import constants +from bot import DEBUG_MODE, api, constants +from bot.async_stats import AsyncStatsClient log = logging.getLogger('bot') @@ -33,6 +33,16 @@ class Bot(commands.Bot): self._resolver = None self._guild_available = asyncio.Event() + statsd_url = constants.Stats.statsd_host + + if DEBUG_MODE: + # Since statsd is UDP, there are no errors for sending to a down port. + # For this reason, setting the statsd host to 127.0.0.1 for development + # will effectively disable stats. + statsd_url = "127.0.0.1" + + self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") + def add_cog(self, cog: commands.Cog) -> None: """Adds a "cog" to the bot and logs the operation.""" super().add_cog(cog) @@ -50,7 +60,7 @@ class Bot(commands.Bot): super().clear() async def close(self) -> None: - """Close the Discord connection and the aiohttp session, connector, and resolver.""" + """Close the Discord connection and the aiohttp session, connector, statsd client, and resolver.""" await super().close() await self.api_client.close() @@ -64,9 +74,13 @@ class Bot(commands.Bot): if self._resolver: await self._resolver.close() + if self.stats._transport: + await self.stats._transport.close() + async def login(self, *args, **kwargs) -> None: """Re-create the connector and set up sessions before logging into Discord.""" self._recreate() + await self.stats.create_socket() await super().login(*args, **kwargs) def _recreate(self) -> None: diff --git a/bot/cogs/antispam.py b/bot/cogs/antispam.py index baa6b9459..d63acbc4a 100644 --- a/bot/cogs/antispam.py +++ b/bot/cogs/antispam.py @@ -182,6 +182,7 @@ class AntiSpam(Cog): # which contains the reason for why the message violated the rule and # an iterable of all members that violated the rule. if result is not None: + self.bot.stats.incr(f"mod_alerts.{rule_name}") reason, members, relevant_messages = result full_reason = f"`{rule_name}` rule: {reason}" diff --git a/bot/cogs/defcon.py b/bot/cogs/defcon.py index cc0f79fe8..56fca002a 100644 --- a/bot/cogs/defcon.py +++ b/bot/cogs/defcon.py @@ -104,6 +104,7 @@ class Defcon(Cog): log.exception(f"Unable to send rejection message to user: {member}") await member.kick(reason="DEFCON active, user is too new") + self.bot.stats.incr("defcon.leaves") message = ( f"{member} (`{member.id}`) was denied entry because their account is too new." @@ -125,6 +126,19 @@ class Defcon(Cog): async def _defcon_action(self, ctx: Context, days: int, action: Action) -> None: """Providing a structured way to do an defcon action.""" + try: + response = await self.bot.api_client.get('bot/bot-settings/defcon') + data = response['data'] + + if "enable_date" in data and action is Action.DISABLED: + enabled = datetime.fromisoformat(data["enable_date"]) + + delta = datetime.now() - enabled + + self.bot.stats.timing("defcon.enabled", delta) + except Exception: + pass + error = None try: await self.bot.api_client.put( @@ -135,6 +149,7 @@ class Defcon(Cog): # TODO: retrieve old days count 'days': days, 'enabled': action is not Action.DISABLED, + 'enable_date': datetime.now().isoformat() } } ) @@ -145,6 +160,8 @@ class Defcon(Cog): await ctx.send(self.build_defcon_msg(action, error)) await self.send_defcon_log(action, ctx.author, error) + self.bot.stats.gauge("defcon.threshold", days) + @defcon_group.command(name='enable', aliases=('on', 'e')) @with_role(Roles.admins, Roles.owners) async def enable_command(self, ctx: Context) -> None: diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index 6a622d2ce..dae283c6a 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -171,19 +171,25 @@ class ErrorHandler(Cog): if isinstance(e, errors.MissingRequiredArgument): await ctx.send(f"Missing required argument `{e.param.name}`.") await ctx.invoke(*help_command) + self.bot.stats.incr("errors.missing_required_argument") elif isinstance(e, errors.TooManyArguments): await ctx.send(f"Too many arguments provided.") await ctx.invoke(*help_command) + self.bot.stats.incr("errors.too_many_arguments") elif isinstance(e, errors.BadArgument): await ctx.send(f"Bad argument: {e}\n") await ctx.invoke(*help_command) + self.bot.stats.incr("errors.bad_argument") elif isinstance(e, errors.BadUnionArgument): await ctx.send(f"Bad argument: {e}\n```{e.errors[-1]}```") + self.bot.stats.incr("errors.bad_union_argument") elif isinstance(e, errors.ArgumentParsingError): await ctx.send(f"Argument parsing error: {e}") + self.bot.stats.incr("errors.argument_parsing_error") else: await ctx.send("Something about your input seems off. Check the arguments:") await ctx.invoke(*help_command) + self.bot.stats.incr("errors.other_user_input_error") @staticmethod async def handle_check_failure(ctx: Context, e: errors.CheckFailure) -> None: @@ -205,10 +211,12 @@ class ErrorHandler(Cog): ) if isinstance(e, bot_missing_errors): + ctx.bot.stats.incr("errors.bot_permission_error") await ctx.send( f"Sorry, it looks like I don't have the permissions or roles I need to do that." ) elif isinstance(e, (InChannelCheckFailure, errors.NoPrivateMessage)): + ctx.bot.stats.incr("errors.wrong_channel_or_dm_error") await ctx.send(e) @staticmethod @@ -217,16 +225,20 @@ class ErrorHandler(Cog): if e.status == 404: await ctx.send("There does not seem to be anything matching your query.") log.debug(f"API responded with 404 for command {ctx.command}") + ctx.bot.stats.incr("errors.api_error_404") elif e.status == 400: content = await e.response.json() log.debug(f"API responded with 400 for command {ctx.command}: %r.", content) await ctx.send("According to the API, your request is malformed.") + ctx.bot.stats.incr("errors.api_error_400") elif 500 <= e.status < 600: await ctx.send("Sorry, there seems to be an internal issue with the API.") log.warning(f"API responded with {e.status} for command {ctx.command}") + ctx.bot.stats.incr("errors.api_internal_server_error") else: await ctx.send(f"Got an unexpected status code from the API (`{e.status}`).") log.warning(f"Unexpected API response for command {ctx.command}: {e.status}") + ctx.bot.stats.incr(f"errors.api_error_{e.status}") @staticmethod async def handle_unexpected_error(ctx: Context, e: errors.CommandError) -> None: @@ -236,6 +248,8 @@ class ErrorHandler(Cog): f"```{e.__class__.__name__}: {e}```" ) + ctx.bot.stats.incr("errors.unexpected") + with push_scope() as scope: scope.user = { "id": ctx.author.id, diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 3f3dbb853..6a703f5a1 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -207,6 +207,8 @@ class Filtering(Cog): log.debug(message) + self.bot.stats.incr(f"filters.{filter_name}") + additional_embeds = None additional_embeds_msg = None diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 797019f69..e73bbdae5 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -5,6 +5,7 @@ import logging import random import typing as t from collections import deque +from contextlib import suppress from datetime import datetime from pathlib import Path @@ -13,7 +14,7 @@ from discord.ext import commands from bot import constants from bot.bot import Bot -from bot.decorators import with_role +from bot.utils.checks import with_role_check from bot.utils.scheduling import Scheduler log = logging.getLogger(__name__) @@ -108,6 +109,9 @@ class HelpChannels(Scheduler, commands.Cog): super().__init__() self.bot = bot + self.help_channel_claimants: ( + t.Dict[discord.TextChannel, t.Union[discord.Member, discord.User]] + ) = {} # Categories self.available_category: discord.CategoryChannel = None @@ -127,6 +131,9 @@ class HelpChannels(Scheduler, commands.Cog): self.on_message_lock = asyncio.Lock() self.init_task = self.bot.loop.create_task(self.init_cog()) + # Stats + self.claim_times = {} + def cog_unload(self) -> None: """Cancel the init task and scheduled tasks when the cog unloads.""" log.trace("Cog unload: cancelling the init_cog task") @@ -187,15 +194,41 @@ class HelpChannels(Scheduler, commands.Cog): log.trace("Populating the name queue with names.") return deque(available_names) + async def dormant_check(self, ctx: commands.Context) -> bool: + """Return True if the user is the help channel claimant or passes the role check.""" + if self.help_channel_claimants.get(ctx.channel) == ctx.author: + log.trace(f"{ctx.author} is the help channel claimant, passing the check for dormant.") + self.bot.stats.incr("help.dormant_invoke.claimant") + return True + + log.trace(f"{ctx.author} is not the help channel claimant, checking roles.") + role_check = with_role_check(ctx, *constants.HelpChannels.cmd_whitelist) + + if role_check: + self.bot.stats.incr("help.dormant_invoke.staff") + + return role_check + @commands.command(name="dormant", aliases=["close"], enabled=False) - @with_role(*constants.HelpChannels.cmd_whitelist) async def dormant_command(self, ctx: commands.Context) -> None: - """Make the current in-use help channel dormant.""" - log.trace("dormant command invoked; checking if the channel is in-use.") + """ + Make the current in-use help channel dormant. + Make the channel dormant if the user passes the `dormant_check`, + delete the message that invoked this, + and reset the send permissions cooldown for the user who started the session. + """ + log.trace("dormant command invoked; checking if the channel is in-use.") if ctx.channel.category == self.in_use_category: - self.cancel_task(ctx.channel.id) - await self.move_to_dormant(ctx.channel) + if await self.dormant_check(ctx): + with suppress(KeyError): + del self.help_channel_claimants[ctx.channel] + + with suppress(discord.errors.HTTPException, discord.errors.NotFound): + await self.reset_claimant_send_permission(ctx.channel) + + await self.move_to_dormant(ctx.channel, "command") + self.cancel_task(ctx.channel.id) else: log.debug(f"{ctx.author} invoked command 'dormant' outside an in-use help channel") @@ -367,14 +400,31 @@ class HelpChannels(Scheduler, commands.Cog): log.info("Cog is ready!") self.ready.set() - @staticmethod - def is_dormant_message(message: t.Optional[discord.Message]) -> bool: + self.report_stats() + + def report_stats(self) -> None: + """Report the channel count stats.""" + total_in_use = sum(1 for _ in self.get_category_channels(self.in_use_category)) + total_available = sum(1 for _ in self.get_category_channels(self.available_category)) + total_dormant = sum(1 for _ in self.get_category_channels(self.dormant_category)) + + self.bot.stats.gauge("help.total.in_use", total_in_use) + self.bot.stats.gauge("help.total.available", total_available) + self.bot.stats.gauge("help.total.dormant", total_dormant) + + def is_dormant_message(self, message: t.Optional[discord.Message]) -> bool: """Return True if the contents of the `message` match `DORMANT_MSG`.""" if not message or not message.embeds: return False embed = message.embeds[0] - return embed.description.strip() == DORMANT_MSG.strip() + return message.author == self.bot.user and embed.description.strip() == DORMANT_MSG.strip() + + @staticmethod + def is_in_category(channel: discord.TextChannel, category_id: int) -> bool: + """Return True if `channel` is within a category with `category_id`.""" + actual_category = getattr(channel, "category", None) + return actual_category is not None and actual_category.id == category_id async def move_idle_channel(self, channel: discord.TextChannel, has_task: bool = True) -> None: """ @@ -394,7 +444,7 @@ class HelpChannels(Scheduler, commands.Cog): f"and will be made dormant." ) - await self.move_to_dormant(channel) + await self.move_to_dormant(channel, "auto") else: # Cancel the existing task, if any. if has_task: @@ -432,9 +482,14 @@ class HelpChannels(Scheduler, commands.Cog): f"synchronized permissions after moving `{channel}` into it." ) await self.ensure_permissions_synchronization(self.available_category) + self.report_stats() + + async def move_to_dormant(self, channel: discord.TextChannel, caller: str) -> None: + """ + Make the `channel` dormant. - async def move_to_dormant(self, channel: discord.TextChannel) -> None: - """Make the `channel` dormant.""" + A caller argument is provided for metrics. + """ log.info(f"Moving #{channel} ({channel.id}) to the Dormant category.") await channel.edit( @@ -444,6 +499,13 @@ class HelpChannels(Scheduler, commands.Cog): topic=DORMANT_TOPIC, ) + self.bot.stats.incr(f"help.dormant_calls.{caller}") + + if channel.id in self.claim_times: + claimed = self.claim_times[channel.id] + in_use_time = datetime.now() - claimed + self.bot.stats.timing("help.in_use_time", in_use_time) + log.trace(f"Position of #{channel} ({channel.id}) is actually {channel.position}.") log.trace(f"Sending dormant message for #{channel} ({channel.id}).") @@ -452,6 +514,7 @@ class HelpChannels(Scheduler, commands.Cog): log.trace(f"Pushing #{channel} ({channel.id}) into the channel queue.") self.channel_queue.put_nowait(channel) + self.report_stats() async def move_to_in_use(self, channel: discord.TextChannel) -> None: """Make a channel in-use and schedule it to be made dormant.""" @@ -469,6 +532,7 @@ class HelpChannels(Scheduler, commands.Cog): log.trace(f"Scheduling #{channel} ({channel.id}) to become dormant in {timeout} sec.") data = TaskData(timeout, self.move_idle_channel(channel)) self.schedule_task(channel.id, data) + self.report_stats() async def notify(self) -> None: """ @@ -509,6 +573,8 @@ class HelpChannels(Scheduler, commands.Cog): f"using the `{constants.Bot.prefix}dormant` command within the channels." ) + self.bot.stats.incr("help.out_of_channel_alerts") + self.last_notification = message.created_at except Exception: # Handle it here cause this feature isn't critical for the functionality of the system. @@ -521,7 +587,7 @@ class HelpChannels(Scheduler, commands.Cog): return # Ignore messages sent by bots. channel = message.channel - if channel.category and channel.category.id != constants.Categories.help_available: + if not self.is_in_category(channel, constants.Categories.help_available): return # Ignore messages outside the Available category. log.trace("Waiting for the cog to be ready before processing messages.") @@ -531,7 +597,7 @@ class HelpChannels(Scheduler, commands.Cog): async with self.on_message_lock: log.trace(f"on_message lock acquired for {message.id}.") - if channel.category and channel.category.id != constants.Categories.help_available: + if not self.is_in_category(channel, constants.Categories.help_available): log.debug( f"Message {message.id} will not make #{channel} ({channel.id}) in-use " f"because another message in the channel already triggered that." @@ -540,6 +606,12 @@ class HelpChannels(Scheduler, commands.Cog): await self.move_to_in_use(channel) await self.revoke_send_permissions(message.author) + # Add user with channel for dormant check. + self.help_channel_claimants[channel] = message.author + + self.bot.stats.incr("help.claimed") + + self.claim_times[channel.id] = datetime.now() log.trace(f"Releasing on_message lock for {message.id}.") @@ -596,6 +668,20 @@ class HelpChannels(Scheduler, commands.Cog): log.trace(f"Ensuring channels in `Help: Available` are synchronized after permissions reset.") await self.ensure_permissions_synchronization(self.available_category) + async def reset_claimant_send_permission(self, channel: discord.TextChannel) -> None: + """Reset send permissions in the Available category for the help `channel` claimant.""" + log.trace(f"Attempting to find claimant for #{channel.name} ({channel.id}).") + try: + member = self.help_channel_claimants[channel] + except KeyError: + log.trace(f"Channel #{channel.name} ({channel.id}) not in claimant cache, permissions unchanged.") + return + + log.trace(f"Resetting send permissions for {member} ({member.id}).") + await self.update_category_permissions(self.available_category, member, overwrite=None) + # Ignore missing task when claim cooldown has passed but the channel still isn't dormant. + self.cancel_task(member.id, ignore_missing=True) + async def revoke_send_permissions(self, member: discord.Member) -> None: """ Disallow `member` to send messages in the Available category for a certain time. diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py new file mode 100644 index 000000000..d253db913 --- /dev/null +++ b/bot/cogs/stats.py @@ -0,0 +1,107 @@ +import string +from datetime import datetime + +from discord import Member, Message, Status +from discord.ext.commands import Bot, Cog, Context + +from bot.constants import Channels, Guild, Stats as StatConf + + +CHANNEL_NAME_OVERRIDES = { + Channels.off_topic_0: "off_topic_0", + Channels.off_topic_1: "off_topic_1", + Channels.off_topic_2: "off_topic_2", + Channels.staff_lounge: "staff_lounge" +} + +ALLOWED_CHARS = string.ascii_letters + string.digits + "_" + + +class Stats(Cog): + """A cog which provides a way to hook onto Discord events and forward to stats.""" + + def __init__(self, bot: Bot): + self.bot = bot + self.last_presence_update = None + + @Cog.listener() + async def on_message(self, message: Message) -> None: + """Report message events in the server to statsd.""" + if message.guild is None: + return + + if message.guild.id != Guild.id: + return + + reformatted_name = message.channel.name.replace('-', '_') + + if CHANNEL_NAME_OVERRIDES.get(message.channel.id): + reformatted_name = CHANNEL_NAME_OVERRIDES.get(message.channel.id) + + reformatted_name = "".join(char for char in reformatted_name if char in ALLOWED_CHARS) + + stat_name = f"channels.{reformatted_name}" + self.bot.stats.incr(stat_name) + + # Increment the total message count + self.bot.stats.incr("messages") + + @Cog.listener() + async def on_command_completion(self, ctx: Context) -> None: + """Report completed commands to statsd.""" + command_name = ctx.command.qualified_name.replace(" ", "_") + + self.bot.stats.incr(f"commands.{command_name}") + + @Cog.listener() + async def on_member_join(self, member: Member) -> None: + """Update member count stat on member join.""" + if member.guild.id != Guild.id: + return + + self.bot.stats.gauge(f"guild.total_members", len(member.guild.members)) + + @Cog.listener() + async def on_member_leave(self, member: Member) -> None: + """Update member count stat on member leave.""" + if member.guild.id != Guild.id: + return + + self.bot.stats.gauge(f"guild.total_members", len(member.guild.members)) + + @Cog.listener() + async def on_member_update(self, _before: Member, after: Member) -> None: + """Update presence estimates on member update.""" + if after.guild.id != Guild.id: + return + + if self.last_presence_update: + if (datetime.now() - self.last_presence_update).seconds < StatConf.presence_update_timeout: + return + + self.last_presence_update = datetime.now() + + online = 0 + idle = 0 + dnd = 0 + offline = 0 + + for member in after.guild.members: + if member.status is Status.online: + online += 1 + elif member.status is Status.dnd: + dnd += 1 + elif member.status is Status.idle: + idle += 1 + elif member.status is Status.offline: + offline += 1 + + self.bot.stats.gauge("guild.status.online", online) + self.bot.stats.gauge("guild.status.idle", idle) + self.bot.stats.gauge("guild.status.do_not_disturb", dnd) + self.bot.stats.gauge("guild.status.offline", offline) + + +def setup(bot: Bot) -> None: + """Load the stats cog.""" + bot.add_cog(Stats(bot)) diff --git a/bot/cogs/sync/syncers.py b/bot/cogs/sync/syncers.py index 003bf3727..e55bf27fd 100644 --- a/bot/cogs/sync/syncers.py +++ b/bot/cogs/sync/syncers.py @@ -1,4 +1,5 @@ import abc +import asyncio import logging import typing as t from collections import namedtuple @@ -122,7 +123,7 @@ class Syncer(abc.ABC): check=partial(self._reaction_check, author, message), timeout=constants.Sync.confirm_timeout ) - except TimeoutError: + except asyncio.TimeoutError: # reaction will remain none thus sync will be aborted in the finally block below. log.debug(f"The {self.name} syncer confirmation prompt timed out.") diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index 8705d0c61..a813ffff5 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -207,6 +207,9 @@ class Tags(Cog): "time": time.time(), "channel": ctx.channel.id } + + self.bot.stats.incr(f"tags.usages.{tag['title'].replace('-', '_')}") + await wait_for_deletion( await ctx.send(embed=Embed.from_dict(tag['embed'])), [ctx.author.id], diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index 421ad23e2..6721f0e02 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -93,6 +93,8 @@ class TokenRemover(Cog): channel_id=Channels.mod_alerts, ) + self.bot.stats.incr("tokens.removed_tokens") + @classmethod def find_token_in_message(cls, msg: Message) -> t.Optional[str]: """Return a seemingly valid token found in `msg` or `None` if no token is found.""" diff --git a/bot/cogs/webhook_remover.py b/bot/cogs/webhook_remover.py index 49692113d..1b5c3f821 100644 --- a/bot/cogs/webhook_remover.py +++ b/bot/cogs/webhook_remover.py @@ -54,6 +54,8 @@ class WebhookRemover(Cog): channel_id=Channels.mod_alerts ) + self.bot.stats.incr("tokens.removed_webhooks") + @Cog.listener() async def on_message(self, msg: Message) -> None: """Check if a Discord webhook URL is in `message`.""" diff --git a/bot/constants.py b/bot/constants.py index 60e3c4897..2add028e7 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -350,6 +350,13 @@ class CleanMessages(metaclass=YAMLGetter): message_limit: int +class Stats(metaclass=YAMLGetter): + section = "bot" + subsection = "stats" + + presence_update_timeout: int + statsd_host: str + class Categories(metaclass=YAMLGetter): section = "guild" diff --git a/config-default.yml b/config-default.yml index 896003973..f2b0bfa9f 100644 --- a/config-default.yml +++ b/config-default.yml @@ -3,6 +3,10 @@ bot: token: !ENV "BOT_TOKEN" sentry_dsn: !ENV "BOT_SENTRY_DSN" + stats: + statsd_host: "graphite" + presence_update_timeout: 300 + cooldowns: # Per channel, per tag. tags: 60 @@ -471,7 +475,6 @@ anti_malware: - '.mp3' - '.wav' - '.ogg' - - '.md' reddit: diff --git a/tests/bot/cogs/sync/test_base.py b/tests/bot/cogs/sync/test_base.py index 6ee9dfda6..70aea2bab 100644 --- a/tests/bot/cogs/sync/test_base.py +++ b/tests/bot/cogs/sync/test_base.py @@ -1,3 +1,4 @@ +import asyncio import unittest from unittest import mock @@ -211,7 +212,7 @@ class SyncerConfirmationTests(unittest.IsolatedAsyncioTestCase): subtests = ( (constants.Emojis.check_mark, True, None), ("InVaLiD", False, None), - (None, False, TimeoutError), + (None, False, asyncio.TimeoutError), ) for emoji, ret_val, side_effect in subtests: |