From f0795ea53247501cc38615f57aabe21685de7251 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Tue, 5 May 2020 02:19:03 +0200 Subject: Create utility function for uploading to paste service. --- bot/utils/__init__.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 9b32e515d..0f39a1bc8 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -1,9 +1,42 @@ +import logging from abc import ABCMeta +from typing import Optional +from aiohttp import ClientConnectorError, ClientSession from discord.ext.commands import CogMeta +from bot.constants import URLs + +log = logging.getLogger(__name__) + class CogABCMeta(CogMeta, ABCMeta): """Metaclass for ABCs meant to be implemented as Cogs.""" pass + + +async def send_to_paste_service(http_session: ClientSession, contents: str, *, extension: str = "") -> Optional[str]: + """ + Upload `contents` to the paste service. + + `http_session` should be the current running ClientSession from aiohttp + `extension` is added to the output URL + + When an error occurs, `None` is returned, otherwise the generated URL with the suffix. + """ + extension = extension and f".{extension}" + log.debug(f"Sending contents of size {len(contents.encode())} bytes to paste service.") + paste_url = URLs.paste_service.format(key="documents") + try: + async with http_session.post(paste_url, data=contents) as response: + response_json = await response.json() + except ClientConnectorError: + log.warning(f"Failed to connect to paste service at url {paste_url}.") + return + if "message" in response_json: + log.warning(f"Paste service returned error {response_json['message']} with status code {response.status}.") + return + elif "key" in response_json: + log.trace(f"Successfully uploaded contents to paste service behind key {response_json['key']}.") + return URLs.paste_service.format(key=response_json['key']) + extension -- cgit v1.2.3 From 4980726e3a68bb2bca966c9c3e09568da2162af0 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Tue, 5 May 2020 02:20:01 +0200 Subject: Attempt requests multiple times with connection errors. --- bot/utils/__init__.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 0f39a1bc8..6b9c890c8 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -9,6 +9,8 @@ from bot.constants import URLs log = logging.getLogger(__name__) +FAILED_REQUEST_ATTEMPTS = 3 + class CogABCMeta(CogMeta, ABCMeta): """Metaclass for ABCs meant to be implemented as Cogs.""" @@ -28,15 +30,18 @@ async def send_to_paste_service(http_session: ClientSession, contents: str, *, e extension = extension and f".{extension}" log.debug(f"Sending contents of size {len(contents.encode())} bytes to paste service.") paste_url = URLs.paste_service.format(key="documents") - try: - async with http_session.post(paste_url, data=contents) as response: - response_json = await response.json() - except ClientConnectorError: - log.warning(f"Failed to connect to paste service at url {paste_url}.") - return - if "message" in response_json: - log.warning(f"Paste service returned error {response_json['message']} with status code {response.status}.") - return - elif "key" in response_json: - log.trace(f"Successfully uploaded contents to paste service behind key {response_json['key']}.") - return URLs.paste_service.format(key=response_json['key']) + extension + for attempt in range(1, FAILED_REQUEST_ATTEMPTS + 1): + try: + async with http_session.post(paste_url, data=contents) as response: + response_json = await response.json() + except ClientConnectorError: + log.warning( + f"Failed to connect to paste service at url {paste_url}, " + f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." + ) + if "message" in response_json: + log.warning(f"Paste service returned error {response_json['message']} with status code {response.status}.") + return + elif "key" in response_json: + log.trace(f"Successfully uploaded contents to paste service behind key {response_json['key']}.") + return URLs.paste_service.format(key=response_json['key']) + extension -- cgit v1.2.3 From 2644316b07fdecbe834083c761ab5c7731e60a09 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Wed, 6 May 2020 02:20:57 +0200 Subject: Send long eval output to paste service. --- bot/cogs/eval.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bot/cogs/eval.py b/bot/cogs/eval.py index 52136fc8d..b739668b0 100644 --- a/bot/cogs/eval.py +++ b/bot/cogs/eval.py @@ -15,6 +15,7 @@ from bot.bot import Bot from bot.constants import Roles from bot.decorators import with_role from bot.interpreter import Interpreter +from bot.utils import send_to_paste_service log = logging.getLogger(__name__) @@ -171,6 +172,15 @@ async def func(): # (None,) -> Any res = traceback.format_exc() out, embed = self._format(code, res) + if len(out) > 1500 or out.count("\n") > 15: + paste_link = await send_to_paste_service(self.bot.http_session, out, extension="py") + await ctx.send( + f"```py\n{out[:1500]}\n```" + f"... response truncated; full contents at {paste_link}", + embed=embed + ) + return + await ctx.send(f"```py\n{out}```", embed=embed) @group(name='internal', aliases=('int',)) -- cgit v1.2.3 From 1b98e6c839a3e841115fb6c150855e673bc1ef5b Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Wed, 6 May 2020 02:43:08 +0200 Subject: Increase log level. --- bot/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 6b9c890c8..011e41227 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -43,5 +43,5 @@ async def send_to_paste_service(http_session: ClientSession, contents: str, *, e log.warning(f"Paste service returned error {response_json['message']} with status code {response.status}.") return elif "key" in response_json: - log.trace(f"Successfully uploaded contents to paste service behind key {response_json['key']}.") + log.info(f"Successfully uploaded contents to paste service behind key {response_json['key']}.") return URLs.paste_service.format(key=response_json['key']) + extension -- cgit v1.2.3 From d0d205409ccf00b14f535573b343831f31bd917c Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Wed, 6 May 2020 02:43:42 +0200 Subject: Handle failed paste uploads. --- bot/cogs/eval.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bot/cogs/eval.py b/bot/cogs/eval.py index b739668b0..c75c1e55f 100644 --- a/bot/cogs/eval.py +++ b/bot/cogs/eval.py @@ -174,9 +174,14 @@ async def func(): # (None,) -> Any out, embed = self._format(code, res) if len(out) > 1500 or out.count("\n") > 15: paste_link = await send_to_paste_service(self.bot.http_session, out, extension="py") + if paste_link is not None: + paste_text = f"full contents at {paste_link}" + else: + paste_text = "failed to upload contents to paste service." + await ctx.send( f"```py\n{out[:1500]}\n```" - f"... response truncated; full contents at {paste_link}", + f"... response truncated; {paste_text}", embed=embed ) return -- cgit v1.2.3 From 077a1ef1eb4eb07325dde5b6b625a84ccb5669ee Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Wed, 6 May 2020 02:47:15 +0200 Subject: Use new util function for uploading output. --- bot/cogs/snekbox.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 8d4688114..2aab8fdb1 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -14,6 +14,7 @@ from discord.ext.commands import Cog, Context, command, guild_only from bot.bot import Bot from bot.constants import Categories, Channels, Roles, URLs from bot.decorators import in_whitelist +from bot.utils import send_to_paste_service from bot.utils.messages import wait_for_deletion log = logging.getLogger(__name__) @@ -70,17 +71,7 @@ class Snekbox(Cog): if len(output) > MAX_PASTE_LEN: log.info("Full output is too long to upload") return "too long to upload" - - url = URLs.paste_service.format(key="documents") - try: - async with self.bot.http_session.post(url, data=output, raise_for_status=True) as resp: - data = await resp.json() - - if "key" in data: - return URLs.paste_service.format(key=data["key"]) - except Exception: - # 400 (Bad Request) means there are too many characters - log.exception("Failed to upload full output to paste service!") + return await send_to_paste_service(self.bot.http_session, output, extension="txt") @staticmethod def prepare_input(code: str) -> str: -- cgit v1.2.3 From c94c0eaef4ccb64ee3f664ed65837b1f5afd5c59 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Thu, 7 May 2020 01:43:25 +0200 Subject: Continue on failed connections. Not using skipping the iteration but continuing directly caused `response_json` being checked but not defined in case of connection errors. Co-authored-by: MarkKoz --- bot/utils/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 011e41227..41e54c3d5 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -39,6 +39,7 @@ async def send_to_paste_service(http_session: ClientSession, contents: str, *, e f"Failed to connect to paste service at url {paste_url}, " f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." ) + continue if "message" in response_json: log.warning(f"Paste service returned error {response_json['message']} with status code {response.status}.") return -- cgit v1.2.3 From 93a805d950f9daf14ba50131547d888e1f6314b3 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Thu, 7 May 2020 01:45:28 +0200 Subject: Handle broad exceptions. In the case an unexpected exception happens, this allows us to try the request again or let the function exit gracefully in the case of multiple fails. --- bot/utils/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 41e54c3d5..b9290e5a6 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -40,6 +40,13 @@ async def send_to_paste_service(http_session: ClientSession, contents: str, *, e f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." ) continue + except Exception: + log.exception( + f"An unexpected error has occurred during handling of the request, " + f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." + ) + continue + if "message" in response_json: log.warning(f"Paste service returned error {response_json['message']} with status code {response.status}.") return -- cgit v1.2.3 From 8f7551540cc2770b498bfe38a9f72c0950bbd929 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Thu, 7 May 2020 01:51:27 +0200 Subject: continue on internal server errors. In the case we receive `"message"` in the json response, the server had an internal error and we can attempt the request again. --- bot/utils/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index b9290e5a6..b273b2cde 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -48,8 +48,11 @@ async def send_to_paste_service(http_session: ClientSession, contents: str, *, e continue if "message" in response_json: - log.warning(f"Paste service returned error {response_json['message']} with status code {response.status}.") - return + log.warning( + f"Paste service returned error {response_json['message']} with status code {response.status}, " + f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." + ) + continue elif "key" in response_json: log.info(f"Successfully uploaded contents to paste service behind key {response_json['key']}.") return URLs.paste_service.format(key=response_json['key']) + extension -- cgit v1.2.3 From d98a418f9cafc8ce907293cb833cabfd68c92fb3 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Thu, 7 May 2020 01:52:44 +0200 Subject: Log unexpected JSON responses. --- bot/utils/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index b273b2cde..ec7cbd214 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -56,3 +56,7 @@ async def send_to_paste_service(http_session: ClientSession, contents: str, *, e elif "key" in response_json: log.info(f"Successfully uploaded contents to paste service behind key {response_json['key']}.") return URLs.paste_service.format(key=response_json['key']) + extension + log.warning( + f"Got unexpected JSON response from paste service: {response_json}\n" + f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." + ) -- cgit v1.2.3 From 5b11b248b945cd2a732c6d8d430d117fc062cc8d Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Thu, 7 May 2020 16:46:32 +0200 Subject: Remove tests from moved function. --- tests/bot/cogs/test_snekbox.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/bot/cogs/test_snekbox.py b/tests/bot/cogs/test_snekbox.py index 1dec0ccaf..d32d80ead 100644 --- a/tests/bot/cogs/test_snekbox.py +++ b/tests/bot/cogs/test_snekbox.py @@ -1,5 +1,4 @@ import asyncio -import logging import unittest from unittest.mock import AsyncMock, MagicMock, Mock, call, create_autospec, patch @@ -53,20 +52,6 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): raise_for_status=True ) - async def test_upload_output_gracefully_fallback_if_exception_during_request(self): - """Output upload gracefully fallback if the upload fail.""" - resp = MagicMock() - resp.json = AsyncMock(side_effect=Exception) - self.bot.http_session.post().__aenter__.return_value = resp - - log = logging.getLogger("bot.cogs.snekbox") - with self.assertLogs(logger=log, level='ERROR'): - await self.cog.upload_output('My awesome output!') - - async def test_upload_output_gracefully_fallback_if_no_key_in_response(self): - """Output upload gracefully fallback if there is no key entry in the response body.""" - self.assertEqual((await self.cog.upload_output('My awesome output!')), None) - def test_prepare_input(self): cases = ( ('print("Hello world!")', 'print("Hello world!")', 'non-formatted'), -- cgit v1.2.3 From 14c670dfa87e142e24c027e2976fa02b07c4d7ac Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Thu, 7 May 2020 17:11:56 +0200 Subject: Adjust behaviour for new func usage. --- tests/bot/cogs/test_snekbox.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/tests/bot/cogs/test_snekbox.py b/tests/bot/cogs/test_snekbox.py index d32d80ead..f4c13fc43 100644 --- a/tests/bot/cogs/test_snekbox.py +++ b/tests/bot/cogs/test_snekbox.py @@ -35,21 +35,12 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): result = await self.cog.upload_output("-" * (snekbox.MAX_PASTE_LEN + 1)) self.assertEqual(result, "too long to upload") - async def test_upload_output(self): + @patch("bot.cogs.snekbox.send_to_paste_service") + async def test_upload_output(self, mock_paste_util): """Upload the eval output to the URLs.paste_service.format(key="documents") endpoint.""" - key = "MarkDiamond" - resp = MagicMock() - resp.json = AsyncMock(return_value={"key": key}) - self.bot.http_session.post().__aenter__.return_value = resp - - self.assertEqual( - await self.cog.upload_output("My awesome output"), - constants.URLs.paste_service.format(key=key) - ) - self.bot.http_session.post.assert_called_with( - constants.URLs.paste_service.format(key="documents"), - data="My awesome output", - raise_for_status=True + await self.cog.upload_output("Test output.") + mock_paste_util.assert_called_once_with( + self.bot.http_session, "Test output.", extension="txt" ) def test_prepare_input(self): -- cgit v1.2.3 From 5d96e96a2e8982ec57c1a19d1a085ceccd35a6d7 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Fri, 8 May 2020 01:38:14 +0200 Subject: Add tests for `send_to_paste_service`. --- tests/bot/utils/test_init.py | 74 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tests/bot/utils/test_init.py diff --git a/tests/bot/utils/test_init.py b/tests/bot/utils/test_init.py new file mode 100644 index 000000000..f3a8f5939 --- /dev/null +++ b/tests/bot/utils/test_init.py @@ -0,0 +1,74 @@ +import logging +import unittest +from unittest.mock import AsyncMock, MagicMock, Mock, patch + +from aiohttp import ClientConnectorError + +from bot.utils import FAILED_REQUEST_ATTEMPTS, send_to_paste_service + + +class PasteTests(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self.http_session = MagicMock() + + @patch("bot.utils.URLs.paste_service", "https://paste_service.com/{key}") + async def test_url_and_sent_contents(self): + """Correct url was used and post was called with expected data.""" + response = MagicMock( + json=AsyncMock(return_value={"key": ""}) + ) + self.http_session.post().__aenter__.return_value = response + self.http_session.post.reset_mock() + await send_to_paste_service(self.http_session, "Content") + self.http_session.post.assert_called_once_with("https://paste_service.com/documents", data="Content") + + @patch("bot.utils.URLs.paste_service", "https://paste_service.com/{key}") + async def test_paste_returns_correct_url_on_success(self): + """Url with specified extension is returned on successful requests.""" + key = "paste_key" + test_cases = ( + (f"https://paste_service.com/{key}.txt", "txt"), + (f"https://paste_service.com/{key}.py", "py"), + (f"https://paste_service.com/{key}", ""), + ) + response = MagicMock( + json=AsyncMock(return_value={"key": key}) + ) + self.http_session.post().__aenter__.return_value = response + + for expected_output, extension in test_cases: + with self.subTest(msg=f"Send contents with extension {repr(extension)}"): + self.assertEqual( + await send_to_paste_service(self.http_session, "", extension=extension), + expected_output + ) + + async def test_request_repeated_on_json_errors(self): + """Json with error message and invalid json are handled as errors and requests repeated.""" + test_cases = ({"message": "error"}, {"unexpected_key": None}, {}) + self.http_session.post().__aenter__.return_value = response = MagicMock() + self.http_session.post.reset_mock() + + for error_json in test_cases: + with self.subTest(error_json=error_json): + response.json = AsyncMock(return_value=error_json) + result = await send_to_paste_service(self.http_session, "") + self.assertEqual(self.http_session.post.call_count, FAILED_REQUEST_ATTEMPTS) + self.assertIsNone(result) + + self.http_session.post.reset_mock() + + async def test_request_repeated_on_connection_errors(self): + """Requests are repeated in the case of connection errors.""" + self.http_session.post = MagicMock(side_effect=ClientConnectorError(Mock(), Mock())) + result = await send_to_paste_service(self.http_session, "") + self.assertEqual(self.http_session.post.call_count, FAILED_REQUEST_ATTEMPTS) + self.assertIsNone(result) + + async def test_general_error_handled_and_request_repeated(self): + """All `Exception`s are handled, logged and request repeated.""" + self.http_session.post = MagicMock(side_effect=Exception) + result = await send_to_paste_service(self.http_session, "") + self.assertEqual(self.http_session.post.call_count, FAILED_REQUEST_ATTEMPTS) + self.assertLogs("bot.utils", logging.ERROR) + self.assertIsNone(result) -- cgit v1.2.3 From 0b266169160a8368a3c7eba3fcdfb404b657232e Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Sat, 30 May 2020 02:41:05 +0200 Subject: Truncate amount of lines in int eval output to 15. Previously the amount of newlines was checked and uploaded to the paste service if above 15 but the sent message was not truncated to only include that amount of lines. --- bot/cogs/eval.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/bot/cogs/eval.py b/bot/cogs/eval.py index c75c1e55f..edb59d286 100644 --- a/bot/cogs/eval.py +++ b/bot/cogs/eval.py @@ -172,7 +172,15 @@ async def func(): # (None,) -> Any res = traceback.format_exc() out, embed = self._format(code, res) - if len(out) > 1500 or out.count("\n") > 15: + # Truncate output to max 15 lines or 1500 characters + newline_truncate_index = find_nth_occurrence(out, "\n", 15) + + if newline_truncate_index is None or newline_truncate_index > 1500: + truncate_index = 1500 + else: + truncate_index = newline_truncate_index + + if len(out) > truncate_index: paste_link = await send_to_paste_service(self.bot.http_session, out, extension="py") if paste_link is not None: paste_text = f"full contents at {paste_link}" @@ -180,7 +188,7 @@ async def func(): # (None,) -> Any paste_text = "failed to upload contents to paste service." await ctx.send( - f"```py\n{out[:1500]}\n```" + f"```py\n{out[:truncate_index]}\n```" f"... response truncated; {paste_text}", embed=embed ) @@ -212,6 +220,16 @@ async def func(): # (None,) -> Any await self._eval(ctx, code) +def find_nth_occurrence(string: str, substring: str, n: int) -> Optional[int]: + """Return index of `n`th occurrence of `substring` in `string`, or None if not found.""" + index = 0 + for _ in range(n): + index = string.find(substring, index+1) + if index == -1: + return None + return index + + def setup(bot: Bot) -> None: """Load the CodeEval cog.""" bot.add_cog(CodeEval(bot)) -- cgit v1.2.3 From c8f5f8597c8eb3cccf9cd7867fbc4777cc4b4f99 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Sat, 30 May 2020 02:42:38 +0200 Subject: Strip empty lines from int eval output. The output generates trailing newlines, which can cause the output to be uploaded to the paste service in cases where it's not needed, as discord will automatically remove those in messages. --- bot/cogs/eval.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/cogs/eval.py b/bot/cogs/eval.py index edb59d286..5b7469bdf 100644 --- a/bot/cogs/eval.py +++ b/bot/cogs/eval.py @@ -172,6 +172,8 @@ async def func(): # (None,) -> Any res = traceback.format_exc() out, embed = self._format(code, res) + out = out.rstrip("\n") # Strip empty lines from output + # Truncate output to max 15 lines or 1500 characters newline_truncate_index = find_nth_occurrence(out, "\n", 15) -- cgit v1.2.3 From 31726ecf6127f6f7dddcab3d16f4a40b8b990f6c Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Wed, 15 Jul 2020 02:30:00 +0200 Subject: Move general helper functions to submodule. --- bot/utils/__init__.py | 17 ++--------------- bot/utils/helpers.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 15 deletions(-) create mode 100644 bot/utils/helpers.py diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 2c8d57bd5..7c29a5981 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -1,25 +1,17 @@ import logging -from abc import ABCMeta from typing import Optional from aiohttp import ClientConnectorError, ClientSession -from discord.ext.commands import CogMeta from bot.constants import URLs +from bot.utils.helpers import CogABCMeta, pad_base64 from bot.utils.redis_cache import RedisCache log = logging.getLogger(__name__) FAILED_REQUEST_ATTEMPTS = 3 - -__all__ = ['RedisCache', 'CogABCMeta', "send_to_paste_service"] - - -class CogABCMeta(CogMeta, ABCMeta): - """Metaclass for ABCs meant to be implemented as Cogs.""" - - pass +__all__ = ['RedisCache', 'CogABCMeta', "pad_base64", "send_to_paste_service"] async def send_to_paste_service(http_session: ClientSession, contents: str, *, extension: str = "") -> Optional[str]: @@ -64,8 +56,3 @@ async def send_to_paste_service(http_session: ClientSession, contents: str, *, e f"Got unexpected JSON response from paste service: {response_json}\n" f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." ) - - -def pad_base64(data: str) -> str: - """Return base64 `data` with padding characters to ensure its length is a multiple of 4.""" - return data + "=" * (-len(data) % 4) diff --git a/bot/utils/helpers.py b/bot/utils/helpers.py new file mode 100644 index 000000000..cfbf47753 --- /dev/null +++ b/bot/utils/helpers.py @@ -0,0 +1,12 @@ +from abc import ABCMeta + +from discord.ext.commands import CogMeta + + +class CogABCMeta(CogMeta, ABCMeta): + """Metaclass for ABCs meant to be implemented as Cogs.""" + + +def pad_base64(data: str) -> str: + """Return base64 `data` with padding characters to ensure its length is a multiple of 4.""" + return data + "=" * (-len(data) % 4) -- cgit v1.2.3 From 326beebe9b097731a39ecc9868e5e1f2bd762aae Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Wed, 15 Jul 2020 02:33:42 +0200 Subject: Move `send_to_paste_service` to services submodule --- bot/utils/__init__.py | 55 +-------------------------------------------------- bot/utils/services.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 54 deletions(-) create mode 100644 bot/utils/services.py diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 7c29a5981..a950f3524 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -1,58 +1,5 @@ -import logging -from typing import Optional - -from aiohttp import ClientConnectorError, ClientSession - -from bot.constants import URLs from bot.utils.helpers import CogABCMeta, pad_base64 from bot.utils.redis_cache import RedisCache - -log = logging.getLogger(__name__) - -FAILED_REQUEST_ATTEMPTS = 3 +from bot.utils.services import send_to_paste_service __all__ = ['RedisCache', 'CogABCMeta', "pad_base64", "send_to_paste_service"] - - -async def send_to_paste_service(http_session: ClientSession, contents: str, *, extension: str = "") -> Optional[str]: - """ - Upload `contents` to the paste service. - - `http_session` should be the current running ClientSession from aiohttp - `extension` is added to the output URL - - When an error occurs, `None` is returned, otherwise the generated URL with the suffix. - """ - extension = extension and f".{extension}" - log.debug(f"Sending contents of size {len(contents.encode())} bytes to paste service.") - paste_url = URLs.paste_service.format(key="documents") - for attempt in range(1, FAILED_REQUEST_ATTEMPTS + 1): - try: - async with http_session.post(paste_url, data=contents) as response: - response_json = await response.json() - except ClientConnectorError: - log.warning( - f"Failed to connect to paste service at url {paste_url}, " - f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." - ) - continue - except Exception: - log.exception( - f"An unexpected error has occurred during handling of the request, " - f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." - ) - continue - - if "message" in response_json: - log.warning( - f"Paste service returned error {response_json['message']} with status code {response.status}, " - f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." - ) - continue - elif "key" in response_json: - log.info(f"Successfully uploaded contents to paste service behind key {response_json['key']}.") - return URLs.paste_service.format(key=response_json['key']) + extension - log.warning( - f"Got unexpected JSON response from paste service: {response_json}\n" - f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." - ) diff --git a/bot/utils/services.py b/bot/utils/services.py new file mode 100644 index 000000000..087b9f969 --- /dev/null +++ b/bot/utils/services.py @@ -0,0 +1,54 @@ +import logging +from typing import Optional + +from aiohttp import ClientConnectorError, ClientSession + +from bot.constants import URLs + +log = logging.getLogger(__name__) + +FAILED_REQUEST_ATTEMPTS = 3 + + +async def send_to_paste_service(http_session: ClientSession, contents: str, *, extension: str = "") -> Optional[str]: + """ + Upload `contents` to the paste service. + + `http_session` should be the current running ClientSession from aiohttp + `extension` is added to the output URL + + When an error occurs, `None` is returned, otherwise the generated URL with the suffix. + """ + extension = extension and f".{extension}" + log.debug(f"Sending contents of size {len(contents.encode())} bytes to paste service.") + paste_url = URLs.paste_service.format(key="documents") + for attempt in range(1, FAILED_REQUEST_ATTEMPTS + 1): + try: + async with http_session.post(paste_url, data=contents) as response: + response_json = await response.json() + except ClientConnectorError: + log.warning( + f"Failed to connect to paste service at url {paste_url}, " + f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." + ) + continue + except Exception: + log.exception( + f"An unexpected error has occurred during handling of the request, " + f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." + ) + continue + + if "message" in response_json: + log.warning( + f"Paste service returned error {response_json['message']} with status code {response.status}, " + f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." + ) + continue + elif "key" in response_json: + log.info(f"Successfully uploaded contents to paste service behind key {response_json['key']}.") + return URLs.paste_service.format(key=response_json['key']) + extension + log.warning( + f"Got unexpected JSON response from paste service: {response_json}\n" + f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." + ) -- cgit v1.2.3 From 2d1877cfb70304ff8d6bd24059459fa514d49e71 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Wed, 15 Jul 2020 02:37:28 +0200 Subject: Move `find_nth_occurrence` to utils helpers --- bot/cogs/eval.py | 12 +----------- bot/utils/__init__.py | 4 ++-- bot/utils/helpers.py | 11 +++++++++++ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/bot/cogs/eval.py b/bot/cogs/eval.py index 52f7ffca7..23e5998d8 100644 --- a/bot/cogs/eval.py +++ b/bot/cogs/eval.py @@ -15,7 +15,7 @@ from bot.bot import Bot from bot.constants import Roles from bot.decorators import with_role from bot.interpreter import Interpreter -from bot.utils import send_to_paste_service +from bot.utils import find_nth_occurrence, send_to_paste_service log = logging.getLogger(__name__) @@ -222,16 +222,6 @@ async def func(): # (None,) -> Any await self._eval(ctx, code) -def find_nth_occurrence(string: str, substring: str, n: int) -> Optional[int]: - """Return index of `n`th occurrence of `substring` in `string`, or None if not found.""" - index = 0 - for _ in range(n): - index = string.find(substring, index+1) - if index == -1: - return None - return index - - def setup(bot: Bot) -> None: """Load the CodeEval cog.""" bot.add_cog(CodeEval(bot)) diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index a950f3524..3e93fcb06 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -1,5 +1,5 @@ -from bot.utils.helpers import CogABCMeta, pad_base64 +from bot.utils.helpers import CogABCMeta, find_nth_occurrence, pad_base64 from bot.utils.redis_cache import RedisCache from bot.utils.services import send_to_paste_service -__all__ = ['RedisCache', 'CogABCMeta', "pad_base64", "send_to_paste_service"] +__all__ = ['RedisCache', 'CogABCMeta', 'find_nth_occurrence', 'pad_base64', 'send_to_paste_service'] diff --git a/bot/utils/helpers.py b/bot/utils/helpers.py index cfbf47753..d9b60af07 100644 --- a/bot/utils/helpers.py +++ b/bot/utils/helpers.py @@ -1,4 +1,5 @@ from abc import ABCMeta +from typing import Optional from discord.ext.commands import CogMeta @@ -7,6 +8,16 @@ class CogABCMeta(CogMeta, ABCMeta): """Metaclass for ABCs meant to be implemented as Cogs.""" +def find_nth_occurrence(string: str, substring: str, n: int) -> Optional[int]: + """Return index of `n`th occurrence of `substring` in `string`, or None if not found.""" + index = 0 + for _ in range(n): + index = string.find(substring, index+1) + if index == -1: + return None + return index + + def pad_base64(data: str) -> str: """Return base64 `data` with padding characters to ensure its length is a multiple of 4.""" return data + "=" * (-len(data) % 4) -- cgit v1.2.3 From c115dcfb72e4d4a86b66bb84a72984705a2afcd4 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Wed, 15 Jul 2020 02:45:31 +0200 Subject: Change tests to work with the new file layout. 326beebe9b097731a39ecc9868e5e1f2bd762aae --- tests/bot/utils/test_init.py | 74 ---------------------------------------- tests/bot/utils/test_services.py | 74 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 74 deletions(-) delete mode 100644 tests/bot/utils/test_init.py create mode 100644 tests/bot/utils/test_services.py diff --git a/tests/bot/utils/test_init.py b/tests/bot/utils/test_init.py deleted file mode 100644 index f3a8f5939..000000000 --- a/tests/bot/utils/test_init.py +++ /dev/null @@ -1,74 +0,0 @@ -import logging -import unittest -from unittest.mock import AsyncMock, MagicMock, Mock, patch - -from aiohttp import ClientConnectorError - -from bot.utils import FAILED_REQUEST_ATTEMPTS, send_to_paste_service - - -class PasteTests(unittest.IsolatedAsyncioTestCase): - def setUp(self) -> None: - self.http_session = MagicMock() - - @patch("bot.utils.URLs.paste_service", "https://paste_service.com/{key}") - async def test_url_and_sent_contents(self): - """Correct url was used and post was called with expected data.""" - response = MagicMock( - json=AsyncMock(return_value={"key": ""}) - ) - self.http_session.post().__aenter__.return_value = response - self.http_session.post.reset_mock() - await send_to_paste_service(self.http_session, "Content") - self.http_session.post.assert_called_once_with("https://paste_service.com/documents", data="Content") - - @patch("bot.utils.URLs.paste_service", "https://paste_service.com/{key}") - async def test_paste_returns_correct_url_on_success(self): - """Url with specified extension is returned on successful requests.""" - key = "paste_key" - test_cases = ( - (f"https://paste_service.com/{key}.txt", "txt"), - (f"https://paste_service.com/{key}.py", "py"), - (f"https://paste_service.com/{key}", ""), - ) - response = MagicMock( - json=AsyncMock(return_value={"key": key}) - ) - self.http_session.post().__aenter__.return_value = response - - for expected_output, extension in test_cases: - with self.subTest(msg=f"Send contents with extension {repr(extension)}"): - self.assertEqual( - await send_to_paste_service(self.http_session, "", extension=extension), - expected_output - ) - - async def test_request_repeated_on_json_errors(self): - """Json with error message and invalid json are handled as errors and requests repeated.""" - test_cases = ({"message": "error"}, {"unexpected_key": None}, {}) - self.http_session.post().__aenter__.return_value = response = MagicMock() - self.http_session.post.reset_mock() - - for error_json in test_cases: - with self.subTest(error_json=error_json): - response.json = AsyncMock(return_value=error_json) - result = await send_to_paste_service(self.http_session, "") - self.assertEqual(self.http_session.post.call_count, FAILED_REQUEST_ATTEMPTS) - self.assertIsNone(result) - - self.http_session.post.reset_mock() - - async def test_request_repeated_on_connection_errors(self): - """Requests are repeated in the case of connection errors.""" - self.http_session.post = MagicMock(side_effect=ClientConnectorError(Mock(), Mock())) - result = await send_to_paste_service(self.http_session, "") - self.assertEqual(self.http_session.post.call_count, FAILED_REQUEST_ATTEMPTS) - self.assertIsNone(result) - - async def test_general_error_handled_and_request_repeated(self): - """All `Exception`s are handled, logged and request repeated.""" - self.http_session.post = MagicMock(side_effect=Exception) - result = await send_to_paste_service(self.http_session, "") - self.assertEqual(self.http_session.post.call_count, FAILED_REQUEST_ATTEMPTS) - self.assertLogs("bot.utils", logging.ERROR) - self.assertIsNone(result) diff --git a/tests/bot/utils/test_services.py b/tests/bot/utils/test_services.py new file mode 100644 index 000000000..5e0855704 --- /dev/null +++ b/tests/bot/utils/test_services.py @@ -0,0 +1,74 @@ +import logging +import unittest +from unittest.mock import AsyncMock, MagicMock, Mock, patch + +from aiohttp import ClientConnectorError + +from bot.utils.services import FAILED_REQUEST_ATTEMPTS, send_to_paste_service + + +class PasteTests(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self.http_session = MagicMock() + + @patch("bot.utils.services.URLs.paste_service", "https://paste_service.com/{key}") + async def test_url_and_sent_contents(self): + """Correct url was used and post was called with expected data.""" + response = MagicMock( + json=AsyncMock(return_value={"key": ""}) + ) + self.http_session.post().__aenter__.return_value = response + self.http_session.post.reset_mock() + await send_to_paste_service(self.http_session, "Content") + self.http_session.post.assert_called_once_with("https://paste_service.com/documents", data="Content") + + @patch("bot.utils.services.URLs.paste_service", "https://paste_service.com/{key}") + async def test_paste_returns_correct_url_on_success(self): + """Url with specified extension is returned on successful requests.""" + key = "paste_key" + test_cases = ( + (f"https://paste_service.com/{key}.txt", "txt"), + (f"https://paste_service.com/{key}.py", "py"), + (f"https://paste_service.com/{key}", ""), + ) + response = MagicMock( + json=AsyncMock(return_value={"key": key}) + ) + self.http_session.post().__aenter__.return_value = response + + for expected_output, extension in test_cases: + with self.subTest(msg=f"Send contents with extension {repr(extension)}"): + self.assertEqual( + await send_to_paste_service(self.http_session, "", extension=extension), + expected_output + ) + + async def test_request_repeated_on_json_errors(self): + """Json with error message and invalid json are handled as errors and requests repeated.""" + test_cases = ({"message": "error"}, {"unexpected_key": None}, {}) + self.http_session.post().__aenter__.return_value = response = MagicMock() + self.http_session.post.reset_mock() + + for error_json in test_cases: + with self.subTest(error_json=error_json): + response.json = AsyncMock(return_value=error_json) + result = await send_to_paste_service(self.http_session, "") + self.assertEqual(self.http_session.post.call_count, FAILED_REQUEST_ATTEMPTS) + self.assertIsNone(result) + + self.http_session.post.reset_mock() + + async def test_request_repeated_on_connection_errors(self): + """Requests are repeated in the case of connection errors.""" + self.http_session.post = MagicMock(side_effect=ClientConnectorError(Mock(), Mock())) + result = await send_to_paste_service(self.http_session, "") + self.assertEqual(self.http_session.post.call_count, FAILED_REQUEST_ATTEMPTS) + self.assertIsNone(result) + + async def test_general_error_handled_and_request_repeated(self): + """All `Exception`s are handled, logged and request repeated.""" + self.http_session.post = MagicMock(side_effect=Exception) + result = await send_to_paste_service(self.http_session, "") + self.assertEqual(self.http_session.post.call_count, FAILED_REQUEST_ATTEMPTS) + self.assertLogs("bot.utils", logging.ERROR) + self.assertIsNone(result) -- cgit v1.2.3 From a2218956c881de81b41129e707091a43f6477d24 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sun, 2 Aug 2020 22:19:22 +0200 Subject: Verification: add initial on join message This message will be sent via direct message to each user who joins the guild. --- bot/cogs/verification.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index ae156cf70..2293cad28 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -12,6 +12,13 @@ from bot.utils.checks import InWhitelistCheckFailure, without_role_check log = logging.getLogger(__name__) +ON_JOIN_MESSAGE = f""" +Hello! Welcome to Python Discord! + +In order to send messages, you first have to accept our rules. To do so, please visit \ +<#{constants.Channels.verification}>. Thank you! +""" + WELCOME_MESSAGE = f""" Hello! Welcome to the server, and thanks for verifying yourself! -- cgit v1.2.3 From fad796101d5f641c0c4315244303c8727df0462f Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sun, 2 Aug 2020 22:25:18 +0200 Subject: Verification: adjust & rename welcome message Let's give it a better name so that it's clear when this message is sent. The initial words are adjusted to avoid repetition after the on join message. --- bot/cogs/verification.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 2293cad28..c10940817 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -19,8 +19,8 @@ In order to send messages, you first have to accept our rules. To do so, please <#{constants.Channels.verification}>. Thank you! """ -WELCOME_MESSAGE = f""" -Hello! Welcome to the server, and thanks for verifying yourself! +VERIFIED_MESSAGE = f""" +Thanks for verifying yourself! For your records, these are the documents you accepted: @@ -121,7 +121,7 @@ class Verification(Cog): log.debug(f"{ctx.author} called !accept. Assigning the 'Developer' role.") await ctx.author.add_roles(Object(constants.Roles.verified), reason="Accepted the rules") try: - await ctx.author.send(WELCOME_MESSAGE) + await ctx.author.send(VERIFIED_MESSAGE) except Forbidden: log.info(f"Sending welcome message failed for {ctx.author}.") finally: -- cgit v1.2.3 From 0521af684f82fec50b46f744aebf76ccee88f318 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sun, 2 Aug 2020 23:06:21 +0200 Subject: Verification: send initial message on member join --- bot/cogs/verification.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index c10940817..1c1919bdf 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -1,7 +1,7 @@ import logging from contextlib import suppress -from discord import Colour, Forbidden, Message, NotFound, Object +from discord import Colour, Forbidden, Member, Message, NotFound, Object from discord.ext.commands import Cog, Context, command from bot import constants @@ -53,6 +53,16 @@ class Verification(Cog): """Get currently loaded ModLog cog instance.""" return self.bot.get_cog("ModLog") + @Cog.listener() + async def on_member_join(self, member: Member) -> None: + """Attempt to send initial direct message to each new member.""" + if member.guild.id != constants.Guild.id: + return # Only listen for PyDis events + + log.trace(f"Sending on join message to new member: {member.id}") + with suppress(Forbidden): + await member.send(ON_JOIN_MESSAGE) + @Cog.listener() async def on_message(self, message: Message) -> None: """Check new message event for messages to the checkpoint channel & process.""" -- cgit v1.2.3 From 226af0e676e072696dba503e3873deeaf73202ac Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 18:01:47 +0200 Subject: Verification: add @Unverified role to config --- bot/constants.py | 1 + config-default.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bot/constants.py b/bot/constants.py index cf4f3f666..cce64a7c4 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -454,6 +454,7 @@ class Roles(metaclass=YAMLGetter): partners: int python_community: int team_leaders: int + unverified: int verified: int # This is the Developers role on PyDis, here named verified for readability reasons. diff --git a/config-default.yml b/config-default.yml index fc093cc32..21a3eca87 100644 --- a/config-default.yml +++ b/config-default.yml @@ -225,8 +225,8 @@ guild: partners: 323426753857191936 python_community: &PY_COMMUNITY_ROLE 458226413825294336 - # This is the Developers role on PyDis, here named verified for readability reasons - verified: 352427296948486144 + unverified: 739794855945044069 + verified: 352427296948486144 # @Developers on PyDis # Staff admins: &ADMINS_ROLE 267628507062992896 -- cgit v1.2.3 From 5006105f14e575366575c1091af7cfd2b2da7abd Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 18:08:32 +0200 Subject: Verification: refactor `discord` imports Let's access these via the qualified name. The amount of imported names was starting to get unwieldy. --- bot/cogs/verification.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 1c1919bdf..f86356f33 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -1,7 +1,7 @@ import logging from contextlib import suppress -from discord import Colour, Forbidden, Member, Message, NotFound, Object +import discord from discord.ext.commands import Cog, Context, command from bot import constants @@ -54,17 +54,17 @@ class Verification(Cog): return self.bot.get_cog("ModLog") @Cog.listener() - async def on_member_join(self, member: Member) -> None: + async def on_member_join(self, member: discord.Member) -> None: """Attempt to send initial direct message to each new member.""" if member.guild.id != constants.Guild.id: return # Only listen for PyDis events log.trace(f"Sending on join message to new member: {member.id}") - with suppress(Forbidden): + with suppress(discord.Forbidden): await member.send(ON_JOIN_MESSAGE) @Cog.listener() - async def on_message(self, message: Message) -> None: + async def on_message(self, message: discord.Message) -> None: """Check new message event for messages to the checkpoint channel & process.""" if message.channel.id != constants.Channels.verification: return # Only listen for #checkpoint messages @@ -91,7 +91,7 @@ class Verification(Cog): # Send pretty mod log embed to mod-alerts await self.mod_log.send_log_message( icon_url=constants.Icons.filtering, - colour=Colour(constants.Colours.soft_red), + colour=discord.Colour(constants.Colours.soft_red), title=f"User/Role mentioned in {message.channel.name}", text=embed_text, thumbnail=message.author.avatar_url_as(static_format="png"), @@ -120,7 +120,7 @@ class Verification(Cog): ) log.trace(f"Deleting the message posted by {ctx.author}") - with suppress(NotFound): + with suppress(discord.NotFound): await ctx.message.delete() @command(name='accept', aliases=('verify', 'verified', 'accepted'), hidden=True) @@ -129,14 +129,14 @@ class Verification(Cog): async def accept_command(self, ctx: Context, *_) -> None: # We don't actually care about the args """Accept our rules and gain access to the rest of the server.""" log.debug(f"{ctx.author} called !accept. Assigning the 'Developer' role.") - await ctx.author.add_roles(Object(constants.Roles.verified), reason="Accepted the rules") + await ctx.author.add_roles(discord.Object(constants.Roles.verified), reason="Accepted the rules") try: await ctx.author.send(VERIFIED_MESSAGE) - except Forbidden: + except discord.Forbidden: log.info(f"Sending welcome message failed for {ctx.author}.") finally: log.trace(f"Deleting accept message by {ctx.author}.") - with suppress(NotFound): + with suppress(discord.NotFound): self.mod_log.ignore(constants.Event.message_delete, ctx.message.id) await ctx.message.delete() @@ -156,7 +156,7 @@ class Verification(Cog): return log.debug(f"{ctx.author} called !subscribe. Assigning the 'Announcements' role.") - await ctx.author.add_roles(Object(constants.Roles.announcements), reason="Subscribed to announcements") + await ctx.author.add_roles(discord.Object(constants.Roles.announcements), reason="Subscribed to announcements") log.trace(f"Deleting the message posted by {ctx.author}.") @@ -180,7 +180,9 @@ class Verification(Cog): return log.debug(f"{ctx.author} called !unsubscribe. Removing the 'Announcements' role.") - await ctx.author.remove_roles(Object(constants.Roles.announcements), reason="Unsubscribed from announcements") + await ctx.author.remove_roles( + discord.Object(constants.Roles.announcements), reason="Unsubscribed from announcements" + ) log.trace(f"Deleting the message posted by {ctx.author}.") -- cgit v1.2.3 From 8415ac8da18054be3258ee7816a70c58a3a9322a Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 18:09:06 +0200 Subject: Verification: define time constants --- bot/cogs/verification.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index f86356f33..95d92899b 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -39,6 +39,9 @@ If you'd like to unsubscribe from the announcement notifications, simply send `! <#{constants.Channels.bot_commands}>. """ +UNVERIFIED_AFTER = 3 # Amount of days after which non-Developers receive the @Unverified role +KICKED_AFTER = 30 # Amount of days after which non-Developers get kicked from the guild + BOT_MESSAGE_DELETE_DELAY = 10 -- cgit v1.2.3 From f0e0dcc599f974bb563e298199fcf5a76b1cfbe7 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 18:13:37 +0200 Subject: Verification: implement `check_users` coroutine See docstring for details. The coroutine will be registered as a task at a later point. --- bot/cogs/verification.py | 60 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 95d92899b..ea4874450 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -1,5 +1,8 @@ +import asyncio import logging +import typing as t from contextlib import suppress +from datetime import datetime, timedelta import discord from discord.ext.commands import Cog, Context, command @@ -51,6 +54,61 @@ class Verification(Cog): def __init__(self, bot: Bot): self.bot = bot + async def _kick_members(self, members: t.Set[discord.Member]) -> int: + """Kick `members` from the PyDis guild.""" + ... + + async def _give_role(self, members: t.Set[discord.Member], role: discord.Role) -> int: + """Give `role` to all `members`.""" + ... + + async def check_users(self) -> None: + """ + Periodically check in on the verification status of PyDis members. + + This coroutine performs two actions: + * Find members who have not verified for `UNVERIFIED_AFTER` and give them the @Unverified role + * Find members who have not verified for `KICKED_AFTER` and kick them from the guild + + Within the body of this coroutine, we only select the members for each action. The work is then + delegated to `_kick_members` and `_give_role`. After each run, a report is sent via modlog. + """ + await self.bot.wait_until_guild_available() # Ensure cache is ready + pydis = self.bot.get_guild(constants.Guild.id) + + unverified = pydis.get_role(constants.Roles.unverified) + current_dt = datetime.utcnow() # Discord timestamps are UTC + + # Users to be given the @Unverified role, and those to be kicked, these should be entirely disjoint + for_role, for_kick = set(), set() + + log.debug("Checking verification status of guild members") + for member in pydis.members: + + # Skip all bots and users for which we don't know their join date + # This should be extremely rare, but can happen according to `joined_at` docs + if member.bot or member.joined_at is None: + continue + + # Now we check roles to determine whether this user has already verified + unverified_roles = {unverified, pydis.default_role} # Verified users have at least one more role + if set(member.roles) - unverified_roles: + continue + + # At this point, we know that `member` is an unverified user, and we will decide what + # to do with them based on time passed since their join date + since_join = current_dt - member.joined_at + + if since_join > timedelta(days=KICKED_AFTER): + for_kick.add(member) # User should be removed from the guild + + elif since_join > timedelta(days=UNVERIFIED_AFTER) and unverified not in member.roles: + for_role.add(member) # User should be given the @Unverified role + + log.debug(f"{len(for_role)} users will be given the {unverified} role, {len(for_kick)} users will be kicked") + n_kicks = await self._kick_members(for_kick) + n_roles = await self._give_role(for_role, unverified) + @property def mod_log(self) -> ModLog: """Get currently loaded ModLog cog instance.""" @@ -184,7 +242,7 @@ class Verification(Cog): log.debug(f"{ctx.author} called !unsubscribe. Removing the 'Announcements' role.") await ctx.author.remove_roles( - discord.Object(constants.Roles.announcements), reason="Unsubscribed from announcements" + discord.Object(constants.Roles.announcements), reason="Unsubscribed from announcements" ) log.trace(f"Deleting the message posted by {ctx.author}.") -- cgit v1.2.3 From 6840dbe5539cd5a094c65b2d09ddda227ad2ca30 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 18:35:17 +0200 Subject: Verification: implement `_give_role` helper --- bot/cogs/verification.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index ea4874450..683e60ddb 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -59,8 +59,27 @@ class Verification(Cog): ... async def _give_role(self, members: t.Set[discord.Member], role: discord.Role) -> int: - """Give `role` to all `members`.""" - ... + """ + Give `role` to all `members`. + + Returns the amount of successful requests. Status codes of unsuccessful requests + are logged at info level. + """ + log.info(f"Assigning {role} role to {len(members)} members (not verified after {UNVERIFIED_AFTER} days)") + n_success, bad_statuses = 0, set() + + for member in members: + try: + await member.add_roles(role, reason=f"User has not verified in {UNVERIFIED_AFTER} days") + except discord.HTTPException as http_exc: + bad_statuses.add(http_exc.status) + else: + n_success += 1 + + if bad_statuses: + log.info(f"Failed to assign {len(members) - n_success} roles due to following statuses: {bad_statuses}") + + return n_success async def check_users(self) -> None: """ -- cgit v1.2.3 From d807324eef804ec4ea002ef34063046e4fdaeca5 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 18:40:03 +0200 Subject: Verification: implement `_kick_members` helper --- bot/cogs/verification.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 683e60ddb..94c21a568 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -1,4 +1,3 @@ -import asyncio import logging import typing as t from contextlib import suppress @@ -55,8 +54,27 @@ class Verification(Cog): self.bot = bot async def _kick_members(self, members: t.Set[discord.Member]) -> int: - """Kick `members` from the PyDis guild.""" - ... + """ + Kick `members` from the PyDis guild. + + Note that this is a potentially destructive operation. Returns the amount of successful + requests. Failed requests are logged at info level. + """ + log.info(f"Kicking {len(members)} members from the guild (not verified after {KICKED_AFTER} days)") + n_kicked, bad_statuses = 0, set() + + for member in members: + try: + await member.kick(reason=f"User has not verified in {KICKED_AFTER} days") + except discord.HTTPException as http_exc: + bad_statuses.add(http_exc.status) + else: + n_kicked += 1 + + if bad_statuses: + log.info(f"Failed to kick {len(members) - n_kicked} members due to following statuses: {bad_statuses}") + + return n_kicked async def _give_role(self, members: t.Set[discord.Member], role: discord.Role) -> int: """ -- cgit v1.2.3 From 59f8ec77fa25519d1bc81052af7c2cc6460cedad Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 18:42:45 +0200 Subject: Verification: implement `_verify_kick` helper This will be used to guard the call to `_kick_members`. --- bot/cogs/verification.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 94c21a568..85a0e3ec4 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -1,3 +1,4 @@ +import asyncio import logging import typing as t from contextlib import suppress @@ -44,6 +45,11 @@ If you'd like to unsubscribe from the announcement notifications, simply send `! UNVERIFIED_AFTER = 3 # Amount of days after which non-Developers receive the @Unverified role KICKED_AFTER = 30 # Amount of days after which non-Developers get kicked from the guild +# Number in range [0, 1] determining the percentage of unverified users that are safe +# to be kicked from the guild in one batch, any larger amount will require staff confirmation, +# set this to 0 to require explicit approval for batches of any size +KICK_CONFIRMATION_THRESHOLD = 0 + BOT_MESSAGE_DELETE_DELAY = 10 @@ -53,6 +59,63 @@ class Verification(Cog): def __init__(self, bot: Bot): self.bot = bot + async def _verify_kick(self, n_members: int) -> bool: + """ + Determine whether `n_members` is a reasonable amount of members to kick. + + First, `n_members` is checked against the size of the PyDis guild. If `n_members` are + more than `KICK_CONFIRMATION_THRESHOLD` of the guild, the operation must be confirmed + by staff in #core-dev. Otherwise, the operation is seen as safe. + """ + log.debug(f"Checking whether {n_members} members are safe to kick") + + await self.bot.wait_until_guild_available() # Ensure cache is populated before we grab the guild + pydis = self.bot.get_guild(constants.Guild.id) + + percentage = n_members / len(pydis.members) + if percentage < KICK_CONFIRMATION_THRESHOLD: + log.debug(f"Kicking {percentage:.2%} of the guild's population is seen as safe") + return True + + # Since `n_members` is a suspiciously large number, we will ask for confirmation + log.debug("Amount of users is too large, requesting staff confirmation") + + core_devs = pydis.get_channel(constants.Channels.dev_core) + confirmation_msg = await core_devs.send( + f"Verification determined that `{n_members}` members should be kicked as they haven't verified in " + f"`{KICKED_AFTER}` days. This is `{percentage:.2%}` of the guild's population. Proceed?" + ) + + options = (constants.Emojis.incident_actioned, constants.Emojis.incident_unactioned) + for option in options: + await confirmation_msg.add_reaction(option) + + def check(reaction: discord.Reaction, user: discord.User) -> bool: + """Check whether `reaction` is a valid reaction to `confirmation_msg`.""" + return ( + reaction.message.id == confirmation_msg.id # Reacted to `confirmation_msg` + and str(reaction.emoji) in options # With one of `options` + and not user.bot # By a human + ) + + timeout = 60 * 5 # Seconds, i.e. 5 minutes + try: + choice, _ = await self.bot.wait_for("reaction_add", check=check, timeout=timeout) + except asyncio.TimeoutError: + log.debug("Staff prompt not answered, aborting operation") + return False + finally: + await confirmation_msg.clear_reactions() + + result = str(choice) == constants.Emojis.incident_actioned + log.debug(f"Received answer: {choice}, result: {result}") + + # Edit the prompt message to reflect the final choice + await confirmation_msg.edit( + content=f"Request to kick `{n_members}` members was {'authorized' if result else 'denied'}!" + ) + return result + async def _kick_members(self, members: t.Set[discord.Member]) -> int: """ Kick `members` from the PyDis guild. -- cgit v1.2.3 From 56f9e84b3bfa07eab4f4623e861cdefada92cdce Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 19:12:31 +0200 Subject: Verification: repurpose & rename `_check_users` Let's only use this function to check on the guild status. It can be exposed via a command in the future. Name adjusted to be more accurate w.r.t. Discord terminology. --- bot/cogs/verification.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 85a0e3ec4..4a9983ac8 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -162,16 +162,15 @@ class Verification(Cog): return n_success - async def check_users(self) -> None: + async def _check_members(self) -> t.Tuple[t.Set[discord.Member], t.Set[discord.Member]]: """ - Periodically check in on the verification status of PyDis members. + Check in on the verification status of PyDis members. - This coroutine performs two actions: - * Find members who have not verified for `UNVERIFIED_AFTER` and give them the @Unverified role - * Find members who have not verified for `KICKED_AFTER` and kick them from the guild + This coroutine finds two sets of users: + * Not verified after `UNVERIFIED_AFTER` days, should be given the @Unverified role + * Not verified after `KICKED_AFTER` days, should be kicked from the guild - Within the body of this coroutine, we only select the members for each action. The work is then - delegated to `_kick_members` and `_give_role`. After each run, a report is sent via modlog. + These sets are always disjoint, i.e. share no common members. """ await self.bot.wait_until_guild_available() # Ensure cache is ready pydis = self.bot.get_guild(constants.Guild.id) @@ -205,9 +204,8 @@ class Verification(Cog): elif since_join > timedelta(days=UNVERIFIED_AFTER) and unverified not in member.roles: for_role.add(member) # User should be given the @Unverified role - log.debug(f"{len(for_role)} users will be given the {unverified} role, {len(for_kick)} users will be kicked") - n_kicks = await self._kick_members(for_kick) - n_roles = await self._give_role(for_role, unverified) + log.debug(f"Found {len(for_role)} users for {unverified} role, {len(for_kick)} users to be kicked") + return for_role, for_kick @property def mod_log(self) -> ModLog: -- cgit v1.2.3 From 4b1100500681d1f4b670f91fe0566e4e85c8371b Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 19:33:42 +0200 Subject: Verification: create task to update unverified members --- bot/cogs/verification.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 4a9983ac8..cc8d8eb7d 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -5,6 +5,7 @@ from contextlib import suppress from datetime import datetime, timedelta import discord +from discord.ext import tasks from discord.ext.commands import Cog, Context, command from bot import constants @@ -207,6 +208,42 @@ class Verification(Cog): log.debug(f"Found {len(for_role)} users for {unverified} role, {len(for_kick)} users to be kicked") return for_role, for_kick + @tasks.loop(minutes=30) + async def update_unverified_members(self) -> None: + """ + Periodically call `_check_members` and update unverified members accordingly. + + After each run, a summary will be sent to the modlog channel. If a suspiciously high + amount of members to be kicked is found, the operation is guarded by `_verify_kick`. + """ + log.info("Updating unverified guild members") + + await self.bot.wait_until_guild_available() + unverified = self.bot.get_guild(constants.Guild.id).get_role(constants.Roles.unverified) + + for_role, for_kick = await self._check_members() + + if not for_role: + role_report = f"Found no users to be assigned the {unverified.mention} role." + else: + n_roles = await self._give_role(for_role, unverified) + role_report = f"Assigned {unverified.mention} role to `{n_roles}`/`{len(for_role)}` members." + + if not for_kick: + kick_report = "Found no users to be kicked." + elif not await self._verify_kick(len(for_kick)): + kick_report = f"Not authorized to kick `{len(for_kick)}` members." + else: + n_kicks = await self._kick_members(for_kick) + kick_report = f"Kicked `{n_kicks}`/`{len(for_kick)}` members from the guild." + + await self.mod_log.send_log_message( + icon_url=self.bot.user.avatar_url, + colour=discord.Colour.blurple(), + title="Verification system", + text=f"{kick_report}\n{role_report}", + ) + @property def mod_log(self) -> ModLog: """Get currently loaded ModLog cog instance.""" -- cgit v1.2.3 From 1305380f2552622cd51e1e22dded80ed2791af44 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 19:41:43 +0200 Subject: Verification: add region comments & move property to top Cog is getting large so let's allow collapsing related bits. --- bot/cogs/verification.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index cc8d8eb7d..951736761 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -60,6 +60,13 @@ class Verification(Cog): def __init__(self, bot: Bot): self.bot = bot + @property + def mod_log(self) -> ModLog: + """Get currently loaded ModLog cog instance.""" + return self.bot.get_cog("ModLog") + + # region: automatically update unverified users + async def _verify_kick(self, n_members: int) -> bool: """ Determine whether `n_members` is a reasonable amount of members to kick. @@ -244,10 +251,8 @@ class Verification(Cog): text=f"{kick_report}\n{role_report}", ) - @property - def mod_log(self) -> ModLog: - """Get currently loaded ModLog cog instance.""" - return self.bot.get_cog("ModLog") + # endregion + # region: listeners @Cog.listener() async def on_member_join(self, member: discord.Member) -> None: @@ -319,6 +324,9 @@ class Verification(Cog): with suppress(discord.NotFound): await ctx.message.delete() + # endregion + # region: accept and subscribe commands + @command(name='accept', aliases=('verify', 'verified', 'accepted'), hidden=True) @without_role(constants.Roles.verified) @in_whitelist(channels=(constants.Channels.verification,)) @@ -386,6 +394,9 @@ class Verification(Cog): f"{ctx.author.mention} Unsubscribed from <#{constants.Channels.announcements}> notifications." ) + # endregion + # region: miscellaneous + # This cannot be static (must have a __func__ attribute). async def cog_command_error(self, ctx: Context, error: Exception) -> None: """Check for & ignore any InWhitelistCheckFailure.""" @@ -400,6 +411,8 @@ class Verification(Cog): else: return True + # endregion + def setup(bot: Bot) -> None: """Load the Verification cog.""" -- cgit v1.2.3 From 18f2f1b8817f0209922112a0576b9b0377c2958d Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 19:44:17 +0200 Subject: Verification: schedule member update task Turns out that it's necessary to cancel the task manually. Otherwise, duplicate tasks can be running concurrently should the extension be reloaded. --- bot/cogs/verification.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 951736761..0534e8d1e 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -57,9 +57,20 @@ BOT_MESSAGE_DELETE_DELAY = 10 class Verification(Cog): """User verification and role self-management.""" - def __init__(self, bot: Bot): + def __init__(self, bot: Bot) -> None: + """Start `update_unverified_members` task.""" self.bot = bot + self.update_unverified_members.start() + + def cog_unload(self) -> None: + """ + Kill `update_unverified_members` task. + + This is necessary, the task is not automatically cancelled on cog unload. + """ + self.update_unverified_members.cancel() + @property def mod_log(self) -> ModLog: """Get currently loaded ModLog cog instance.""" -- cgit v1.2.3 From ef0e2049b64a0ba61878161d4ac7edb6015acbc2 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 20:07:24 +0200 Subject: Verification: make authorization message ping core devs --- bot/cogs/verification.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 0534e8d1e..803cb055b 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -101,8 +101,9 @@ class Verification(Cog): core_devs = pydis.get_channel(constants.Channels.dev_core) confirmation_msg = await core_devs.send( - f"Verification determined that `{n_members}` members should be kicked as they haven't verified in " - f"`{KICKED_AFTER}` days. This is `{percentage:.2%}` of the guild's population. Proceed?" + f"<@&{constants.Roles.core_developers}> Verification determined that `{n_members}` members should " + f"be kicked as they haven't verified in `{KICKED_AFTER}` days. This is `{percentage:.2%}` of the " + f"guild's population. Proceed?" ) options = (constants.Emojis.incident_actioned, constants.Emojis.incident_unactioned) -- cgit v1.2.3 From f4721ba580c5d47d0cd5fb5beb60e6af54098244 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 20:11:01 +0200 Subject: Verification: move time constants above messages Allows referencing the constants within the message bodies. --- bot/cogs/verification.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 803cb055b..7244d041d 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -16,6 +16,16 @@ from bot.utils.checks import InWhitelistCheckFailure, without_role_check log = logging.getLogger(__name__) +UNVERIFIED_AFTER = 3 # Amount of days after which non-Developers receive the @Unverified role +KICKED_AFTER = 30 # Amount of days after which non-Developers get kicked from the guild + +# Number in range [0, 1] determining the percentage of unverified users that are safe +# to be kicked from the guild in one batch, any larger amount will require staff confirmation, +# set this to 0 to require explicit approval for batches of any size +KICK_CONFIRMATION_THRESHOLD = 0 + +BOT_MESSAGE_DELETE_DELAY = 10 + ON_JOIN_MESSAGE = f""" Hello! Welcome to Python Discord! @@ -43,16 +53,6 @@ If you'd like to unsubscribe from the announcement notifications, simply send `! <#{constants.Channels.bot_commands}>. """ -UNVERIFIED_AFTER = 3 # Amount of days after which non-Developers receive the @Unverified role -KICKED_AFTER = 30 # Amount of days after which non-Developers get kicked from the guild - -# Number in range [0, 1] determining the percentage of unverified users that are safe -# to be kicked from the guild in one batch, any larger amount will require staff confirmation, -# set this to 0 to require explicit approval for batches of any size -KICK_CONFIRMATION_THRESHOLD = 0 - -BOT_MESSAGE_DELETE_DELAY = 10 - class Verification(Cog): """User verification and role self-management.""" -- cgit v1.2.3 From f101f5608dd840ae79db353b562a7c2f800533b2 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 20:12:48 +0200 Subject: Verification: add reminder ping message & frequency --- bot/cogs/verification.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 7244d041d..a01c25010 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -53,6 +53,17 @@ If you'd like to unsubscribe from the announcement notifications, simply send `! <#{constants.Channels.bot_commands}>. """ +REMINDER_MESSAGE = f""" +<@&{constants.Roles.unverified}> + +Welcome to Python Discord! Please read the documents mentioned above and type `!accept` to gain permissions \ +to send messages in the community! + +You will be kicked if you don't verify within `{KICKED_AFTER}` days. +""" + +REMINDER_FREQUENCY = 28 # Hours to wait between sending `REMINDER_MESSAGE` + class Verification(Cog): """User verification and role self-management.""" -- cgit v1.2.3 From a8a583068e0ecacd9f2279a4e24fea0f5920fb51 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 20:13:59 +0200 Subject: Verification: comment message uses --- bot/cogs/verification.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index a01c25010..3502fe5b5 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -26,6 +26,7 @@ KICK_CONFIRMATION_THRESHOLD = 0 BOT_MESSAGE_DELETE_DELAY = 10 +# Sent via DMs once user joins the guild ON_JOIN_MESSAGE = f""" Hello! Welcome to Python Discord! @@ -33,6 +34,7 @@ In order to send messages, you first have to accept our rules. To do so, please <#{constants.Channels.verification}>. Thank you! """ +# Sent via DMs once user verifies VERIFIED_MESSAGE = f""" Thanks for verifying yourself! @@ -53,6 +55,7 @@ If you'd like to unsubscribe from the announcement notifications, simply send `! <#{constants.Channels.bot_commands}>. """ +# Sent periodically in the verification channel REMINDER_MESSAGE = f""" <@&{constants.Roles.unverified}> -- cgit v1.2.3 From b1d761cecf5612d49de47c50994e12ab45b20e5e Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 20:19:00 +0200 Subject: Verification: add reminder cache --- bot/cogs/verification.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 3502fe5b5..e32224554 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -13,6 +13,7 @@ from bot.bot import Bot from bot.cogs.moderation import ModLog from bot.decorators import in_whitelist, without_role from bot.utils.checks import InWhitelistCheckFailure, without_role_check +from bot.utils.redis_cache import RedisCache log = logging.getLogger(__name__) @@ -71,6 +72,10 @@ REMINDER_FREQUENCY = 28 # Hours to wait between sending `REMINDER_MESSAGE` class Verification(Cog): """User verification and role self-management.""" + # Cache last sent `REMINDER_MESSAGE` id + # RedisCache[str, discord.Message.id] + reminder_cache = RedisCache() + def __init__(self, bot: Bot) -> None: """Start `update_unverified_members` task.""" self.bot = bot -- cgit v1.2.3 From 39d7b32b258def7f9fcf01bebb4f82013ae2de76 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 21:26:47 +0200 Subject: Verification: ignore verification reminder message event --- bot/cogs/verification.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index e32224554..ca7631db2 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -301,6 +301,9 @@ class Verification(Cog): if message.channel.id != constants.Channels.verification: return # Only listen for #checkpoint messages + if message.content == REMINDER_MESSAGE.strip(): + return # Ignore bots own verification reminder + if message.author.bot: # They're a bot, delete their message after the delay. await message.delete(delay=BOT_MESSAGE_DELETE_DELAY) -- cgit v1.2.3 From 8f548f158e17481245809801b1285b17af279fb4 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 22:12:12 +0200 Subject: Verification: implement unverified role ping task We're making good use of d.py's tasks framework. RedisCache is used to persist the reminder message ids, which can conveniently be converted into timestamps. It is therefore trivial to determine the time to sleep before the first ping. After that, the bot simply pings every n hours. --- bot/cogs/verification.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index ca7631db2..42088896d 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -7,6 +7,7 @@ from datetime import datetime, timedelta import discord from discord.ext import tasks from discord.ext.commands import Cog, Context, command +from discord.utils import snowflake_time from bot import constants from bot.bot import Bot @@ -282,6 +283,57 @@ class Verification(Cog): text=f"{kick_report}\n{role_report}", ) + # endregion + # region: periodically ping @Unverified + + @tasks.loop(hours=REMINDER_FREQUENCY) + async def ping_unverified(self) -> None: + """ + Delete latest `REMINDER_MESSAGE` and send it again. + + This utilizes RedisCache to persist the latest reminder message id. + """ + await self.bot.wait_until_guild_available() + verification = self.bot.get_guild(constants.Guild.id).get_channel(constants.Channels.verification) + + last_reminder: t.Optional[int] = await self.reminder_cache.get("last_reminder") + + if last_reminder is not None: + log.trace(f"Found verification reminder message in cache, deleting: {last_reminder}") + + with suppress(discord.HTTPException): # If something goes wrong, just ignore it + await self.bot.http.delete_message(verification.id, last_reminder) + + log.trace("Sending verification reminder") + new_reminder = await verification.send(REMINDER_MESSAGE) + + await self.reminder_cache.set("last_reminder", new_reminder.id) + + @ping_unverified.before_loop + async def _before_first_ping(self) -> None: + """ + Sleep until `REMINDER_MESSAGE` should be sent again. + + If latest reminder is not cached, exit instantly. Otherwise, wait wait until the + configured `REMINDER_FREQUENCY` has passed. + """ + last_reminder: t.Optional[int] = await self.reminder_cache.get("last_reminder") + + if last_reminder is None: + log.trace("Latest verification reminder message not cached, task will not wait") + return + + # Convert cached message id into a timestamp + time_since = datetime.utcnow() - snowflake_time(last_reminder) + log.trace(f"Time since latest verification reminder: {time_since}") + + to_sleep = timedelta(hours=REMINDER_FREQUENCY) - time_since + log.trace(f"Time to sleep until next ping: {to_sleep}") + + # Delta can be negative if `REMINDER_FREQUENCY` has already passed + secs = max(to_sleep.total_seconds(), 0) + await asyncio.sleep(secs) + # endregion # region: listeners -- cgit v1.2.3 From 04db02199c72dc0855da8ac90cb514a750dd1f22 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Tue, 4 Aug 2020 22:13:15 +0200 Subject: Verification: schedule ping task --- bot/cogs/verification.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 42088896d..5586be040 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -78,18 +78,20 @@ class Verification(Cog): reminder_cache = RedisCache() def __init__(self, bot: Bot) -> None: - """Start `update_unverified_members` task.""" + """Start internal tasks.""" self.bot = bot self.update_unverified_members.start() + self.ping_unverified.start() def cog_unload(self) -> None: """ - Kill `update_unverified_members` task. + Cancel internal tasks. - This is necessary, the task is not automatically cancelled on cog unload. + This is necessary, as tasks are not automatically cancelled on cog unload. """ self.update_unverified_members.cancel() + self.ping_unverified.cancel() @property def mod_log(self) -> ModLog: -- cgit v1.2.3 From fb042c89cd519a41f39eba1559df58fc31a97832 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Thu, 6 Aug 2020 10:53:57 +0200 Subject: Verification: remove unverified role on accept --- bot/cogs/verification.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 5586be040..5bc4f81c1 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -422,6 +422,11 @@ class Verification(Cog): """Accept our rules and gain access to the rest of the server.""" log.debug(f"{ctx.author} called !accept. Assigning the 'Developer' role.") await ctx.author.add_roles(discord.Object(constants.Roles.verified), reason="Accepted the rules") + + if constants.Roles.unverified in [role.id for role in ctx.author.roles]: + log.debug(f"Removing Unverified role from: {ctx.author}") + await ctx.author.remove_roles(discord.Object(constants.Roles.unverified)) + try: await ctx.author.send(VERIFIED_MESSAGE) except discord.Forbidden: -- cgit v1.2.3 From 78c19d2f57a41acce231d6950b45dde0fa8832c0 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Thu, 6 Aug 2020 13:04:53 +0200 Subject: Verification: add stats collection --- bot/cogs/verification.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 5bc4f81c1..64ff4d8e6 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -176,6 +176,8 @@ class Verification(Cog): else: n_kicked += 1 + self.bot.stats.incr("verification.kicked", count=n_kicked) + if bad_statuses: log.info(f"Failed to kick {len(members) - n_kicked} members due to following statuses: {bad_statuses}") @@ -415,6 +417,30 @@ class Verification(Cog): # endregion # region: accept and subscribe commands + def _bump_verified_stats(self, verified_member: discord.Member) -> None: + """ + Increment verification stats for `verified_member`. + + Each member falls into one of the three categories: + * Verified within 24 hours after joining + * Does not have @Unverified role yet + * Does have @Unverified role + + Stats for member kicking are handled separately. + """ + if verified_member.joined_at is None: # Docs mention this can happen + return + + if (datetime.utcnow() - verified_member.joined_at) < timedelta(hours=24): + category = "accepted_on_day_one" + elif constants.Roles.unverified not in [role.id for role in verified_member.roles]: + category = "accepted_before_unverified" + else: + category = "accepted_after_unverified" + + log.trace(f"Bumping verification stats in category: {category}") + self.bot.stats.incr(f"verification.{category}") + @command(name='accept', aliases=('verify', 'verified', 'accepted'), hidden=True) @without_role(constants.Roles.verified) @in_whitelist(channels=(constants.Channels.verification,)) @@ -423,6 +449,8 @@ class Verification(Cog): log.debug(f"{ctx.author} called !accept. Assigning the 'Developer' role.") await ctx.author.add_roles(discord.Object(constants.Roles.verified), reason="Accepted the rules") + self._bump_verified_stats(ctx.author) # This checks for @Unverified so make sure it's not yet removed + if constants.Roles.unverified in [role.id for role in ctx.author.roles]: log.debug(f"Removing Unverified role from: {ctx.author}") await ctx.author.remove_roles(discord.Object(constants.Roles.unverified)) -- cgit v1.2.3 From b1c800c623f90f46c4ecaff8da2269efcd04ee05 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Thu, 6 Aug 2020 13:12:06 +0200 Subject: Verification: disable burst shared filter in verification We will begin pinging users in the verification channel, prompting them to join. This can cause a surge of activity that may trigger the filter. A better solution would involve allowing per-filter channel config, but after internal discussion this is seen as unnecessary for now. --- bot/rules/burst_shared.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bot/rules/burst_shared.py b/bot/rules/burst_shared.py index bbe9271b3..0e66df69c 100644 --- a/bot/rules/burst_shared.py +++ b/bot/rules/burst_shared.py @@ -2,11 +2,20 @@ from typing import Dict, Iterable, List, Optional, Tuple from discord import Member, Message +from bot.constants import Channels + async def apply( last_message: Message, recent_messages: List[Message], config: Dict[str, int] ) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: - """Detects repeated messages sent by multiple users.""" + """ + Detects repeated messages sent by multiple users. + + This filter never triggers in the verification channel. + """ + if last_message.channel.id == Channels.verification: + return + total_recent = len(recent_messages) if total_recent > config['max']: -- cgit v1.2.3 From 0a9769c4fa8d378fb7949212e8733531e9c1591a Mon Sep 17 00:00:00 2001 From: kwzrd Date: Thu, 6 Aug 2020 13:52:57 +0200 Subject: Verification: enable role pings --- bot/cogs/verification.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 64ff4d8e6..2872e704a 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -69,6 +69,13 @@ You will be kicked if you don't verify within `{KICKED_AFTER}` days. REMINDER_FREQUENCY = 28 # Hours to wait between sending `REMINDER_MESSAGE` +MENTION_CORE_DEVS = discord.AllowedMentions( + everyone=False, roles=[discord.Object(constants.Roles.core_developers)] +) +MENTION_UNVERIFIED = discord.AllowedMentions( + everyone=False, roles=[discord.Object(constants.Roles.unverified)] +) + class Verification(Cog): """User verification and role self-management.""" @@ -125,7 +132,8 @@ class Verification(Cog): confirmation_msg = await core_devs.send( f"<@&{constants.Roles.core_developers}> Verification determined that `{n_members}` members should " f"be kicked as they haven't verified in `{KICKED_AFTER}` days. This is `{percentage:.2%}` of the " - f"guild's population. Proceed?" + f"guild's population. Proceed?", + allowed_mentions=MENTION_CORE_DEVS, ) options = (constants.Emojis.incident_actioned, constants.Emojis.incident_unactioned) @@ -309,7 +317,7 @@ class Verification(Cog): await self.bot.http.delete_message(verification.id, last_reminder) log.trace("Sending verification reminder") - new_reminder = await verification.send(REMINDER_MESSAGE) + new_reminder = await verification.send(REMINDER_MESSAGE, allowed_mentions=MENTION_UNVERIFIED) await self.reminder_cache.set("last_reminder", new_reminder.id) -- cgit v1.2.3 From c9f2a2accea4a380eccec9f14fe389e230144242 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Thu, 6 Aug 2020 14:02:27 +0200 Subject: Verification: send DM to kicked members --- bot/cogs/verification.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 2872e704a..ac488497a 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -57,6 +57,12 @@ If you'd like to unsubscribe from the announcement notifications, simply send `! <#{constants.Channels.bot_commands}>. """ +# Sent via DMs to users kicked for failing to verify +KICKED_MESSAGE = f""" +Hi! You have been automatically kicked from Python Discord as you have failed to accept our rules \ +within `{KICKED_AFTER}` days. If this was an accident, please feel free to join again. +""" + # Sent periodically in the verification channel REMINDER_MESSAGE = f""" <@&{constants.Roles.unverified}> @@ -177,6 +183,8 @@ class Verification(Cog): n_kicked, bad_statuses = 0, set() for member in members: + with suppress(discord.Forbidden): + await member.send(KICKED_MESSAGE) # Send message while user is still in guild try: await member.kick(reason=f"User has not verified in {KICKED_AFTER} days") except discord.HTTPException as http_exc: -- cgit v1.2.3 From c7eebaa1b70ad6bdd6a84bd0e980d5ea66f0002f Mon Sep 17 00:00:00 2001 From: kwzrd Date: Thu, 6 Aug 2020 14:15:47 +0200 Subject: Verification: bump confirmation threshold to 1% --- bot/cogs/verification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index ac488497a..23b61a337 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -24,7 +24,7 @@ KICKED_AFTER = 30 # Amount of days after which non-Developers get kicked from t # Number in range [0, 1] determining the percentage of unverified users that are safe # to be kicked from the guild in one batch, any larger amount will require staff confirmation, # set this to 0 to require explicit approval for batches of any size -KICK_CONFIRMATION_THRESHOLD = 0 +KICK_CONFIRMATION_THRESHOLD = 0.01 # 1% BOT_MESSAGE_DELETE_DELAY = 10 -- cgit v1.2.3 From 92cbc03204a3fde78b919fec9d5f17144d24bd83 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Thu, 6 Aug 2020 14:38:57 +0200 Subject: Verification: make on-join message more accurate It now explains that new users can only see a limited amount of public channels, and that there will be more once they verify. Co-authored-by: Sebastiaan Zeeff --- bot/cogs/verification.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 23b61a337..ff4b358c7 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -32,8 +32,10 @@ BOT_MESSAGE_DELETE_DELAY = 10 ON_JOIN_MESSAGE = f""" Hello! Welcome to Python Discord! -In order to send messages, you first have to accept our rules. To do so, please visit \ -<#{constants.Channels.verification}>. Thank you! +As a new user, you have read-only access to a few select channels to give you a taste of what our server is like. + +In order to see the rest of the channels and to send messages, you first have to accept our rules. To do so, \ +please visit <#{constants.Channels.verification}>. Thank you! """ # Sent via DMs once user verifies -- cgit v1.2.3 From 16eec3d2d69af5178b03fb574b0f277dbcf1dea8 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Thu, 6 Aug 2020 18:17:38 +0200 Subject: Verification: extend cog docstring --- bot/cogs/verification.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index ff4b358c7..963a2369e 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -86,7 +86,23 @@ MENTION_UNVERIFIED = discord.AllowedMentions( class Verification(Cog): - """User verification and role self-management.""" + """ + User verification and role management. + + There are two internal tasks in this cog: + + * `update_unverified_members` + * Unverified members are given the @Unverified role after `UNVERIFIED_AFTER` days + * Unverified members are kicked after `UNVERIFIED_AFTER` days + + * `ping_unverified` + * Periodically ping the @Unverified role in the verification channel + + Statistics are collected in the 'verification.' namespace. + + Additionally, this cog offers the !accept, !subscribe and !unsubscribe commands, + and keeps the verification channel clean by deleting messages. + """ # Cache last sent `REMINDER_MESSAGE` id # RedisCache[str, discord.Message.id] -- cgit v1.2.3 From fe50d6457081e0e6ef86d821bcfab81a8a164ca5 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Thu, 6 Aug 2020 23:45:53 +0200 Subject: Verification: add command interface for task management Allow checking whether tasks are running, starting them, and stopping them. Currently, the tasks cannot be started or stopped separately. It is not believed that we would need such a level of granularity. Calling `cancel` on a task that isn't running is a no-op. --- bot/cogs/verification.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 963a2369e..152118d92 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -6,13 +6,13 @@ from datetime import datetime, timedelta import discord from discord.ext import tasks -from discord.ext.commands import Cog, Context, command +from discord.ext.commands import Cog, Context, command, group from discord.utils import snowflake_time from bot import constants from bot.bot import Bot from bot.cogs.moderation import ModLog -from bot.decorators import in_whitelist, without_role +from bot.decorators import in_whitelist, with_role, without_role from bot.utils.checks import InWhitelistCheckFailure, without_role_check from bot.utils.redis_cache import RedisCache @@ -448,6 +448,64 @@ class Verification(Cog): with suppress(discord.NotFound): await ctx.message.delete() + # endregion + # region: task management commands + + @with_role(*constants.MODERATION_ROLES) + @group(name="verification") + async def verification_group(self, ctx: Context) -> None: + """Manage internal verification tasks.""" + if ctx.invoked_subcommand is None: + await ctx.send_help(ctx.command) + + @verification_group.command(name="status") + async def status_cmd(self, ctx: Context) -> None: + """Check whether verification tasks are running.""" + log.trace("Checking status of verification tasks") + + if self.update_unverified_members.is_running(): + update_status = f"{constants.Emojis.incident_actioned} Member update task is running." + else: + update_status = f"{constants.Emojis.incident_unactioned} Member update task is **not** running." + + mention = f"<@&{constants.Roles.unverified}>" + if self.ping_unverified.is_running(): + ping_status = f"{constants.Emojis.incident_actioned} Ping {mention} is running." + else: + ping_status = f"{constants.Emojis.incident_unactioned} Ping {mention} is **not** running." + + embed = discord.Embed( + title="Verification system", + description=f"{update_status}\n{ping_status}", + colour=discord.Colour.blurple(), + ) + await ctx.send(embed=embed) + + @verification_group.command(name="start") + async def start_cmd(self, ctx: Context) -> None: + """Start verification tasks if they are not already running.""" + log.info("Starting verification tasks") + + if not self.update_unverified_members.is_running(): + self.update_unverified_members.start() + + if not self.ping_unverified.is_running(): + self.ping_unverified.start() + + colour = discord.Colour.blurple() + await ctx.send(embed=discord.Embed(title="Verification system", description="Done. :ok_hand:", colour=colour)) + + @verification_group.command(name="stop", aliases=["kill"]) + async def stop_cmd(self, ctx: Context) -> None: + """Stop verification tasks.""" + log.info("Stopping verification tasks") + + self.update_unverified_members.cancel() + self.ping_unverified.cancel() + + colour = discord.Colour.blurple() + await ctx.send(embed=discord.Embed(title="Verification system", description="Tasks canceled.", colour=colour)) + # endregion # region: accept and subscribe commands -- cgit v1.2.3 From 2d24e4730e0f9678c0d7833c2332c7f0821eb7e2 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Thu, 6 Aug 2020 23:48:06 +0200 Subject: Verification: persist task settings in Redis If tasks are stopped manually, they will not automatically restart on cog reload or bot restart. Using `maybe_start_tasks` is necessary because we cannot interface with Redis from a sync context. We're using 1 and 0 because RedisCache does not currently permit bool values due to a typestring conversion bug. --- bot/cogs/verification.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 152118d92..b4dc1f145 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -111,9 +111,7 @@ class Verification(Cog): def __init__(self, bot: Bot) -> None: """Start internal tasks.""" self.bot = bot - - self.update_unverified_members.start() - self.ping_unverified.start() + self.bot.loop.create_task(self.maybe_start_tasks()) def cog_unload(self) -> None: """ @@ -129,6 +127,20 @@ class Verification(Cog): """Get currently loaded ModLog cog instance.""" return self.bot.get_cog("ModLog") + async def maybe_start_tasks(self) -> None: + """ + Poll Redis to check whether internal tasks should start. + + Redis must be interfaced with from an async function. + """ + log.trace("Checking whether background tasks should begin") + setting: t.Optional[int] = await self.reminder_cache.get("tasks_running") # This can be None if never set + + if setting: + log.trace("Background tasks will be started") + self.update_unverified_members.start() + self.ping_unverified.start() + # region: automatically update unverified users async def _verify_kick(self, n_members: int) -> bool: @@ -492,6 +504,8 @@ class Verification(Cog): if not self.ping_unverified.is_running(): self.ping_unverified.start() + await self.reminder_cache.set("tasks_running", 1) + colour = discord.Colour.blurple() await ctx.send(embed=discord.Embed(title="Verification system", description="Done. :ok_hand:", colour=colour)) @@ -503,6 +517,8 @@ class Verification(Cog): self.update_unverified_members.cancel() self.ping_unverified.cancel() + await self.reminder_cache.set("tasks_running", 0) + colour = discord.Colour.blurple() await ctx.send(embed=discord.Embed(title="Verification system", description="Tasks canceled.", colour=colour)) -- cgit v1.2.3 From 80063705dc2264c1a320100f3620b5a384780699 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Fri, 7 Aug 2020 10:21:28 +0200 Subject: Verification: rename cache & document new use --- bot/cogs/verification.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index b4dc1f145..6b245d574 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -100,13 +100,19 @@ class Verification(Cog): Statistics are collected in the 'verification.' namespace. + Moderators+ can use the `verification` command group to start or stop both internal + tasks, if necessary. Settings are persisted in Redis across sessions. + Additionally, this cog offers the !accept, !subscribe and !unsubscribe commands, and keeps the verification channel clean by deleting messages. """ - # Cache last sent `REMINDER_MESSAGE` id - # RedisCache[str, discord.Message.id] - reminder_cache = RedisCache() + # Persist task settings & last sent `REMINDER_MESSAGE` id + # RedisCache[ + # "tasks_running": int (0 or 1), + # "last_reminder": int (discord.Message.id), + # ] + task_cache = RedisCache() def __init__(self, bot: Bot) -> None: """Start internal tasks.""" @@ -134,7 +140,7 @@ class Verification(Cog): Redis must be interfaced with from an async function. """ log.trace("Checking whether background tasks should begin") - setting: t.Optional[int] = await self.reminder_cache.get("tasks_running") # This can be None if never set + setting: t.Optional[int] = await self.task_cache.get("tasks_running") # This can be None if never set if setting: log.trace("Background tasks will be started") @@ -346,7 +352,7 @@ class Verification(Cog): await self.bot.wait_until_guild_available() verification = self.bot.get_guild(constants.Guild.id).get_channel(constants.Channels.verification) - last_reminder: t.Optional[int] = await self.reminder_cache.get("last_reminder") + last_reminder: t.Optional[int] = await self.task_cache.get("last_reminder") if last_reminder is not None: log.trace(f"Found verification reminder message in cache, deleting: {last_reminder}") @@ -357,7 +363,7 @@ class Verification(Cog): log.trace("Sending verification reminder") new_reminder = await verification.send(REMINDER_MESSAGE, allowed_mentions=MENTION_UNVERIFIED) - await self.reminder_cache.set("last_reminder", new_reminder.id) + await self.task_cache.set("last_reminder", new_reminder.id) @ping_unverified.before_loop async def _before_first_ping(self) -> None: @@ -367,7 +373,7 @@ class Verification(Cog): If latest reminder is not cached, exit instantly. Otherwise, wait wait until the configured `REMINDER_FREQUENCY` has passed. """ - last_reminder: t.Optional[int] = await self.reminder_cache.get("last_reminder") + last_reminder: t.Optional[int] = await self.task_cache.get("last_reminder") if last_reminder is None: log.trace("Latest verification reminder message not cached, task will not wait") @@ -504,7 +510,7 @@ class Verification(Cog): if not self.ping_unverified.is_running(): self.ping_unverified.start() - await self.reminder_cache.set("tasks_running", 1) + await self.task_cache.set("tasks_running", 1) colour = discord.Colour.blurple() await ctx.send(embed=discord.Embed(title="Verification system", description="Done. :ok_hand:", colour=colour)) @@ -517,7 +523,7 @@ class Verification(Cog): self.update_unverified_members.cancel() self.ping_unverified.cancel() - await self.reminder_cache.set("tasks_running", 0) + await self.task_cache.set("tasks_running", 0) colour = discord.Colour.blurple() await ctx.send(embed=discord.Embed(title="Verification system", description="Tasks canceled.", colour=colour)) -- cgit v1.2.3 From 6be9f0d24caa792b23d8f93ce9d87e48df3e92a5 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Fri, 7 Aug 2020 15:22:56 +0200 Subject: Verification: address member update race condition In an edge case, the `_kick_members` and `_give_role` could act on a member who has verified *after* being marked by `_check_members` as unverified. To address this, we perform one additional check just before sending the request. Testing seems to indicate that the `discord.Member` instance get updates as appropriate, so this should at least reduce the chances of such a race happening to very close to nil. --- bot/cogs/verification.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 6b245d574..ed03b0a14 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -85,6 +85,20 @@ MENTION_UNVERIFIED = discord.AllowedMentions( ) +def is_verified(member: discord.Member) -> bool: + """ + Check whether `member` is considered verified. + + Members are considered verified if they have at least 1 role other than + the default role (@everyone) and the @Unverified role. + """ + unverified_roles = { + member.guild.get_role(constants.Roles.unverified), + member.guild.default_role, + } + return bool(set(member.roles) - unverified_roles) + + class Verification(Cog): """ User verification and role management. @@ -219,6 +233,8 @@ class Verification(Cog): n_kicked, bad_statuses = 0, set() for member in members: + if is_verified(member): # Member could have verified in the meantime + continue with suppress(discord.Forbidden): await member.send(KICKED_MESSAGE) # Send message while user is still in guild try: @@ -246,6 +262,8 @@ class Verification(Cog): n_success, bad_statuses = 0, set() for member in members: + if is_verified(member): # Member could have verified in the meantime + continue try: await member.add_roles(role, reason=f"User has not verified in {UNVERIFIED_AFTER} days") except discord.HTTPException as http_exc: @@ -280,14 +298,9 @@ class Verification(Cog): log.debug("Checking verification status of guild members") for member in pydis.members: - # Skip all bots and users for which we don't know their join date - # This should be extremely rare, but can happen according to `joined_at` docs - if member.bot or member.joined_at is None: - continue - - # Now we check roles to determine whether this user has already verified - unverified_roles = {unverified, pydis.default_role} # Verified users have at least one more role - if set(member.roles) - unverified_roles: + # Skip verified members, bots, and members for which we do not know their join date, + # this should be extremely rare but docs mention that it can happen + if is_verified(member) or member.bot or member.joined_at is None: continue # At this point, we know that `member` is an unverified user, and we will decide what -- cgit v1.2.3 From 553b6a9118dc21634d0fd78fdb58f98cb02c3c7f Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sat, 8 Aug 2020 11:02:51 +0200 Subject: Verification: improve `is_verified` check This just reads better. Co-authored-by: MarkKoz --- bot/cogs/verification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index ed03b0a14..d4064cff7 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -96,7 +96,7 @@ def is_verified(member: discord.Member) -> bool: member.guild.get_role(constants.Roles.unverified), member.guild.default_role, } - return bool(set(member.roles) - unverified_roles) + return len(set(member.roles) - unverified_roles) > 0 class Verification(Cog): -- cgit v1.2.3 From f3c16f77d812be50c2b7bed4c046cd67f3b9b761 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sat, 8 Aug 2020 11:04:24 +0200 Subject: Verification: only take reactions from core devs Co-authored-by: MarkKoz --- bot/cogs/verification.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index d4064cff7..da2f81e2d 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -196,12 +196,14 @@ class Verification(Cog): for option in options: await confirmation_msg.add_reaction(option) + core_dev_ids = [member.id for member in pydis.get_role(constants.Roles.core_developers).members] + def check(reaction: discord.Reaction, user: discord.User) -> bool: """Check whether `reaction` is a valid reaction to `confirmation_msg`.""" return ( reaction.message.id == confirmation_msg.id # Reacted to `confirmation_msg` and str(reaction.emoji) in options # With one of `options` - and not user.bot # By a human + and user.id in core_dev_ids # By a core developer ) timeout = 60 * 5 # Seconds, i.e. 5 minutes -- cgit v1.2.3 From 9b91847950a31f094c92a77974edc19d7766f514 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sat, 8 Aug 2020 11:05:57 +0200 Subject: Verification: widen set type annotation Co-authored-by: MarkKoz --- bot/cogs/verification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index da2f81e2d..9dc65da1c 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -224,7 +224,7 @@ class Verification(Cog): ) return result - async def _kick_members(self, members: t.Set[discord.Member]) -> int: + async def _kick_members(self, members: t.Collection[discord.Member]) -> int: """ Kick `members` from the PyDis guild. @@ -253,7 +253,7 @@ class Verification(Cog): return n_kicked - async def _give_role(self, members: t.Set[discord.Member], role: discord.Role) -> int: + async def _give_role(self, members: t.Collection[discord.Member], role: discord.Role) -> int: """ Give `role` to all `members`. -- cgit v1.2.3 From 174796a9bf8fcb117f38e8d6dc1a4b17c3849334 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sat, 8 Aug 2020 11:50:50 +0200 Subject: Verification: strip reminder message once and for all Co-authored-by: MarkKoz --- bot/cogs/verification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 9dc65da1c..a22b91e5d 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -73,7 +73,7 @@ Welcome to Python Discord! Please read the documents mentioned above and type `! to send messages in the community! You will be kicked if you don't verify within `{KICKED_AFTER}` days. -""" +""".strip() REMINDER_FREQUENCY = 28 # Hours to wait between sending `REMINDER_MESSAGE` @@ -424,7 +424,7 @@ class Verification(Cog): if message.channel.id != constants.Channels.verification: return # Only listen for #checkpoint messages - if message.content == REMINDER_MESSAGE.strip(): + if message.content == REMINDER_MESSAGE: return # Ignore bots own verification reminder if message.author.bot: -- cgit v1.2.3 From 286cdccb21ed035d697128c2212d88368cb48e8d Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sat, 8 Aug 2020 13:56:24 +0200 Subject: Verification: improve confirmation message handling Suppress errors coming from Discord when changing the confirmation message in case it gets deleted, or something else goes wrong. This commit also adds either the ok hand or the warning emoji to the edited message content, as with the guild syncer confirmation. Co-authored-by: MarkKoz --- bot/cogs/verification.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index a22b91e5d..cbf2c51c3 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -213,15 +213,21 @@ class Verification(Cog): log.debug("Staff prompt not answered, aborting operation") return False finally: - await confirmation_msg.clear_reactions() + with suppress(discord.HTTPException): + await confirmation_msg.clear_reactions() result = str(choice) == constants.Emojis.incident_actioned log.debug(f"Received answer: {choice}, result: {result}") # Edit the prompt message to reflect the final choice - await confirmation_msg.edit( - content=f"Request to kick `{n_members}` members was {'authorized' if result else 'denied'}!" - ) + if result is True: + result_msg = f":ok_hand: Request to kick `{n_members}` members was authorized!" + else: + result_msg = f":warning: Request to kick `{n_members}` members was denied!" + + with suppress(discord.HTTPException): + await confirmation_msg.edit(content=result_msg) + return result async def _kick_members(self, members: t.Collection[discord.Member]) -> int: -- cgit v1.2.3 From 1958978e71dc5bd9e4ae007091db72de147afc12 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Thu, 13 Aug 2020 18:49:47 +0200 Subject: Verification: add `_send_requests` helper Generic request dispatch method to avoid code duplication with error handling & bad status logging. --- bot/cogs/verification.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index cbf2c51c3..e89f491cf 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -84,6 +84,9 @@ MENTION_UNVERIFIED = discord.AllowedMentions( everyone=False, roles=[discord.Object(constants.Roles.unverified)] ) +# An async function taking a Member param +Request = t.Callable[[discord.Member], t.Awaitable] + def is_verified(member: discord.Member) -> bool: """ @@ -230,6 +233,33 @@ class Verification(Cog): return result + async def _send_requests(self, members: t.Collection[discord.Member], request: Request) -> int: + """ + Pass `members` one by one to `request` handling Discord exceptions. + + This coroutine serves as a generic `request` executor for kicking members and adding + roles, as it allows us to define the error handling logic in one place only. + + Returns the amount of successful requests. Failed requests are logged at info level. + """ + log.info(f"Sending {len(members)} requests") + n_success, bad_statuses = 0, set() + + for member in members: + if is_verified(member): # Member could have verified in the meantime + continue + try: + await request(member) + except discord.HTTPException as http_exc: + bad_statuses.add(http_exc.status) + else: + n_success += 1 + + if bad_statuses: + log.info(f"Failed to send {len(members) - n_success} requests due to following statuses: {bad_statuses}") + + return n_success + async def _kick_members(self, members: t.Collection[discord.Member]) -> int: """ Kick `members` from the PyDis guild. -- cgit v1.2.3 From dc70a018bfbdb5546c05af0a60e0c58dad5e4de1 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Thu, 13 Aug 2020 18:51:12 +0200 Subject: Verification: adjust coroutines to use generic dispatch --- bot/cogs/verification.py | 43 +++++++++++-------------------------------- 1 file changed, 11 insertions(+), 32 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index e89f491cf..8f1a773a8 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -264,55 +264,34 @@ class Verification(Cog): """ Kick `members` from the PyDis guild. - Note that this is a potentially destructive operation. Returns the amount of successful - requests. Failed requests are logged at info level. + Note that this is a potentially destructive operation. Returns the amount of successful requests. """ log.info(f"Kicking {len(members)} members from the guild (not verified after {KICKED_AFTER} days)") - n_kicked, bad_statuses = 0, set() - for member in members: - if is_verified(member): # Member could have verified in the meantime - continue + async def kick_request(member: discord.Member) -> None: + """Send `KICKED_MESSAGE` to `member` and kick them from the guild.""" with suppress(discord.Forbidden): - await member.send(KICKED_MESSAGE) # Send message while user is still in guild - try: - await member.kick(reason=f"User has not verified in {KICKED_AFTER} days") - except discord.HTTPException as http_exc: - bad_statuses.add(http_exc.status) - else: - n_kicked += 1 + await member.send(KICKED_MESSAGE) + await member.kick(reason=f"User has not verified in {KICKED_AFTER} days") + n_kicked = await self._send_requests(members, kick_request) self.bot.stats.incr("verification.kicked", count=n_kicked) - if bad_statuses: - log.info(f"Failed to kick {len(members) - n_kicked} members due to following statuses: {bad_statuses}") - return n_kicked async def _give_role(self, members: t.Collection[discord.Member], role: discord.Role) -> int: """ Give `role` to all `members`. - Returns the amount of successful requests. Status codes of unsuccessful requests - are logged at info level. + Returns the amount of successful requests. """ log.info(f"Assigning {role} role to {len(members)} members (not verified after {UNVERIFIED_AFTER} days)") - n_success, bad_statuses = 0, set() - for member in members: - if is_verified(member): # Member could have verified in the meantime - continue - try: - await member.add_roles(role, reason=f"User has not verified in {UNVERIFIED_AFTER} days") - except discord.HTTPException as http_exc: - bad_statuses.add(http_exc.status) - else: - n_success += 1 + async def role_request(member: discord.Member) -> None: + """Add `role` to `member`.""" + await member.add_roles(role, reason=f"User has not verified in {UNVERIFIED_AFTER} days") - if bad_statuses: - log.info(f"Failed to assign {len(members) - n_success} roles due to following statuses: {bad_statuses}") - - return n_success + return await self._send_requests(members, role_request) async def _check_members(self) -> t.Tuple[t.Set[discord.Member], t.Set[discord.Member]]: """ -- cgit v1.2.3 From 689e5203f993604f1cb40be65e22b79ceb04d382 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Fri, 14 Aug 2020 19:39:07 +0200 Subject: Verification: pause request execution after each batch The Limit values are mostly assumptions, as this feature is very difficult to test at scale. Please see docstring amendmends for further information. --- bot/cogs/verification.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 8f1a773a8..14c0abfda 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -88,6 +88,13 @@ MENTION_UNVERIFIED = discord.AllowedMentions( Request = t.Callable[[discord.Member], t.Awaitable] +class Limit(t.NamedTuple): + """Composition over config for throttling requests.""" + + batch_size: int # Amount of requests after which to pause + sleep_secs: int # Sleep this many seconds after each batch + + def is_verified(member: discord.Member) -> bool: """ Check whether `member` is considered verified. @@ -233,19 +240,22 @@ class Verification(Cog): return result - async def _send_requests(self, members: t.Collection[discord.Member], request: Request) -> int: + async def _send_requests(self, members: t.Collection[discord.Member], request: Request, limit: Limit) -> int: """ Pass `members` one by one to `request` handling Discord exceptions. This coroutine serves as a generic `request` executor for kicking members and adding roles, as it allows us to define the error handling logic in one place only. + To avoid rate-limits, pass a `limit` configuring the batch size and the amount of seconds + to sleep between batches. + Returns the amount of successful requests. Failed requests are logged at info level. """ log.info(f"Sending {len(members)} requests") n_success, bad_statuses = 0, set() - for member in members: + for progress, member in enumerate(members, start=1): if is_verified(member): # Member could have verified in the meantime continue try: @@ -255,6 +265,10 @@ class Verification(Cog): else: n_success += 1 + if progress % limit.batch_size == 0: + log.trace(f"Processed {progress} requests, pausing for {limit.sleep_secs} seconds") + await asyncio.sleep(limit.sleep_secs) + if bad_statuses: log.info(f"Failed to send {len(members) - n_success} requests due to following statuses: {bad_statuses}") @@ -264,6 +278,9 @@ class Verification(Cog): """ Kick `members` from the PyDis guild. + Due to strict ratelimits on sending messages (120 requests / 60 secs), we sleep for a second + after each 2 requests to allow breathing room for other features. + Note that this is a potentially destructive operation. Returns the amount of successful requests. """ log.info(f"Kicking {len(members)} members from the guild (not verified after {KICKED_AFTER} days)") @@ -274,7 +291,7 @@ class Verification(Cog): await member.send(KICKED_MESSAGE) await member.kick(reason=f"User has not verified in {KICKED_AFTER} days") - n_kicked = await self._send_requests(members, kick_request) + n_kicked = await self._send_requests(members, kick_request, Limit(batch_size=2, sleep_secs=1)) self.bot.stats.incr("verification.kicked", count=n_kicked) return n_kicked @@ -283,6 +300,8 @@ class Verification(Cog): """ Give `role` to all `members`. + We pause for a second after batches of 25 requests to ensure ratelimits aren't exceeded. + Returns the amount of successful requests. """ log.info(f"Assigning {role} role to {len(members)} members (not verified after {UNVERIFIED_AFTER} days)") @@ -291,7 +310,7 @@ class Verification(Cog): """Add `role` to `member`.""" await member.add_roles(role, reason=f"User has not verified in {UNVERIFIED_AFTER} days") - return await self._send_requests(members, role_request) + return await self._send_requests(members, role_request, Limit(batch_size=25, sleep_secs=1)) async def _check_members(self) -> t.Tuple[t.Set[discord.Member], t.Set[discord.Member]]: """ -- cgit v1.2.3 From ff91b76348d95e308589ac131898aba9f7cca986 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Wed, 19 Aug 2020 20:06:23 +0200 Subject: Verification: add missing word to task status message --- bot/cogs/verification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 14c0abfda..08d54d575 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -537,9 +537,9 @@ class Verification(Cog): mention = f"<@&{constants.Roles.unverified}>" if self.ping_unverified.is_running(): - ping_status = f"{constants.Emojis.incident_actioned} Ping {mention} is running." + ping_status = f"{constants.Emojis.incident_actioned} Ping {mention} task is running." else: - ping_status = f"{constants.Emojis.incident_unactioned} Ping {mention} is **not** running." + ping_status = f"{constants.Emojis.incident_unactioned} Ping {mention} task is **not** running." embed = discord.Embed( title="Verification system", -- cgit v1.2.3 From a565bb529cbfcb241a7cf63c114a17f695451721 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Wed, 26 Aug 2020 18:31:42 +0200 Subject: Verification: add guild invite to config --- bot/constants.py | 1 + config-default.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/bot/constants.py b/bot/constants.py index 0902858ac..daef6c095 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -458,6 +458,7 @@ class Guild(metaclass=YAMLGetter): section = "guild" id: int + invite: str # Discord invite, gets embedded in chat moderation_channels: List[int] moderation_roles: List[int] modlog_blacklist: List[int] diff --git a/config-default.yml b/config-default.yml index 58bdbe20f..a98fd14ef 100644 --- a/config-default.yml +++ b/config-default.yml @@ -123,6 +123,7 @@ style: guild: id: 267624335836053506 + invite: "https://discord.gg/python" categories: help_available: 691405807388196926 -- cgit v1.2.3 From 53189e815a5c260bb2636914ecd79f4f4c1182a0 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Wed, 26 Aug 2020 18:33:16 +0200 Subject: Verification: send guild invite with kick message Makes it easy for users to re-join. Co-authored-by: Joe Banks --- bot/cogs/verification.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 08d54d575..56b469a3e 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -62,7 +62,8 @@ If you'd like to unsubscribe from the announcement notifications, simply send `! # Sent via DMs to users kicked for failing to verify KICKED_MESSAGE = f""" Hi! You have been automatically kicked from Python Discord as you have failed to accept our rules \ -within `{KICKED_AFTER}` days. If this was an accident, please feel free to join again. +within `{KICKED_AFTER}` days. If this was an accident, please feel free to join us again! +{constants.Guild.invite} """ # Sent periodically in the verification channel -- cgit v1.2.3 From 3ea04e3ddeaeef1d7931df8f5b84293d5eac6a04 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Wed, 26 Aug 2020 18:50:57 +0200 Subject: Verification: separate guild invite by empty line Co-authored-by: Joe Banks --- bot/cogs/verification.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 56b469a3e..a35681988 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -63,6 +63,7 @@ If you'd like to unsubscribe from the announcement notifications, simply send `! KICKED_MESSAGE = f""" Hi! You have been automatically kicked from Python Discord as you have failed to accept our rules \ within `{KICKED_AFTER}` days. If this was an accident, please feel free to join us again! + {constants.Guild.invite} """ -- cgit v1.2.3 From eff2f75321fce0e8a8d11a1c85c2dad48552ded8 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Wed, 26 Aug 2020 19:34:14 +0200 Subject: Verification: remove explicit everyones from allowed mentions If the kwarg isn't passed, it uses the value that was given to the bot on init (False), despite the kwarg defaulting to True. Thanks to Mark and Senjan for helping me understand this. Co-authored-by: MarkKoz Co-authored-by: Senjan21 <53477086+senjan21@users.noreply.github.com> --- bot/cogs/verification.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index a35681988..a0a82be0c 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -79,12 +79,8 @@ You will be kicked if you don't verify within `{KICKED_AFTER}` days. REMINDER_FREQUENCY = 28 # Hours to wait between sending `REMINDER_MESSAGE` -MENTION_CORE_DEVS = discord.AllowedMentions( - everyone=False, roles=[discord.Object(constants.Roles.core_developers)] -) -MENTION_UNVERIFIED = discord.AllowedMentions( - everyone=False, roles=[discord.Object(constants.Roles.unverified)] -) +MENTION_CORE_DEVS = discord.AllowedMentions(roles=[discord.Object(constants.Roles.core_developers)]) +MENTION_UNVERIFIED = discord.AllowedMentions(roles=[discord.Object(constants.Roles.unverified)]) # An async function taking a Member param Request = t.Callable[[discord.Member], t.Awaitable] -- cgit v1.2.3 From 44aae4528ecec5eef8e2b56f7ac851219b35f080 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Wed, 26 Aug 2020 19:42:31 +0200 Subject: Verification: retain ping in edited confirmation msg Prevent a ghost ping from occurring upon reaction. Co-authored-by: Senjan21 <53477086+senjan21@users.noreply.github.com> --- bot/cogs/verification.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index a0a82be0c..d21395a1c 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -192,11 +192,12 @@ class Verification(Cog): # Since `n_members` is a suspiciously large number, we will ask for confirmation log.debug("Amount of users is too large, requesting staff confirmation") - core_devs = pydis.get_channel(constants.Channels.dev_core) - confirmation_msg = await core_devs.send( - f"<@&{constants.Roles.core_developers}> Verification determined that `{n_members}` members should " - f"be kicked as they haven't verified in `{KICKED_AFTER}` days. This is `{percentage:.2%}` of the " - f"guild's population. Proceed?", + core_dev_channel = pydis.get_channel(constants.Channels.dev_core) + core_dev_ping = f"<@&{constants.Roles.core_developers}>" + + confirmation_msg = await core_dev_channel.send( + f"{core_dev_ping} Verification determined that `{n_members}` members should be kicked as they haven't " + f"verified in `{KICKED_AFTER}` days. This is `{percentage:.2%}` of the guild's population. Proceed?", allowed_mentions=MENTION_CORE_DEVS, ) @@ -229,9 +230,9 @@ class Verification(Cog): # Edit the prompt message to reflect the final choice if result is True: - result_msg = f":ok_hand: Request to kick `{n_members}` members was authorized!" + result_msg = f":ok_hand: {core_dev_ping} Request to kick `{n_members}` members was authorized!" else: - result_msg = f":warning: Request to kick `{n_members}` members was denied!" + result_msg = f":warning: {core_dev_ping} Request to kick `{n_members}` members was denied!" with suppress(discord.HTTPException): await confirmation_msg.edit(content=result_msg) -- cgit v1.2.3 From 3672da9d6ad16452205e00a86162314f457fbbd0 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Wed, 26 Aug 2020 22:26:15 +0200 Subject: Verification: add helper for alerting admins --- bot/cogs/verification.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index d21395a1c..d5216c7c0 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -79,6 +79,7 @@ You will be kicked if you don't verify within `{KICKED_AFTER}` days. REMINDER_FREQUENCY = 28 # Hours to wait between sending `REMINDER_MESSAGE` +MENTION_ADMINS = discord.AllowedMentions(roles=[discord.Object(constants.Roles.admins)]) MENTION_CORE_DEVS = discord.AllowedMentions(roles=[discord.Object(constants.Roles.core_developers)]) MENTION_UNVERIFIED = discord.AllowedMentions(roles=[discord.Object(constants.Roles.unverified)]) @@ -239,6 +240,23 @@ class Verification(Cog): return result + async def _alert_admins(self, exception: discord.HTTPException) -> None: + """ + Ping @Admins with information about `exception`. + + This is used when a critical `exception` caused a verification task to abort. + """ + await self.bot.wait_until_guild_available() + log.info(f"Sending admin alert regarding exception: {exception}") + + admins_channel = self.bot.get_guild(constants.Guild.id).get_channel(constants.Channels.admins) + ping = f"<@&{constants.Roles.admins}>" + + await admins_channel.send( + f"{ping} Aborted updating unverified users due to the following exception:\n```{exception}```", + allowed_mentions=MENTION_ADMINS, + ) + async def _send_requests(self, members: t.Collection[discord.Member], request: Request, limit: Limit) -> int: """ Pass `members` one by one to `request` handling Discord exceptions. -- cgit v1.2.3 From 1d6d845d2f052c74d6d92a1a98b537430296cc85 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Wed, 26 Aug 2020 22:27:47 +0200 Subject: Verification: stop kicking members on suspicious 403 A Discord error code 50_0007 signifies that the DM dispatch failed because the target user does not accept DMs from the bot. Such errors are ignored as before. Any other 403s will however cause the bot to stop making requests. This is in case the bot gets caught by an anti-spam filter and should immediately stop. --- bot/cogs/verification.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index d5216c7c0..196808b0d 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -87,6 +87,14 @@ MENTION_UNVERIFIED = discord.AllowedMentions(roles=[discord.Object(constants.Rol Request = t.Callable[[discord.Member], t.Awaitable] +class StopExecution(Exception): + """Signals that a task should halt immediately & alert admins.""" + + def __init__(self, reason: discord.HTTPException) -> None: + super().__init__() + self.reason = reason + + class Limit(t.NamedTuple): """Composition over config for throttling requests.""" @@ -277,6 +285,9 @@ class Verification(Cog): continue try: await request(member) + except StopExecution as stop_execution: + await self._alert_admins(stop_execution.reason) + break except discord.HTTPException as http_exc: bad_statuses.add(http_exc.status) else: @@ -304,8 +315,12 @@ class Verification(Cog): async def kick_request(member: discord.Member) -> None: """Send `KICKED_MESSAGE` to `member` and kick them from the guild.""" - with suppress(discord.Forbidden): + try: await member.send(KICKED_MESSAGE) + except discord.Forbidden as exc_403: + log.trace(f"DM dispatch failed on 403 error with code: {exc_403.code}") + if exc_403.code != 50_007: # 403 raised for any other reason than disabled DMs + raise StopExecution(reason=exc_403) await member.kick(reason=f"User has not verified in {KICKED_AFTER} days") n_kicked = await self._send_requests(members, kick_request, Limit(batch_size=2, sleep_secs=1)) -- cgit v1.2.3 From 0ba23a7c47813280ab6157396e43d61e8fc7b4d2 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Thu, 27 Aug 2020 10:07:23 +0200 Subject: Verification: document StopExecution handling --- bot/cogs/verification.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 196808b0d..984f3cc95 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -272,6 +272,9 @@ class Verification(Cog): This coroutine serves as a generic `request` executor for kicking members and adding roles, as it allows us to define the error handling logic in one place only. + Any `request` has the ability to completely abort the execution by raising `StopExecution`. + In such a case, the @Admins will be alerted of the reason attribute. + To avoid rate-limits, pass a `limit` configuring the batch size and the amount of seconds to sleep between batches. -- cgit v1.2.3 From 69314d4b812361f2b2a02018093f9a504ac4674f Mon Sep 17 00:00:00 2001 From: kwzrd Date: Thu, 27 Aug 2020 14:29:21 +0200 Subject: Verification: improve allowed mentions handling I really didn't like the constants, but the construction of allowed mentions instances is syntactically noisy, so I prefer to keep it out of the important logic. Abstracting it behind a function seems to be the best approach yet. --- bot/cogs/verification.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 984f3cc95..0ae3c5b4c 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -79,10 +79,6 @@ You will be kicked if you don't verify within `{KICKED_AFTER}` days. REMINDER_FREQUENCY = 28 # Hours to wait between sending `REMINDER_MESSAGE` -MENTION_ADMINS = discord.AllowedMentions(roles=[discord.Object(constants.Roles.admins)]) -MENTION_CORE_DEVS = discord.AllowedMentions(roles=[discord.Object(constants.Roles.core_developers)]) -MENTION_UNVERIFIED = discord.AllowedMentions(roles=[discord.Object(constants.Roles.unverified)]) - # An async function taking a Member param Request = t.Callable[[discord.Member], t.Awaitable] @@ -102,6 +98,11 @@ class Limit(t.NamedTuple): sleep_secs: int # Sleep this many seconds after each batch +def mention_role(role_id: int) -> discord.AllowedMentions: + """Construct an allowed mentions instance that allows pinging `role_id`.""" + return discord.AllowedMentions(roles=[discord.Object(role_id)]) + + def is_verified(member: discord.Member) -> bool: """ Check whether `member` is considered verified. @@ -207,7 +208,7 @@ class Verification(Cog): confirmation_msg = await core_dev_channel.send( f"{core_dev_ping} Verification determined that `{n_members}` members should be kicked as they haven't " f"verified in `{KICKED_AFTER}` days. This is `{percentage:.2%}` of the guild's population. Proceed?", - allowed_mentions=MENTION_CORE_DEVS, + allowed_mentions=mention_role(constants.Roles.core_developers), ) options = (constants.Emojis.incident_actioned, constants.Emojis.incident_unactioned) @@ -262,7 +263,7 @@ class Verification(Cog): await admins_channel.send( f"{ping} Aborted updating unverified users due to the following exception:\n```{exception}```", - allowed_mentions=MENTION_ADMINS, + allowed_mentions=mention_role(constants.Roles.admins), ) async def _send_requests(self, members: t.Collection[discord.Member], request: Request, limit: Limit) -> int: @@ -445,7 +446,9 @@ class Verification(Cog): await self.bot.http.delete_message(verification.id, last_reminder) log.trace("Sending verification reminder") - new_reminder = await verification.send(REMINDER_MESSAGE, allowed_mentions=MENTION_UNVERIFIED) + new_reminder = await verification.send( + REMINDER_MESSAGE, allowed_mentions=mention_role(constants.Roles.unverified), + ) await self.task_cache.set("last_reminder", new_reminder.id) -- cgit v1.2.3 From c58ae662069d098dae45a36a5203b6d5f0048924 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sat, 29 Aug 2020 18:29:24 +0200 Subject: Verification: add helper for stopping tasks --- bot/cogs/verification.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 0ae3c5b4c..a013a1b12 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -157,8 +157,7 @@ class Verification(Cog): This is necessary, as tasks are not automatically cancelled on cog unload. """ - self.update_unverified_members.cancel() - self.ping_unverified.cancel() + self._stop_tasks(gracefully=False) @property def mod_log(self) -> ModLog: @@ -179,6 +178,21 @@ class Verification(Cog): self.update_unverified_members.start() self.ping_unverified.start() + def _stop_tasks(self, *, gracefully: bool) -> None: + """ + Stop the update users & ping @Unverified tasks. + + If `gracefully` is True, the tasks will be able to finish their current iteration. + Otherwise, they are cancelled immediately. + """ + log.info(f"Stopping internal tasks ({gracefully=})") + if gracefully: + self.update_unverified_members.stop() + self.ping_unverified.stop() + else: + self.update_unverified_members.cancel() + self.ping_unverified.cancel() + # region: automatically update unverified users async def _verify_kick(self, n_members: int) -> bool: @@ -607,9 +621,7 @@ class Verification(Cog): """Stop verification tasks.""" log.info("Stopping verification tasks") - self.update_unverified_members.cancel() - self.ping_unverified.cancel() - + self._stop_tasks(gracefully=False) await self.task_cache.set("tasks_running", 0) colour = discord.Colour.blurple() -- cgit v1.2.3 From cf7388ff8490be95f1d677f424e8bec86de0e46a Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sat, 29 Aug 2020 18:32:12 +0200 Subject: Verification: stop tasks on suspicious 403 --- bot/cogs/verification.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index a013a1b12..107ae1178 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -276,7 +276,9 @@ class Verification(Cog): ping = f"<@&{constants.Roles.admins}>" await admins_channel.send( - f"{ping} Aborted updating unverified users due to the following exception:\n```{exception}```", + f"{ping} Aborted updating unverified users due to the following exception:\n" + f"```{exception}```\n" + f"Internal tasks will be stopped.", allowed_mentions=mention_role(constants.Roles.admins), ) @@ -305,6 +307,7 @@ class Verification(Cog): await request(member) except StopExecution as stop_execution: await self._alert_admins(stop_execution.reason) + self._stop_tasks(gracefully=True) # Gracefully finish current iteration, then stop break except discord.HTTPException as http_exc: bad_statuses.add(http_exc.status) -- cgit v1.2.3 From ad305f2eb3bb33f5ce7bb3abd7def9f436928c8e Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sat, 29 Aug 2020 18:37:32 +0200 Subject: Verification: denote `_maybe_start_tasks` as private Consistency with the new `_stop_tasks` method. --- bot/cogs/verification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 107ae1178..5c8962577 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -149,7 +149,7 @@ class Verification(Cog): def __init__(self, bot: Bot) -> None: """Start internal tasks.""" self.bot = bot - self.bot.loop.create_task(self.maybe_start_tasks()) + self.bot.loop.create_task(self._maybe_start_tasks()) def cog_unload(self) -> None: """ @@ -164,7 +164,7 @@ class Verification(Cog): """Get currently loaded ModLog cog instance.""" return self.bot.get_cog("ModLog") - async def maybe_start_tasks(self) -> None: + async def _maybe_start_tasks(self) -> None: """ Poll Redis to check whether internal tasks should start. -- cgit v1.2.3 From 6c98df046c535282c6d4b194b0766afcddbdf669 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Sat, 29 Aug 2020 19:54:27 +0200 Subject: Verification: set 'tasks_running' to 0 on suspicious 403s Prevent the tasks from starting again if the bot restarts. --- bot/cogs/verification.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 5c8962577..08f7c282e 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -307,6 +307,7 @@ class Verification(Cog): await request(member) except StopExecution as stop_execution: await self._alert_admins(stop_execution.reason) + await self.task_cache.set("tasks_running", 0) self._stop_tasks(gracefully=True) # Gracefully finish current iteration, then stop break except discord.HTTPException as http_exc: -- cgit v1.2.3 From d2e7dd3763d24a2224fe0eefd78852e2a2389850 Mon Sep 17 00:00:00 2001 From: Numerlor <25886452+Numerlor@users.noreply.github.com> Date: Fri, 4 Sep 2020 20:25:26 +0200 Subject: Move bolding markdown outside of text link. On some devices the markdown gets rendered improperly, leaving the asterisks in the message without bolding. --- bot/cogs/help_channels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 541c6f336..0f9cac89e 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -36,7 +36,7 @@ the **Help: Dormant** category. Try to write the best question you can by providing a detailed description and telling us what \ you've tried already. For more information on asking a good question, \ -check out our guide on [**asking good questions**]({ASKING_GUIDE_URL}). +check out our guide on **[asking good questions]({ASKING_GUIDE_URL})**. """ DORMANT_MSG = f""" @@ -47,7 +47,7 @@ channel until it becomes available again. If your question wasn't answered yet, you can claim a new help channel from the \ **Help: Available** category by simply asking your question again. Consider rephrasing the \ question to maximize your chance of getting a good answer. If you're not sure how, have a look \ -through our guide for [**asking a good question**]({ASKING_GUIDE_URL}). +through our guide for **[asking a good question]({ASKING_GUIDE_URL})**. """ CoroutineFunc = t.Callable[..., t.Coroutine] -- cgit v1.2.3 From 53cb77bb2d541d0be61bc3c25e37b54601963b7c Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff Date: Sat, 5 Sep 2020 11:07:58 +0200 Subject: Disable everyone_ping filter in AntiSpam cog As there are a few bugs in the implementation, I've temporarily disabled the at-everyone ping filter in the AntiSpam cog. We can disable it after we've fixed the bugs. Signed-off-by: Sebastiaan Zeeff --- bot/cogs/antispam.py | 4 +++- config-default.yml | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bot/cogs/antispam.py b/bot/cogs/antispam.py index b8939113f..3ad487d8c 100644 --- a/bot/cogs/antispam.py +++ b/bot/cogs/antispam.py @@ -36,7 +36,9 @@ RULE_FUNCTION_MAPPING = { 'mentions': rules.apply_mentions, 'newlines': rules.apply_newlines, 'role_mentions': rules.apply_role_mentions, - 'everyone_ping': rules.apply_everyone_ping, + # the everyone filter is temporarily disabled until + # it has been improved. + # 'everyone_ping': rules.apply_everyone_ping, } diff --git a/config-default.yml b/config-default.yml index e9324c62f..6e7cff92d 100644 --- a/config-default.yml +++ b/config-default.yml @@ -389,9 +389,11 @@ anti_spam: interval: 10 max: 3 - everyone_ping: - interval: 10 - max: 0 + # The everyone ping filter is temporarily disabled + # until we've fixed a couple of bugs. + # everyone_ping: + # interval: 10 + # max: 0 reddit: -- cgit v1.2.3 From 1b3a5c5d90bb658895fea8a2bed91366d2f2f76e Mon Sep 17 00:00:00 2001 From: kwzrd Date: Thu, 10 Sep 2020 18:41:07 +0200 Subject: Verification: move constants to config --- bot/cogs/verification.py | 45 ++++++++++++++++++++------------------------- bot/constants.py | 10 ++++++++++ config-default.yml | 13 +++++++++++++ 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 08f7c282e..0092a0898 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -18,16 +18,6 @@ from bot.utils.redis_cache import RedisCache log = logging.getLogger(__name__) -UNVERIFIED_AFTER = 3 # Amount of days after which non-Developers receive the @Unverified role -KICKED_AFTER = 30 # Amount of days after which non-Developers get kicked from the guild - -# Number in range [0, 1] determining the percentage of unverified users that are safe -# to be kicked from the guild in one batch, any larger amount will require staff confirmation, -# set this to 0 to require explicit approval for batches of any size -KICK_CONFIRMATION_THRESHOLD = 0.01 # 1% - -BOT_MESSAGE_DELETE_DELAY = 10 - # Sent via DMs once user joins the guild ON_JOIN_MESSAGE = f""" Hello! Welcome to Python Discord! @@ -62,7 +52,7 @@ If you'd like to unsubscribe from the announcement notifications, simply send `! # Sent via DMs to users kicked for failing to verify KICKED_MESSAGE = f""" Hi! You have been automatically kicked from Python Discord as you have failed to accept our rules \ -within `{KICKED_AFTER}` days. If this was an accident, please feel free to join us again! +within `{constants.Verification.kicked_after}` days. If this was an accident, please feel free to join us again! {constants.Guild.invite} """ @@ -74,11 +64,9 @@ REMINDER_MESSAGE = f""" Welcome to Python Discord! Please read the documents mentioned above and type `!accept` to gain permissions \ to send messages in the community! -You will be kicked if you don't verify within `{KICKED_AFTER}` days. +You will be kicked if you don't verify within `{constants.Verification.kicked_after}` days. """.strip() -REMINDER_FREQUENCY = 28 # Hours to wait between sending `REMINDER_MESSAGE` - # An async function taking a Member param Request = t.Callable[[discord.Member], t.Awaitable] @@ -209,7 +197,7 @@ class Verification(Cog): pydis = self.bot.get_guild(constants.Guild.id) percentage = n_members / len(pydis.members) - if percentage < KICK_CONFIRMATION_THRESHOLD: + if percentage < constants.Verification.kick_confirmation_threshold: log.debug(f"Kicking {percentage:.2%} of the guild's population is seen as safe") return True @@ -221,7 +209,8 @@ class Verification(Cog): confirmation_msg = await core_dev_channel.send( f"{core_dev_ping} Verification determined that `{n_members}` members should be kicked as they haven't " - f"verified in `{KICKED_AFTER}` days. This is `{percentage:.2%}` of the guild's population. Proceed?", + f"verified in `{constants.Verification.kicked_after}` days. This is `{percentage:.2%}` of the guild's " + f"population. Proceed?", allowed_mentions=mention_role(constants.Roles.core_developers), ) @@ -333,7 +322,7 @@ class Verification(Cog): Note that this is a potentially destructive operation. Returns the amount of successful requests. """ - log.info(f"Kicking {len(members)} members from the guild (not verified after {KICKED_AFTER} days)") + log.info(f"Kicking {len(members)} members (not verified after {constants.Verification.kicked_after} days)") async def kick_request(member: discord.Member) -> None: """Send `KICKED_MESSAGE` to `member` and kick them from the guild.""" @@ -343,7 +332,7 @@ class Verification(Cog): log.trace(f"DM dispatch failed on 403 error with code: {exc_403.code}") if exc_403.code != 50_007: # 403 raised for any other reason than disabled DMs raise StopExecution(reason=exc_403) - await member.kick(reason=f"User has not verified in {KICKED_AFTER} days") + await member.kick(reason=f"User has not verified in {constants.Verification.kicked_after} days") n_kicked = await self._send_requests(members, kick_request, Limit(batch_size=2, sleep_secs=1)) self.bot.stats.incr("verification.kicked", count=n_kicked) @@ -358,11 +347,14 @@ class Verification(Cog): Returns the amount of successful requests. """ - log.info(f"Assigning {role} role to {len(members)} members (not verified after {UNVERIFIED_AFTER} days)") + log.info( + f"Assigning {role} role to {len(members)} members (not verified " + f"after {constants.Verification.unverified_after} days)" + ) async def role_request(member: discord.Member) -> None: """Add `role` to `member`.""" - await member.add_roles(role, reason=f"User has not verified in {UNVERIFIED_AFTER} days") + await member.add_roles(role, reason=f"Not verified after {constants.Verification.unverified_after} days") return await self._send_requests(members, role_request, Limit(batch_size=25, sleep_secs=1)) @@ -397,10 +389,13 @@ class Verification(Cog): # to do with them based on time passed since their join date since_join = current_dt - member.joined_at - if since_join > timedelta(days=KICKED_AFTER): + if since_join > timedelta(days=constants.Verification.kicked_after): for_kick.add(member) # User should be removed from the guild - elif since_join > timedelta(days=UNVERIFIED_AFTER) and unverified not in member.roles: + elif ( + since_join > timedelta(days=constants.Verification.unverified_after) + and unverified not in member.roles + ): for_role.add(member) # User should be given the @Unverified role log.debug(f"Found {len(for_role)} users for {unverified} role, {len(for_kick)} users to be kicked") @@ -445,7 +440,7 @@ class Verification(Cog): # endregion # region: periodically ping @Unverified - @tasks.loop(hours=REMINDER_FREQUENCY) + @tasks.loop(hours=constants.Verification.reminder_frequency) async def ping_unverified(self) -> None: """ Delete latest `REMINDER_MESSAGE` and send it again. @@ -488,7 +483,7 @@ class Verification(Cog): time_since = datetime.utcnow() - snowflake_time(last_reminder) log.trace(f"Time since latest verification reminder: {time_since}") - to_sleep = timedelta(hours=REMINDER_FREQUENCY) - time_since + to_sleep = timedelta(hours=constants.Verification.reminder_frequency) - time_since log.trace(f"Time to sleep until next ping: {to_sleep}") # Delta can be negative if `REMINDER_FREQUENCY` has already passed @@ -519,7 +514,7 @@ class Verification(Cog): if message.author.bot: # They're a bot, delete their message after the delay. - await message.delete(delay=BOT_MESSAGE_DELETE_DELAY) + await message.delete(delay=constants.Verification.bot_message_delete_delay) return # if a user mentions a role or guild member diff --git a/bot/constants.py b/bot/constants.py index daef6c095..820828a19 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -589,6 +589,16 @@ class PythonNews(metaclass=YAMLGetter): webhook: int +class Verification(metaclass=YAMLGetter): + section = "verification" + + unverified_after: int + kicked_after: int + reminder_frequency: int + bot_message_delete_delay: int + kick_confirmation_threshold: float + + class Event(Enum): """ Event names. This does not include every event (for example, raw diff --git a/config-default.yml b/config-default.yml index a98fd14ef..c89695bd9 100644 --- a/config-default.yml +++ b/config-default.yml @@ -493,5 +493,18 @@ python_news: channel: *PYNEWS_CHANNEL webhook: *PYNEWS_WEBHOOK + +verification: + unverified_after: 3 # Days after which non-Developers receive the @Unverified role + kicked_after: 30 # Days after which non-Developers get kicked from the guild + reminder_frequency: 28 # Hours between @Unverified pings + bot_message_delete_delay: 10 # Seconds before deleting bots response in #verification + + # Number in range [0, 1] determining the percentage of unverified users that are safe + # to be kicked from the guild in one batch, any larger amount will require staff confirmation, + # set this to 0 to require explicit approval for batches of any size + kick_confirmation_threshold: 0.01 # 1% + + config: required_keys: ['bot.token'] -- cgit v1.2.3 From 7ebc481c32cf9b7f0ee1124b22e4ed8c68de9386 Mon Sep 17 00:00:00 2001 From: kwzrd Date: Fri, 11 Sep 2020 19:46:48 +0200 Subject: Verification: update & improve docstrings After moving constants to config, the docstring references were not updated accordingly, and remained uppercase. This commit also removed the redundant list indentation. --- bot/cogs/verification.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 0092a0898..9ae92a228 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -111,12 +111,11 @@ class Verification(Cog): There are two internal tasks in this cog: - * `update_unverified_members` - * Unverified members are given the @Unverified role after `UNVERIFIED_AFTER` days - * Unverified members are kicked after `UNVERIFIED_AFTER` days - - * `ping_unverified` - * Periodically ping the @Unverified role in the verification channel + * `update_unverified_members` + * Unverified members are given the @Unverified role after configured `unverified_after` days + * Unverified members are kicked after configured `kicked_after` days + * `ping_unverified` + * Periodically ping the @Unverified role in the verification channel Statistics are collected in the 'verification.' namespace. @@ -188,8 +187,8 @@ class Verification(Cog): Determine whether `n_members` is a reasonable amount of members to kick. First, `n_members` is checked against the size of the PyDis guild. If `n_members` are - more than `KICK_CONFIRMATION_THRESHOLD` of the guild, the operation must be confirmed - by staff in #core-dev. Otherwise, the operation is seen as safe. + more than the configured `kick_confirmation_threshold` of the guild, the operation + must be confirmed by staff in #core-dev. Otherwise, the operation is seen as safe. """ log.debug(f"Checking whether {n_members} members are safe to kick") @@ -363,8 +362,8 @@ class Verification(Cog): Check in on the verification status of PyDis members. This coroutine finds two sets of users: - * Not verified after `UNVERIFIED_AFTER` days, should be given the @Unverified role - * Not verified after `KICKED_AFTER` days, should be kicked from the guild + * Not verified after configured `unverified_after` days, should be given the @Unverified role + * Not verified after configured `kicked_after` days, should be kicked from the guild These sets are always disjoint, i.e. share no common members. """ @@ -471,7 +470,7 @@ class Verification(Cog): Sleep until `REMINDER_MESSAGE` should be sent again. If latest reminder is not cached, exit instantly. Otherwise, wait wait until the - configured `REMINDER_FREQUENCY` has passed. + configured `reminder_frequency` has passed. """ last_reminder: t.Optional[int] = await self.task_cache.get("last_reminder") @@ -486,7 +485,7 @@ class Verification(Cog): to_sleep = timedelta(hours=constants.Verification.reminder_frequency) - time_since log.trace(f"Time to sleep until next ping: {to_sleep}") - # Delta can be negative if `REMINDER_FREQUENCY` has already passed + # Delta can be negative if `reminder_frequency` has already passed secs = max(to_sleep.total_seconds(), 0) await asyncio.sleep(secs) -- cgit v1.2.3 From 93145528d7859842602df5c6535f3995187ffadb Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Wed, 16 Sep 2020 17:34:12 +0200 Subject: Updating names of reddit emotes. --- config-default.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/config-default.yml b/config-default.yml index 20254d584..e97ebf0e8 100644 --- a/config-default.yml +++ b/config-default.yml @@ -76,9 +76,10 @@ style: ducky_maul: &DUCKY_MAUL 640137724958867467 ducky_santa: &DUCKY_SANTA 655360331002019870 - upvotes: "<:upvotes:638729835245731840>" - comments: "<:comments:638729835073765387>" - user: "<:user:638729835442602003>" + # emotes used for #reddit + upvotes: "<:reddit_upvotes:638729835245731840>" + comments: "<:reddit_comments:638729835073765387>" + user: "<:reddit_user:638729835442602003>" icons: crown_blurple: "https://cdn.discordapp.com/emojis/469964153289965568.png" -- cgit v1.2.3 From f8272b4cdcab08145c1cecedd8bbbfea4bbf3239 Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Wed, 16 Sep 2020 19:42:10 +0200 Subject: update the reddit emojis to actual emojis' --- config-default.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config-default.yml b/config-default.yml index e97ebf0e8..87bfd1378 100644 --- a/config-default.yml +++ b/config-default.yml @@ -77,9 +77,9 @@ style: ducky_santa: &DUCKY_SANTA 655360331002019870 # emotes used for #reddit - upvotes: "<:reddit_upvotes:638729835245731840>" - comments: "<:reddit_comments:638729835073765387>" - user: "<:reddit_user:638729835442602003>" + upvotes: "<:reddit_upvotes:755845219890757644> " + comments: "<:reddit_comments:755845255001014384>" + user: "<:reddit_users:755845303822974997>" icons: crown_blurple: "https://cdn.discordapp.com/emojis/469964153289965568.png" -- cgit v1.2.3 From 6cef33ee7b68d07a1026b784f0350b146469702d Mon Sep 17 00:00:00 2001 From: Senjan21 <53477086+Senjan21@users.noreply.github.com> Date: Wed, 16 Sep 2020 19:43:44 +0200 Subject: remove random space in `upvotes` value --- config-default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config-default.yml b/config-default.yml index 87bfd1378..58651f548 100644 --- a/config-default.yml +++ b/config-default.yml @@ -77,7 +77,7 @@ style: ducky_santa: &DUCKY_SANTA 655360331002019870 # emotes used for #reddit - upvotes: "<:reddit_upvotes:755845219890757644> " + upvotes: "<:reddit_upvotes:755845219890757644>" comments: "<:reddit_comments:755845255001014384>" user: "<:reddit_users:755845303822974997>" -- cgit v1.2.3