diff options
| author | 2020-10-01 16:01:39 +0530 | |
|---|---|---|
| committer | 2020-10-01 16:01:39 +0530 | |
| commit | ba9efee8b1501894f3e73d0a322386620c5c4343 (patch) | |
| tree | 8cc0f9a4e393c0ed2a8521e6fee0711a4bf8c5fe | |
| parent | remove redundant type hints and improve existing function annotations (diff) | |
| parent | Merge pull request #1199 from python-discord/bug/backend/1181/wait-for-deleti... (diff) | |
Merge branch 'master' into smart_syncing_users
| -rw-r--r-- | Pipfile.lock | 132 | ||||
| -rw-r--r-- | bot/constants.py | 8 | ||||
| -rw-r--r-- | bot/exts/backend/sync/_syncers.py | 172 | ||||
| -rw-r--r-- | bot/exts/filters/token_remover.py | 77 | ||||
| -rw-r--r-- | bot/exts/info/information.py | 4 | ||||
| -rw-r--r-- | bot/exts/moderation/incidents.py | 2 | ||||
| -rw-r--r-- | bot/exts/moderation/modlog.py | 2 | ||||
| -rw-r--r-- | bot/exts/moderation/verification.py | 50 | ||||
| -rw-r--r-- | bot/exts/utils/snekbox.py | 2 | ||||
| -rw-r--r-- | bot/utils/messages.py | 6 | ||||
| -rw-r--r-- | config-default.yml | 5 | ||||
| -rw-r--r-- | tests/bot/exts/backend/sync/test_base.py | 359 | ||||
| -rw-r--r-- | tests/bot/exts/filters/test_token_remover.py | 150 | 
13 files changed, 330 insertions, 639 deletions
| diff --git a/Pipfile.lock b/Pipfile.lock index f75852081..4c63277de 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -86,12 +86,12 @@                  "fakeredis"              ],              "hashes": [ -                "sha256:407aed1aad97bf22f690eca5369806d22eefc8ca104a52c1f1bd47dd6db45fc2", -                "sha256:563aaff79ec611a92a0ad78e39ff159e3a4b4cf0bea41e061de5f3701a17d50c" +                "sha256:6be8a657d724ccbcfb1946d29a80c3478c5f9ecd2f78a0a26d2f4013a622258f", +                "sha256:c25e4fff73f64d20645254783c3224a4c49e083e3fab67c44f17af944c5e26af"              ],              "index": "pypi",              "markers": "python_version ~= '3.7'", -            "version": "==0.1.2" +            "version": "==0.1.4"          },          "async-timeout": {              "hashes": [ @@ -119,12 +119,12 @@          },          "beautifulsoup4": {              "hashes": [ -                "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7", -                "sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8", -                "sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c" +                "sha256:1edf5e39f3a5bc6e38b235b369128416c7239b34f692acccececb040233032a1", +                "sha256:5dfe44f8fddc89ac5453f02659d3ab1668f2c0d9684839f0785037e8c6d9ac8d", +                "sha256:645d833a828722357038299b7f6879940c11dddd95b900fe5387c258b72bb883"              ],              "index": "pypi", -            "version": "==4.9.1" +            "version": "==4.9.2"          },          "certifi": {              "hashes": [ @@ -135,36 +135,44 @@          },          "cffi": {              "hashes": [ -                "sha256:0da50dcbccd7cb7e6c741ab7912b2eff48e85af217d72b57f80ebc616257125e", -                "sha256:12a453e03124069b6896107ee133ae3ab04c624bb10683e1ed1c1663df17c13c", -                "sha256:15419020b0e812b40d96ec9d369b2bc8109cc3295eac6e013d3261343580cc7e", -                "sha256:15a5f59a4808f82d8ec7364cbace851df591c2d43bc76bcbe5c4543a7ddd1bf1", -                "sha256:23e44937d7695c27c66a54d793dd4b45889a81b35c0751ba91040fe825ec59c4", -                "sha256:29c4688ace466a365b85a51dcc5e3c853c1d283f293dfcc12f7a77e498f160d2", -                "sha256:57214fa5430399dffd54f4be37b56fe22cedb2b98862550d43cc085fb698dc2c", -                "sha256:577791f948d34d569acb2d1add5831731c59d5a0c50a6d9f629ae1cefd9ca4a0", -                "sha256:6539314d84c4d36f28d73adc1b45e9f4ee2a89cdc7e5d2b0a6dbacba31906798", -                "sha256:65867d63f0fd1b500fa343d7798fa64e9e681b594e0a07dc934c13e76ee28fb1", -                "sha256:672b539db20fef6b03d6f7a14b5825d57c98e4026401fce838849f8de73fe4d4", -                "sha256:6843db0343e12e3f52cc58430ad559d850a53684f5b352540ca3f1bc56df0731", -                "sha256:7057613efefd36cacabbdbcef010e0a9c20a88fc07eb3e616019ea1692fa5df4", -                "sha256:76ada88d62eb24de7051c5157a1a78fd853cca9b91c0713c2e973e4196271d0c", -                "sha256:837398c2ec00228679513802e3744d1e8e3cb1204aa6ad408b6aff081e99a487", -                "sha256:8662aabfeab00cea149a3d1c2999b0731e70c6b5bac596d95d13f643e76d3d4e", -                "sha256:95e9094162fa712f18b4f60896e34b621df99147c2cee216cfa8f022294e8e9f", -                "sha256:99cc66b33c418cd579c0f03b77b94263c305c389cb0c6972dac420f24b3bf123", -                "sha256:9b219511d8b64d3fa14261963933be34028ea0e57455baf6781fe399c2c3206c", -                "sha256:ae8f34d50af2c2154035984b8b5fc5d9ed63f32fe615646ab435b05b132ca91b", -                "sha256:b9aa9d8818c2e917fa2c105ad538e222a5bce59777133840b93134022a7ce650", -                "sha256:bf44a9a0141a082e89c90e8d785b212a872db793a0080c20f6ae6e2a0ebf82ad", -                "sha256:c0b48b98d79cf795b0916c57bebbc6d16bb43b9fc9b8c9f57f4cf05881904c75", -                "sha256:da9d3c506f43e220336433dffe643fbfa40096d408cb9b7f2477892f369d5f82", -                "sha256:e4082d832e36e7f9b2278bc774886ca8207346b99f278e54c9de4834f17232f7", -                "sha256:e4b9b7af398c32e408c00eb4e0d33ced2f9121fd9fb978e6c1b57edd014a7d15", -                "sha256:e613514a82539fc48291d01933951a13ae93b6b444a88782480be32245ed4afa", -                "sha256:f5033952def24172e60493b68717792e3aebb387a8d186c43c020d9363ee7281" -            ], -            "version": "==1.14.2" +                "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d", +                "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b", +                "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4", +                "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f", +                "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3", +                "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579", +                "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537", +                "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e", +                "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05", +                "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171", +                "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca", +                "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522", +                "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c", +                "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc", +                "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d", +                "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808", +                "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828", +                "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869", +                "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d", +                "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9", +                "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0", +                "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc", +                "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15", +                "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c", +                "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a", +                "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3", +                "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1", +                "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768", +                "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d", +                "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b", +                "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e", +                "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d", +                "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730", +                "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394", +                "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1", +                "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591" +            ], +            "version": "==1.14.3"          },          "chardet": {              "hashes": [ @@ -575,11 +583,11 @@          },          "sentry-sdk": {              "hashes": [ -                "sha256:1a086486ff9da15791f294f6e9915eb3747d161ef64dee2d038a4d0b4a369b24", -                "sha256:45486deb031cea6bbb25a540d7adb4dd48cd8a1cc31e6a5ce9fb4f792a572e9a" +                "sha256:c9c0fa1412bad87104c4eee8dd36c7bbf60b0d92ae917ab519094779b22e6d9a", +                "sha256:e159f7c919d19ae86e5a4ff370fccc45149fab461fbeb93fb5a735a0b33a9cb1"              ],              "index": "pypi", -            "version": "==0.17.6" +            "version": "==0.17.8"          },          "six": {              "hashes": [ @@ -608,7 +616,7 @@                  "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",                  "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"              ], -            "markers": "python_version >= '3.5'", +            "markers": "python_version >= '3.0'",              "version": "==2.0.1"          },          "sphinx": { @@ -685,26 +693,26 @@          },          "yarl": {              "hashes": [ -                "sha256:040b237f58ff7d800e6e0fd89c8439b841f777dd99b4a9cca04d6935564b9409", -                "sha256:17668ec6722b1b7a3a05cc0167659f6c95b436d25a36c2d52db0eca7d3f72593", -                "sha256:3a584b28086bc93c888a6c2aa5c92ed1ae20932f078c46509a66dce9ea5533f2", -                "sha256:4439be27e4eee76c7632c2427ca5e73703151b22cae23e64adb243a9c2f565d8", -                "sha256:48e918b05850fffb070a496d2b5f97fc31d15d94ca33d3d08a4f86e26d4e7c5d", -                "sha256:9102b59e8337f9874638fcfc9ac3734a0cfadb100e47d55c20d0dc6087fb4692", -                "sha256:9b930776c0ae0c691776f4d2891ebc5362af86f152dd0da463a6614074cb1b02", -                "sha256:b3b9ad80f8b68519cc3372a6ca85ae02cc5a8807723ac366b53c0f089db19e4a", -                "sha256:bc2f976c0e918659f723401c4f834deb8a8e7798a71be4382e024bcc3f7e23a8", -                "sha256:c22c75b5f394f3d47105045ea551e08a3e804dc7e01b37800ca35b58f856c3d6", -                "sha256:c52ce2883dc193824989a9b97a76ca86ecd1fa7955b14f87bf367a61b6232511", -                "sha256:ce584af5de8830d8701b8979b18fcf450cef9a382b1a3c8ef189bedc408faf1e", -                "sha256:da456eeec17fa8aa4594d9a9f27c0b1060b6a75f2419fe0c00609587b2695f4a", -                "sha256:db6db0f45d2c63ddb1a9d18d1b9b22f308e52c83638c26b422d520a815c4b3fb", -                "sha256:df89642981b94e7db5596818499c4b2219028f2a528c9c37cc1de45bf2fd3a3f", -                "sha256:f18d68f2be6bf0e89f1521af2b1bb46e66ab0018faafa81d70f358153170a317", -                "sha256:f379b7f83f23fe12823085cd6b906edc49df969eb99757f58ff382349a3303c6" +                "sha256:04a54f126a0732af75e5edc9addeaa2113e2ca7c6fce8974a63549a70a25e50e", +                "sha256:3cc860d72ed989f3b1f3abbd6ecf38e412de722fb38b8f1b1a086315cf0d69c5", +                "sha256:5d84cc36981eb5a8533be79d6c43454c8e6a39ee3118ceaadbd3c029ab2ee580", +                "sha256:5e447e7f3780f44f890360ea973418025e8c0cdcd7d6a1b221d952600fd945dc", +                "sha256:61d3ea3c175fe45f1498af868879c6ffeb989d4143ac542163c45538ba5ec21b", +                "sha256:67c5ea0970da882eaf9efcf65b66792557c526f8e55f752194eff8ec722c75c2", +                "sha256:6f6898429ec3c4cfbef12907047136fd7b9e81a6ee9f105b45505e633427330a", +                "sha256:7ce35944e8e61927a8f4eb78f5bc5d1e6da6d40eadd77e3f79d4e9399e263921", +                "sha256:b7c199d2cbaf892ba0f91ed36d12ff41ecd0dde46cbf64ff4bfe997a3ebc925e", +                "sha256:c15d71a640fb1f8e98a1423f9c64d7f1f6a3a168f803042eaf3a5b5022fde0c1", +                "sha256:c22607421f49c0cb6ff3ed593a49b6a99c6ffdeaaa6c944cdda83c2393c8864d", +                "sha256:c604998ab8115db802cc55cb1b91619b2831a6128a62ca7eea577fc8ea4d3131", +                "sha256:d088ea9319e49273f25b1c96a3763bf19a882cff774d1792ae6fba34bd40550a", +                "sha256:db9eb8307219d7e09b33bcb43287222ef35cbcf1586ba9472b0a4b833666ada1", +                "sha256:e31fef4e7b68184545c3d68baec7074532e077bd1906b040ecfba659737df188", +                "sha256:e32f0fb443afcfe7f01f95172b66f279938fbc6bdaebe294b0ff6747fb6db020", +                "sha256:fcbe419805c9b20db9a51d33b942feddbf6e7fb468cb20686fd7089d4164c12a"              ],              "markers": "python_version >= '3.5'", -            "version": "==1.5.1" +            "version": "==1.6.0"          }      },      "develop": { @@ -857,11 +865,11 @@          },          "identify": {              "hashes": [ -                "sha256:c770074ae1f19e08aadbda1c886bc6d0cb55ffdc503a8c0fe8699af2fc9664ae", -                "sha256:d02d004568c5a01261839a05e91705e3e9f5c57a3551648f9b3fb2b9c62c0f62" +                "sha256:7c22c384a2c9b32c5cc891d13f923f6b2653aa83e2d75d8f79be240d6c86c4f4", +                "sha256:da683bfb7669fa749fc7731f378229e2dbf29a1d1337cbde04106f02236eb29d"              ],              "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", -            "version": "==1.5.3" +            "version": "==1.5.5"          },          "mccabe": {              "hashes": [ diff --git a/bot/constants.py b/bot/constants.py index c710e2dff..c21fd52e0 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -391,6 +391,7 @@ class Channels(metaclass=YAMLGetter):      big_brother_logs: int      bot_commands: int      change_log: int +    code_help_voice: int      cooldown: int      defcon: int      dev_contrib: int @@ -559,13 +560,6 @@ class RedirectOutput(metaclass=YAMLGetter):      delete_delay: int -class Sync(metaclass=YAMLGetter): -    section = 'sync' - -    confirm_timeout: int -    max_diff: int - -  class PythonNews(metaclass=YAMLGetter):      section = 'python_news' diff --git a/bot/exts/backend/sync/_syncers.py b/bot/exts/backend/sync/_syncers.py index ea0f2bcb6..759af96d7 100644 --- a/bot/exts/backend/sync/_syncers.py +++ b/bot/exts/backend/sync/_syncers.py @@ -1,16 +1,13 @@  import abc -import asyncio  import logging  import typing as t  from collections import namedtuple  from functools import partial  from urllib.parse import parse_qsl, urlparse -import discord -from discord import Guild, HTTPException, Member, Message, Reaction, User +from discord import Guild  from discord.ext.commands import Context -from bot import constants  from bot.api import ResponseCodeError  from bot.bot import Bot @@ -26,9 +23,6 @@ _Diff = namedtuple('Diff', ('created', 'updated', 'deleted'))  class Syncer(abc.ABC):      """Base class for synchronising the database with objects in the Discord cache.""" -    _CORE_DEV_MENTION = f"<@&{constants.Roles.core_developers}> " -    _REACTION_EMOJIS = (constants.Emojis.check_mark, constants.Emojis.cross_mark) -      def __init__(self, bot: Bot) -> None:          self.bot = bot @@ -38,112 +32,6 @@ class Syncer(abc.ABC):          """The name of the syncer; used in output messages and logging."""          raise NotImplementedError  # pragma: no cover -    async def _send_prompt(self, message: t.Optional[Message] = None) -> t.Optional[Message]: -        """ -        Send a prompt to confirm or abort a sync using reactions and return the sent message. - -        If a message is given, it is edited to display the prompt and reactions. Otherwise, a new -        message is sent to the dev-core channel and mentions the core developers role. If the -        channel cannot be retrieved, return None. -        """ -        log.trace(f"Sending {self.name} sync confirmation prompt.") - -        msg_content = ( -            f'Possible cache issue while syncing {self.name}s. ' -            f'More than {constants.Sync.max_diff} {self.name}s were changed. ' -            f'React to confirm or abort the sync.' -        ) - -        # Send to core developers if it's an automatic sync. -        if not message: -            log.trace("Message not provided for confirmation; creating a new one in dev-core.") -            channel = self.bot.get_channel(constants.Channels.dev_core) - -            if not channel: -                log.debug("Failed to get the dev-core channel from cache; attempting to fetch it.") -                try: -                    channel = await self.bot.fetch_channel(constants.Channels.dev_core) -                except HTTPException: -                    log.exception( -                        f"Failed to fetch channel for sending sync confirmation prompt; " -                        f"aborting {self.name} sync." -                    ) -                    return None - -            allowed_roles = [discord.Object(constants.Roles.core_developers)] -            message = await channel.send( -                f"{self._CORE_DEV_MENTION}{msg_content}", -                allowed_mentions=discord.AllowedMentions(everyone=False, roles=allowed_roles) -            ) -        else: -            await message.edit(content=msg_content) - -        # Add the initial reactions. -        log.trace(f"Adding reactions to {self.name} syncer confirmation prompt.") -        for emoji in self._REACTION_EMOJIS: -            await message.add_reaction(emoji) - -        return message - -    def _reaction_check( -        self, -        author: Member, -        message: Message, -        reaction: Reaction, -        user: t.Union[Member, User] -    ) -> bool: -        """ -        Return True if the `reaction` is a valid confirmation or abort reaction on `message`. - -        If the `author` of the prompt is a bot, then a reaction by any core developer will be -        considered valid. Otherwise, the author of the reaction (`user`) will have to be the -        `author` of the prompt. -        """ -        # For automatic syncs, check for the core dev role instead of an exact author -        has_role = any(constants.Roles.core_developers == role.id for role in user.roles) -        return ( -            reaction.message.id == message.id -            and not user.bot -            and (has_role if author.bot else user == author) -            and str(reaction.emoji) in self._REACTION_EMOJIS -        ) - -    async def _wait_for_confirmation(self, author: Member, message: Message) -> bool: -        """ -        Wait for a confirmation reaction by `author` on `message` and return True if confirmed. - -        Uses the `_reaction_check` function to determine if a reaction is valid. - -        If there is no reaction within `bot.constants.Sync.confirm_timeout` seconds, return False. -        To acknowledge the reaction (or lack thereof), `message` will be edited. -        """ -        # Preserve the core-dev role mention in the message edits so users aren't confused about -        # where notifications came from. -        mention = self._CORE_DEV_MENTION if author.bot else "" - -        reaction = None -        try: -            log.trace(f"Waiting for a reaction to the {self.name} syncer confirmation prompt.") -            reaction, _ = await self.bot.wait_for( -                'reaction_add', -                check=partial(self._reaction_check, author, message), -                timeout=constants.Sync.confirm_timeout -            ) -        except asyncio.TimeoutError: -            # reaction will remain none thus sync will be aborted in the finally block below. -            log.debug(f"The {self.name} syncer confirmation prompt timed out.") - -        if str(reaction) == constants.Emojis.check_mark: -            log.trace(f"The {self.name} syncer was confirmed.") -            await message.edit(content=f':ok_hand: {mention}{self.name} sync will proceed.') -            return True -        else: -            log.info(f"The {self.name} syncer was aborted or timed out!") -            await message.edit( -                content=f':warning: {mention}{self.name} sync aborted or timed out!' -            ) -            return False -      @abc.abstractmethod      async def _get_diff(self, guild: Guild) -> _Diff:          """Return the difference between the cache of `guild` and the database.""" @@ -154,62 +42,19 @@ class Syncer(abc.ABC):          """Perform the API calls for synchronisation."""          raise NotImplementedError  # pragma: no cover -    async def _get_confirmation_result( -        self, -        diff_size: int, -        author: Member, -        message: t.Optional[Message] = None -    ) -> t.Tuple[bool, t.Optional[Message]]: -        """ -        Prompt for confirmation and return a tuple of the result and the prompt message. - -        `diff_size` is the size of the diff of the sync. If it is greater than -        `bot.constants.Sync.max_diff`, the prompt will be sent. The `author` is the invoked of the -        sync and the `message` is an extant message to edit to display the prompt. - -        If confirmed or no confirmation was needed, the result is True. The returned message will -        either be the given `message` or a new one which was created when sending the prompt. -        """ -        log.trace(f"Determining if confirmation prompt should be sent for {self.name} syncer.") -        if diff_size > constants.Sync.max_diff: -            message = await self._send_prompt(message) -            if not message: -                return False, None  # Couldn't get channel. - -            confirmed = await self._wait_for_confirmation(author, message) -            if not confirmed: -                return False, message  # Sync aborted. - -        return True, message -      async def sync(self, guild: Guild, ctx: t.Optional[Context] = None) -> None:          """          Synchronise the database with the cache of `guild`. -        If the differences between the cache and the database are greater than -        `bot.constants.Sync.max_diff`, then a confirmation prompt will be sent to the dev-core -        channel. The confirmation can be optionally redirect to `ctx` instead. +        If `ctx` is given, send a message with the results.          """          log.info(f"Starting {self.name} syncer.") -        message = None -        author = self.bot.user          if ctx:              message = await ctx.send(f"📊 Synchronising {self.name}s.") -            author = ctx.author - +        else: +            message = None          diff = await self._get_diff(guild) -        diff_dict = diff._asdict()  # Ugly method for transforming the NamedTuple into a dict -        totals = {k: len(v) for k, v in diff_dict.items() if v is not None} -        diff_size = sum(totals.values()) - -        confirmed, message = await self._get_confirmation_result(diff_size, author, message) -        if not confirmed: -            return - -        # Preserve the core-dev role mention in the message edits so users aren't confused about -        # where notifications came from. -        mention = self._CORE_DEV_MENTION if author.bot else ""          try:              await self._sync(diff) @@ -218,11 +63,14 @@ class Syncer(abc.ABC):              # Don't show response text because it's probably some really long HTML.              results = f"status {e.status}\n```{e.response_json or 'See log output for details'}```" -            content = f":x: {mention}Synchronisation of {self.name}s failed: {results}" +            content = f":x: Synchronisation of {self.name}s failed: {results}"          else: -            results = ", ".join(f"{name} `{total}`" for name, total in totals.items()) +            diff_dict = diff._asdict() +            results = (f"{name} `{len(val)}`" for name, val in diff_dict.items() if val is not None) +            results = ", ".join(results) +              log.info(f"{self.name} syncer finished: {results}.") -            content = f":ok_hand: {mention}Synchronisation of {self.name}s complete: {results}" +            content = f":ok_hand: Synchronisation of {self.name}s complete: {results}"          if message:              await message.edit(content=content) diff --git a/bot/exts/filters/token_remover.py b/bot/exts/filters/token_remover.py index ba86e557a..bd6a1f97a 100644 --- a/bot/exts/filters/token_remover.py +++ b/bot/exts/filters/token_remover.py @@ -19,6 +19,11 @@ LOG_MESSAGE = (      "Censored a seemingly valid token sent by {author} in {channel}, "      "token was `{user_id}.{timestamp}.{hmac}`"  ) +UNKNOWN_USER_LOG_MESSAGE = "Decoded user ID: `{user_id}` (Not present in server)." +KNOWN_USER_LOG_MESSAGE = ( +    "Decoded user ID: `{user_id}` **(Present in server)**.\n" +    "This matches `{user_name}` and means this is likely a valid **{kind}** token." +)  DELETION_MESSAGE_TEMPLATE = (      "Hey {mention}! I noticed you posted a seemingly valid Discord API "      "token in your message and have removed your message. " @@ -94,6 +99,7 @@ class TokenRemover(Cog):          await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention))          log_message = self.format_log_message(msg, found_token) +        userid_message, mention_everyone = self.format_userid_log_message(msg, found_token)          log.debug(log_message)          # Send pretty mod log embed to mod-alerts @@ -101,16 +107,41 @@ class TokenRemover(Cog):              icon_url=Icons.token_removed,              colour=Colour(Colours.soft_red),              title="Token removed!", -            text=log_message, +            text=log_message + "\n" + userid_message,              thumbnail=msg.author.avatar_url_as(static_format="png"),              channel_id=Channels.mod_alerts, +            ping_everyone=mention_everyone,          )          self.bot.stats.incr("tokens.removed_tokens") +    @classmethod +    def format_userid_log_message(cls, msg: Message, token: Token) -> t.Tuple[str, bool]: +        """ +        Format the portion of the log message that includes details about the detected user ID. + +        If the user is resolved to a member, the format includes the user ID, name, and the +        kind of user detected. + +        If we resolve to a member and it is not a bot, we also return True to ping everyone. + +        Returns a tuple of (log_message, mention_everyone) +        """ +        user_id = cls.extract_user_id(token.user_id) +        user = msg.guild.get_member(user_id) + +        if user: +            return KNOWN_USER_LOG_MESSAGE.format( +                user_id=user_id, +                user_name=str(user), +                kind="BOT" if user.bot else "USER", +            ), not user.bot +        else: +            return UNKNOWN_USER_LOG_MESSAGE.format(user_id=user_id), False +      @staticmethod      def format_log_message(msg: Message, token: Token) -> str: -        """Return the log message to send for `token` being censored in `msg`.""" +        """Return the generic portion of the log message to send for `token` being censored in `msg`."""          return LOG_MESSAGE.format(              author=format_user(msg.author),              channel=msg.channel.mention, @@ -126,7 +157,11 @@ class TokenRemover(Cog):          # token check (e.g. `message.channel.send` also matches our token pattern)          for match in TOKEN_RE.finditer(msg.content):              token = Token(*match.groups()) -            if cls.is_valid_user_id(token.user_id) and cls.is_valid_timestamp(token.timestamp): +            if ( +                (cls.extract_user_id(token.user_id) is not None) +                and cls.is_valid_timestamp(token.timestamp) +                and cls.is_maybe_valid_hmac(token.hmac) +            ):                  # Short-circuit on first match                  return token @@ -134,22 +169,20 @@ class TokenRemover(Cog):          return      @staticmethod -    def is_valid_user_id(b64_content: str) -> bool: -        """ -        Check potential token to see if it contains a valid Discord user ID. - -        See: https://discordapp.com/developers/docs/reference#snowflakes -        """ +    def extract_user_id(b64_content: str) -> t.Optional[int]: +        """Return a user ID integer from part of a potential token, or None if it couldn't be decoded."""          b64_content = utils.pad_base64(b64_content)          try:              decoded_bytes = base64.urlsafe_b64decode(b64_content)              string = decoded_bytes.decode('utf-8') - -            # isdigit on its own would match a lot of other Unicode characters, hence the isascii. -            return string.isascii() and string.isdigit() +            if not (string.isascii() and string.isdigit()): +                # This case triggers if there are fancy unicode digits in the base64 encoding, +                # that means it's not a valid user id. +                return None +            return int(string)          except (binascii.Error, ValueError): -            return False +            return None      @staticmethod      def is_valid_timestamp(b64_content: str) -> bool: @@ -176,6 +209,24 @@ class TokenRemover(Cog):              log.debug(f"Invalid token timestamp '{b64_content}': smaller than Discord epoch")              return False +    @staticmethod +    def is_maybe_valid_hmac(b64_content: str) -> bool: +        """ +        Determine if a given HMAC portion of a token is potentially valid. + +        If the HMAC has 3 or less characters, it's probably a dummy value like "xxxxxxxxxx", +        and thus the token can probably be skipped. +        """ +        unique = len(set(b64_content.lower())) +        if unique <= 3: +            log.debug( +                f"Considering the HMAC {b64_content} a dummy because it has {unique}" +                " case-insensitively unique characters" +            ) +            return False +        else: +            return True +  def setup(bot: Bot) -> None:      """Load the TokenRemover cog.""" diff --git a/bot/exts/info/information.py b/bot/exts/info/information.py index 156dfec35..f6ed176f1 100644 --- a/bot/exts/info/information.py +++ b/bot/exts/info/information.py @@ -77,7 +77,7 @@ class Information(Cog):          channel_type_list = sorted(channel_type_list)          return "\n".join(channel_type_list) -    @has_any_role(*constants.MODERATION_ROLES) +    @has_any_role(*constants.STAFF_ROLES)      @command(name="roles")      async def roles_info(self, ctx: Context) -> None:          """Returns a list of all roles and their corresponding IDs.""" @@ -97,7 +97,7 @@ class Information(Cog):          await LinePaginator.paginate(role_list, ctx, embed, empty=False) -    @has_any_role(*constants.MODERATION_ROLES) +    @has_any_role(*constants.STAFF_ROLES)      @command(name="role")      async def role_info(self, ctx: Context, *roles: Union[Role, str]) -> None:          """ diff --git a/bot/exts/moderation/incidents.py b/bot/exts/moderation/incidents.py index 31be48a43..0e479d33f 100644 --- a/bot/exts/moderation/incidents.py +++ b/bot/exts/moderation/incidents.py @@ -319,7 +319,7 @@ class Incidents(Cog):          try:              await confirmation_task          except asyncio.TimeoutError: -            log.warning(f"Did not receive incident deletion confirmation within {timeout} seconds!") +            log.info(f"Did not receive incident deletion confirmation within {timeout} seconds!")          else:              log.trace("Deletion was confirmed") diff --git a/bot/exts/moderation/modlog.py b/bot/exts/moderation/modlog.py index 41ed46b69..b01de0ee3 100644 --- a/bot/exts/moderation/modlog.py +++ b/bot/exts/moderation/modlog.py @@ -63,7 +63,7 @@ class ModLog(Cog, name="ModLog"):                          'id': message.id,                          'author': message.author.id,                          'channel_id': message.channel.id, -                        'content': message.content, +                        'content': message.content.replace("\0", ""),  # Null chars cause 400.                          'embeds': [embed.to_dict() for embed in message.embeds],                          'attachments': attachment,                      } diff --git a/bot/exts/moderation/verification.py b/bot/exts/moderation/verification.py index 6bbe81701..206556483 100644 --- a/bot/exts/moderation/verification.py +++ b/bot/exts/moderation/verification.py @@ -21,12 +21,15 @@ log = logging.getLogger(__name__)  # Sent via DMs once user joins the guild  ON_JOIN_MESSAGE = f""" -Hello! Welcome to Python Discord! +Welcome to Python Discord! -As a new user, you have read-only access to a few select channels to give you a taste of what our server is like. +To show you what kind of community we are, we've created this video: +https://youtu.be/ZH26PuX3re0 -In order to see the rest of the channels and to send messages, you first have to accept our rules. To do so, \ -please visit <#{constants.Channels.verification}>. Thank you! +As a new user, you have read-only access to a few select channels to give you a taste of what our server is like. \ +In order to see the rest of the channels and to send messages, you first have to accept our rules. + +Please visit <#{constants.Channels.verification}> to get started. Thank you!  """  # Sent via DMs once user verifies @@ -106,6 +109,25 @@ def is_verified(member: discord.Member) -> bool:      return len(set(member.roles) - unverified_roles) > 0 +async def safe_dm(coro: t.Coroutine) -> None: +    """ +    Execute `coro` ignoring disabled DM warnings. + +    The 50_0007 error code indicates that the target user does not accept DMs. +    As it turns out, this error code can appear on both 400 and 403 statuses, +    we therefore catch any Discord exception. + +    If the request fails on any other error code, the exception propagates, +    and must be handled by the caller. +    """ +    try: +        await coro +    except discord.HTTPException as discord_exc: +        log.trace(f"DM dispatch failed on status {discord_exc.status} with code: {discord_exc.code}") +        if discord_exc.code != 50_007:  # If any reason other than disabled DMs +            raise + +  class Verification(Cog):      """      User verification and role management. @@ -327,11 +349,9 @@ class Verification(Cog):          async def kick_request(member: discord.Member) -> None:              """Send `KICKED_MESSAGE` to `member` and kick them from the guild."""              try: -                await member.send(KICKED_MESSAGE) -            except discord.Forbidden as exc_403: -                log.trace(f"DM dispatch failed on 403 error with code: {exc_403.code}") -                if exc_403.code != 50_007:  # 403 raised for any other reason than disabled DMs -                    raise StopExecution(reason=exc_403) +                await safe_dm(member.send(KICKED_MESSAGE))  # Suppress disabled DMs +            except discord.HTTPException as suspicious_exception: +                raise StopExecution(reason=suspicious_exception)              await member.kick(reason=f"User has not verified in {constants.Verification.kicked_after} days")          n_kicked = await self._send_requests(members, kick_request, Limit(batch_size=2, sleep_secs=1)) @@ -500,8 +520,10 @@ class Verification(Cog):              return  # Only listen for PyDis events          log.trace(f"Sending on join message to new member: {member.id}") -        with suppress(discord.Forbidden): -            await member.send(ON_JOIN_MESSAGE) +        try: +            await safe_dm(member.send(ON_JOIN_MESSAGE)) +        except discord.HTTPException: +            log.exception("DM dispatch failed on unexpected error code")      @Cog.listener()      async def on_message(self, message: discord.Message) -> None: @@ -668,9 +690,9 @@ class Verification(Cog):              await ctx.author.remove_roles(discord.Object(constants.Roles.unverified))          try: -            await ctx.author.send(VERIFIED_MESSAGE) -        except discord.Forbidden: -            log.info(f"Sending welcome message failed for {ctx.author}.") +            await safe_dm(ctx.author.send(VERIFIED_MESSAGE)) +        except discord.HTTPException: +            log.exception(f"Sending welcome message failed for {ctx.author}.")          finally:              log.trace(f"Deleting accept message by {ctx.author}.")              with suppress(discord.NotFound): diff --git a/bot/exts/utils/snekbox.py b/bot/exts/utils/snekbox.py index b3baffba2..18b9a5014 100644 --- a/bot/exts/utils/snekbox.py +++ b/bot/exts/utils/snekbox.py @@ -41,7 +41,7 @@ RAW_CODE_REGEX = re.compile(  MAX_PASTE_LEN = 1000  # `!eval` command whitelists -EVAL_CHANNELS = (Channels.bot_commands, Channels.esoteric) +EVAL_CHANNELS = (Channels.bot_commands, Channels.esoteric, Channels.code_help_voice)  EVAL_CATEGORIES = (Categories.help_available, Categories.help_in_use)  EVAL_ROLES = (Roles.helpers, Roles.moderators, Roles.admins, Roles.owners, Roles.python_community, Roles.partners) diff --git a/bot/utils/messages.py b/bot/utils/messages.py index 9cc0d8a34..d0b2342b3 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -34,7 +34,11 @@ async def wait_for_deletion(      if attach_emojis:          for emoji in deletion_emojis: -            await message.add_reaction(emoji) +            try: +                await message.add_reaction(emoji) +            except discord.NotFound: +                log.trace(f"Aborting wait_for_deletion: message {message.id} deleted prematurely.") +                return      def check(reaction: discord.Reaction, user: discord.Member) -> bool:          """Check that the deletion emoji is reacted by the appropriate user.""" diff --git a/config-default.yml b/config-default.yml index e7669e6db..4f7b1e217 100644 --- a/config-default.yml +++ b/config-default.yml @@ -190,6 +190,7 @@ guild:          admin_announcements:    &ADMIN_ANNOUNCEMENTS    749736155569848370          # Voice +        code_help_voice:                    755154969761677312          admins_voice:       &ADMINS_VOICE   500734494840717332          staff_voice:        &STAFF_VOICE    412375055910043655 @@ -445,10 +446,6 @@ redirect_output:      delete_invocation: true      delete_delay: 15 -sync: -    confirm_timeout: 300 -    max_diff: 10 -  duck_pond:      threshold: 4      channel_blacklist: diff --git a/tests/bot/exts/backend/sync/test_base.py b/tests/bot/exts/backend/sync/test_base.py index 886c243cf..4953550f9 100644 --- a/tests/bot/exts/backend/sync/test_base.py +++ b/tests/bot/exts/backend/sync/test_base.py @@ -1,12 +1,9 @@ -import asyncio  import unittest  from unittest import mock -import discord -from bot import constants  from bot.api import ResponseCodeError -from bot.exts.backend.sync._syncers import Syncer, _Diff +from bot.exts.backend.sync._syncers import Syncer  from tests import helpers @@ -30,280 +27,16 @@ class SyncerBaseTests(unittest.TestCase):              Syncer(self.bot) -class SyncerSendPromptTests(unittest.IsolatedAsyncioTestCase): -    """Tests for sending the sync confirmation prompt.""" - -    def setUp(self): -        self.bot = helpers.MockBot() -        self.syncer = TestSyncer(self.bot) - -    def mock_get_channel(self): -        """Fixture to return a mock channel and message for when `get_channel` is used.""" -        self.bot.reset_mock() - -        mock_channel = helpers.MockTextChannel() -        mock_message = helpers.MockMessage() - -        mock_channel.send.return_value = mock_message -        self.bot.get_channel.return_value = mock_channel - -        return mock_channel, mock_message - -    def mock_fetch_channel(self): -        """Fixture to return a mock channel and message for when `fetch_channel` is used.""" -        self.bot.reset_mock() - -        mock_channel = helpers.MockTextChannel() -        mock_message = helpers.MockMessage() - -        self.bot.get_channel.return_value = None -        mock_channel.send.return_value = mock_message -        self.bot.fetch_channel.return_value = mock_channel - -        return mock_channel, mock_message - -    async def test_send_prompt_edits_and_returns_message(self): -        """The given message should be edited to display the prompt and then should be returned.""" -        msg = helpers.MockMessage() -        ret_val = await self.syncer._send_prompt(msg) - -        msg.edit.assert_called_once() -        self.assertIn("content", msg.edit.call_args[1]) -        self.assertEqual(ret_val, msg) - -    async def test_send_prompt_gets_dev_core_channel(self): -        """The dev-core channel should be retrieved if an extant message isn't given.""" -        subtests = ( -            (self.bot.get_channel, self.mock_get_channel), -            (self.bot.fetch_channel, self.mock_fetch_channel), -        ) - -        for method, mock_ in subtests: -            with self.subTest(method=method, msg=mock_.__name__): -                mock_() -                await self.syncer._send_prompt() - -                method.assert_called_once_with(constants.Channels.dev_core) - -    async def test_send_prompt_returns_none_if_channel_fetch_fails(self): -        """None should be returned if there's an HTTPException when fetching the channel.""" -        self.bot.get_channel.return_value = None -        self.bot.fetch_channel.side_effect = discord.HTTPException(mock.MagicMock(), "test error!") - -        ret_val = await self.syncer._send_prompt() - -        self.assertIsNone(ret_val) - -    async def test_send_prompt_sends_and_returns_new_message_if_not_given(self): -        """A new message mentioning core devs should be sent and returned if message isn't given.""" -        for mock_ in (self.mock_get_channel, self.mock_fetch_channel): -            with self.subTest(msg=mock_.__name__): -                mock_channel, mock_message = mock_() -                ret_val = await self.syncer._send_prompt() - -                mock_channel.send.assert_called_once() -                self.assertIn(self.syncer._CORE_DEV_MENTION, mock_channel.send.call_args[0][0]) -                self.assertEqual(ret_val, mock_message) - -    async def test_send_prompt_adds_reactions(self): -        """The message should have reactions for confirmation added.""" -        extant_message = helpers.MockMessage() -        subtests = ( -            (extant_message, lambda: (None, extant_message)), -            (None, self.mock_get_channel), -            (None, self.mock_fetch_channel), -        ) - -        for message_arg, mock_ in subtests: -            subtest_msg = "Extant message" if mock_.__name__ == "<lambda>" else mock_.__name__ - -            with self.subTest(msg=subtest_msg): -                _, mock_message = mock_() -                await self.syncer._send_prompt(message_arg) - -                calls = [mock.call(emoji) for emoji in self.syncer._REACTION_EMOJIS] -                mock_message.add_reaction.assert_has_calls(calls) - - -class SyncerConfirmationTests(unittest.IsolatedAsyncioTestCase): -    """Tests for waiting for a sync confirmation reaction on the prompt.""" - -    def setUp(self): -        self.bot = helpers.MockBot() -        self.syncer = TestSyncer(self.bot) -        self.core_dev_role = helpers.MockRole(id=constants.Roles.core_developers) - -    @staticmethod -    def get_message_reaction(emoji): -        """Fixture to return a mock message an reaction from the given `emoji`.""" -        message = helpers.MockMessage() -        reaction = helpers.MockReaction(emoji=emoji, message=message) - -        return message, reaction - -    def test_reaction_check_for_valid_emoji_and_authors(self): -        """Should return True if authors are identical or are a bot and a core dev, respectively.""" -        user_subtests = ( -            ( -                helpers.MockMember(id=77), -                helpers.MockMember(id=77), -                "identical users", -            ), -            ( -                helpers.MockMember(id=77, bot=True), -                helpers.MockMember(id=43, roles=[self.core_dev_role]), -                "bot author and core-dev reactor", -            ), -        ) - -        for emoji in self.syncer._REACTION_EMOJIS: -            for author, user, msg in user_subtests: -                with self.subTest(author=author, user=user, emoji=emoji, msg=msg): -                    message, reaction = self.get_message_reaction(emoji) -                    ret_val = self.syncer._reaction_check(author, message, reaction, user) - -                    self.assertTrue(ret_val) - -    def test_reaction_check_for_invalid_reactions(self): -        """Should return False for invalid reaction events.""" -        valid_emoji = self.syncer._REACTION_EMOJIS[0] -        subtests = ( -            ( -                helpers.MockMember(id=77), -                *self.get_message_reaction(valid_emoji), -                helpers.MockMember(id=43, roles=[self.core_dev_role]), -                "users are not identical", -            ), -            ( -                helpers.MockMember(id=77, bot=True), -                *self.get_message_reaction(valid_emoji), -                helpers.MockMember(id=43), -                "reactor lacks the core-dev role", -            ), -            ( -                helpers.MockMember(id=77, bot=True, roles=[self.core_dev_role]), -                *self.get_message_reaction(valid_emoji), -                helpers.MockMember(id=77, bot=True, roles=[self.core_dev_role]), -                "reactor is a bot", -            ), -            ( -                helpers.MockMember(id=77), -                helpers.MockMessage(id=95), -                helpers.MockReaction(emoji=valid_emoji, message=helpers.MockMessage(id=26)), -                helpers.MockMember(id=77), -                "messages are not identical", -            ), -            ( -                helpers.MockMember(id=77), -                *self.get_message_reaction("InVaLiD"), -                helpers.MockMember(id=77), -                "emoji is invalid", -            ), -        ) - -        for *args, msg in subtests: -            kwargs = dict(zip(("author", "message", "reaction", "user"), args)) -            with self.subTest(**kwargs, msg=msg): -                ret_val = self.syncer._reaction_check(*args) -                self.assertFalse(ret_val) - -    async def test_wait_for_confirmation(self): -        """The message should always be edited and only return True if the emoji is a check mark.""" -        subtests = ( -            (constants.Emojis.check_mark, True, None), -            ("InVaLiD", False, None), -            (None, False, asyncio.TimeoutError), -        ) - -        for emoji, ret_val, side_effect in subtests: -            for bot in (True, False): -                with self.subTest(emoji=emoji, ret_val=ret_val, side_effect=side_effect, bot=bot): -                    # Set up mocks -                    message = helpers.MockMessage() -                    member = helpers.MockMember(bot=bot) - -                    self.bot.wait_for.reset_mock() -                    self.bot.wait_for.return_value = (helpers.MockReaction(emoji=emoji), None) -                    self.bot.wait_for.side_effect = side_effect - -                    # Call the function -                    actual_return = await self.syncer._wait_for_confirmation(member, message) - -                    # Perform assertions -                    self.bot.wait_for.assert_called_once() -                    self.assertIn("reaction_add", self.bot.wait_for.call_args[0]) - -                    message.edit.assert_called_once() -                    kwargs = message.edit.call_args[1] -                    self.assertIn("content", kwargs) - -                    # Core devs should only be mentioned if the author is a bot. -                    if bot: -                        self.assertIn(self.syncer._CORE_DEV_MENTION, kwargs["content"]) -                    else: -                        self.assertNotIn(self.syncer._CORE_DEV_MENTION, kwargs["content"]) - -                    self.assertIs(actual_return, ret_val) - -  class SyncerSyncTests(unittest.IsolatedAsyncioTestCase):      """Tests for main function orchestrating the sync."""      def setUp(self):          self.bot = helpers.MockBot(user=helpers.MockMember(bot=True))          self.syncer = TestSyncer(self.bot) +        self.guild = helpers.MockGuild() -    async def test_sync_respects_confirmation_result(self): -        """The sync should abort if confirmation fails and continue if confirmed.""" -        mock_message = helpers.MockMessage() -        subtests = ( -            (True, mock_message), -            (False, None), -        ) - -        for confirmed, message in subtests: -            with self.subTest(confirmed=confirmed): -                self.syncer._sync.reset_mock() -                self.syncer._get_diff.reset_mock() - -                diff = _Diff({1, 2, 3}, {4, 5}, None) -                self.syncer._get_diff.return_value = diff -                self.syncer._get_confirmation_result = mock.AsyncMock( -                    return_value=(confirmed, message) -                ) - -                guild = helpers.MockGuild() -                await self.syncer.sync(guild) - -                self.syncer._get_diff.assert_called_once_with(guild) -                self.syncer._get_confirmation_result.assert_called_once() - -                if confirmed: -                    self.syncer._sync.assert_called_once_with(diff) -                else: -                    self.syncer._sync.assert_not_called() - -    async def test_sync_diff_size(self): -        """The diff size should be correctly calculated.""" -        subtests = ( -            (6, _Diff({1, 2}, {3, 4}, {5, 6})), -            (5, _Diff({1, 2, 3}, None, {4, 5})), -            (0, _Diff(None, None, None)), -            (0, _Diff(set(), set(), set())), -        ) - -        for size, diff in subtests: -            with self.subTest(size=size, diff=diff): -                self.syncer._get_diff.reset_mock() -                self.syncer._get_diff.return_value = diff -                self.syncer._get_confirmation_result = mock.AsyncMock(return_value=(False, None)) - -                guild = helpers.MockGuild() -                await self.syncer.sync(guild) - -                self.syncer._get_diff.assert_called_once_with(guild) -                self.syncer._get_confirmation_result.assert_called_once() -                self.assertEqual(self.syncer._get_confirmation_result.call_args[0][0], size) +        # Make sure `_get_diff` returns a MagicMock, not an AsyncMock +        self.syncer._get_diff.return_value = mock.MagicMock()      async def test_sync_message_edited(self):          """The message should be edited if one was sent, even if the sync has an API error.""" @@ -316,89 +49,25 @@ class SyncerSyncTests(unittest.IsolatedAsyncioTestCase):          for message, side_effect, should_edit in subtests:              with self.subTest(message=message, side_effect=side_effect, should_edit=should_edit):                  self.syncer._sync.side_effect = side_effect -                self.syncer._get_confirmation_result = mock.AsyncMock( -                    return_value=(True, message) -                ) +                ctx = helpers.MockContext() +                ctx.send.return_value = message -                guild = helpers.MockGuild() -                await self.syncer.sync(guild) +                await self.syncer.sync(self.guild, ctx)                  if should_edit:                      message.edit.assert_called_once()                      self.assertIn("content", message.edit.call_args[1]) -    async def test_sync_confirmation_context_redirect(self): -        """If ctx is given, a new message should be sent and author should be ctx's author.""" -        mock_member = helpers.MockMember() +    async def test_sync_message_sent(self): +        """If ctx is given, a new message should be sent."""          subtests = ( -            (None, self.bot.user, None), -            (helpers.MockContext(author=mock_member), mock_member, helpers.MockMessage()), +            (None, None), +            (helpers.MockContext(), helpers.MockMessage()),          ) -        for ctx, author, message in subtests: -            with self.subTest(ctx=ctx, author=author, message=message): -                if ctx is not None: -                    ctx.send.return_value = message - -                # Make sure `_get_diff` returns a MagicMock, not an AsyncMock -                self.syncer._get_diff.return_value = mock.MagicMock() - -                self.syncer._get_confirmation_result = mock.AsyncMock(return_value=(False, None)) - -                guild = helpers.MockGuild() -                await self.syncer.sync(guild, ctx) +        for ctx, message in subtests: +            with self.subTest(ctx=ctx, message=message): +                await self.syncer.sync(self.guild, ctx)                  if ctx is not None:                      ctx.send.assert_called_once() - -                self.syncer._get_confirmation_result.assert_called_once() -                self.assertEqual(self.syncer._get_confirmation_result.call_args[0][1], author) -                self.assertEqual(self.syncer._get_confirmation_result.call_args[0][2], message) - -    @mock.patch.object(constants.Sync, "max_diff", new=3) -    async def test_confirmation_result_small_diff(self): -        """Should always return True and the given message if the diff size is too small.""" -        author = helpers.MockMember() -        expected_message = helpers.MockMessage() - -        for size in (3, 2):  # pragma: no cover -            with self.subTest(size=size): -                self.syncer._send_prompt = mock.AsyncMock() -                self.syncer._wait_for_confirmation = mock.AsyncMock() - -                coro = self.syncer._get_confirmation_result(size, author, expected_message) -                result, actual_message = await coro - -                self.assertTrue(result) -                self.assertEqual(actual_message, expected_message) -                self.syncer._send_prompt.assert_not_called() -                self.syncer._wait_for_confirmation.assert_not_called() - -    @mock.patch.object(constants.Sync, "max_diff", new=3) -    async def test_confirmation_result_large_diff(self): -        """Should return True if confirmed and False if _send_prompt fails or aborted.""" -        author = helpers.MockMember() -        mock_message = helpers.MockMessage() - -        subtests = ( -            (True, mock_message, True, "confirmed"), -            (False, None, False, "_send_prompt failed"), -            (False, mock_message, False, "aborted"), -        ) - -        for expected_result, expected_message, confirmed, msg in subtests:  # pragma: no cover -            with self.subTest(msg=msg): -                self.syncer._send_prompt = mock.AsyncMock(return_value=expected_message) -                self.syncer._wait_for_confirmation = mock.AsyncMock(return_value=confirmed) - -                coro = self.syncer._get_confirmation_result(4, author) -                actual_result, actual_message = await coro - -                self.syncer._send_prompt.assert_called_once_with(None)  # message defaults to None -                self.assertIs(actual_result, expected_result) -                self.assertEqual(actual_message, expected_message) - -                if expected_message: -                    self.syncer._wait_for_confirmation.assert_called_once_with( -                        author, expected_message -                    ) diff --git a/tests/bot/exts/filters/test_token_remover.py b/tests/bot/exts/filters/test_token_remover.py index ea822053b..f99cc3370 100644 --- a/tests/bot/exts/filters/test_token_remover.py +++ b/tests/bot/exts/filters/test_token_remover.py @@ -23,23 +23,25 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase):          self.msg = MockMessage(id=555, content="hello world")          self.msg.channel.mention = "#lemonade-stand" +        self.msg.guild.get_member.return_value.bot = False +        self.msg.guild.get_member.return_value.__str__.return_value = "Woody"          self.msg.author.__str__ = MagicMock(return_value=self.msg.author.name)          self.msg.author.avatar_url_as.return_value = "picture-lemon.png" -    def test_is_valid_user_id_valid(self): -        """Should consider user IDs valid if they decode entirely to ASCII digits.""" -        ids = ( -            "NDcyMjY1OTQzMDYyNDEzMzMy", -            "NDc1MDczNjI5Mzk5NTQ3OTA0", -            "NDY3MjIzMjMwNjUwNzc3NjQx", +    def test_extract_user_id_valid(self): +        """Should consider user IDs valid if they decode into an integer ID.""" +        id_pairs = ( +            ("NDcyMjY1OTQzMDYyNDEzMzMy", 472265943062413332), +            ("NDc1MDczNjI5Mzk5NTQ3OTA0", 475073629399547904), +            ("NDY3MjIzMjMwNjUwNzc3NjQx", 467223230650777641),          ) -        for user_id in ids: -            with self.subTest(user_id=user_id): -                result = TokenRemover.is_valid_user_id(user_id) -                self.assertTrue(result) +        for token_id, user_id in id_pairs: +            with self.subTest(token_id=token_id): +                result = TokenRemover.extract_user_id(token_id) +                self.assertEqual(result, user_id) -    def test_is_valid_user_id_invalid(self): +    def test_extract_user_id_invalid(self):          """Should consider non-digit and non-ASCII IDs invalid."""          ids = (              ("SGVsbG8gd29ybGQ", "non-digit ASCII"), @@ -53,8 +55,8 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase):          for user_id, msg in ids:              with self.subTest(msg=msg): -                result = TokenRemover.is_valid_user_id(user_id) -                self.assertFalse(result) +                result = TokenRemover.extract_user_id(user_id) +                self.assertIsNone(result)      def test_is_valid_timestamp_valid(self):          """Should consider timestamps valid if they're greater than the Discord epoch.""" @@ -86,6 +88,34 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase):                  result = TokenRemover.is_valid_timestamp(timestamp)                  self.assertFalse(result) +    def test_is_valid_hmac_valid(self): +        """Should consider an HMAC valid if it has at least 3 unique characters.""" +        valid_hmacs = ( +            "VXmErH7j511turNpfURmb0rVNm8", +            "Ysnu2wacjaKs7qnoo46S8Dm2us8", +            "sJf6omBPORBPju3WJEIAcwW9Zds", +            "s45jqDV_Iisn-symw0yDRrk_jf4", +        ) + +        for hmac in valid_hmacs: +            with self.subTest(msg=hmac): +                result = TokenRemover.is_maybe_valid_hmac(hmac) +                self.assertTrue(result) + +    def test_is_invalid_hmac_invalid(self): +        """Should consider an HMAC invalid if has fewer than 3 unique characters.""" +        invalid_hmacs = ( +            ("xxxxxxxxxxxxxxxxxx", "Single character"), +            ("XxXxXxXxXxXxXxXxXx", "Single character alternating case"), +            ("ASFasfASFasfASFASsf", "Three characters alternating-case"), +            ("asdasdasdasdasdasdasd", "Three characters one case"), +        ) + +        for hmac, msg in invalid_hmacs: +            with self.subTest(msg=msg): +                result = TokenRemover.is_maybe_valid_hmac(hmac) +                self.assertFalse(result) +      def test_mod_log_property(self):          """The `mod_log` property should ask the bot to return the `ModLog` cog."""          self.bot.get_cog.return_value = 'lemon' @@ -143,11 +173,18 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase):          self.assertIsNone(return_value)          token_re.finditer.assert_called_once_with(self.msg.content) -    @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") +    @autospec(TokenRemover, "extract_user_id", "is_valid_timestamp", "is_maybe_valid_hmac")      @autospec("bot.exts.filters.token_remover", "Token")      @autospec("bot.exts.filters.token_remover", "TOKEN_RE") -    def test_find_token_valid_match(self, token_re, token_cls, is_valid_id, is_valid_timestamp): -        """The first match with a valid user ID and timestamp should be returned as a `Token`.""" +    def test_find_token_valid_match( +        self, +        token_re, +        token_cls, +        extract_user_id, +        is_valid_timestamp, +        is_maybe_valid_hmac, +    ): +        """The first match with a valid user ID, timestamp, and HMAC should be returned as a `Token`."""          matches = [              mock.create_autospec(Match, spec_set=True, instance=True),              mock.create_autospec(Match, spec_set=True, instance=True), @@ -159,23 +196,32 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase):          token_re.finditer.return_value = matches          token_cls.side_effect = tokens -        is_valid_id.side_effect = (False, True)  # The 1st match will be invalid, 2nd one valid. +        extract_user_id.side_effect = (None, True)  # The 1st match will be invalid, 2nd one valid.          is_valid_timestamp.return_value = True +        is_maybe_valid_hmac.return_value = True          return_value = TokenRemover.find_token_in_message(self.msg)          self.assertEqual(tokens[1], return_value)          token_re.finditer.assert_called_once_with(self.msg.content) -    @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") +    @autospec(TokenRemover, "extract_user_id", "is_valid_timestamp", "is_maybe_valid_hmac")      @autospec("bot.exts.filters.token_remover", "Token")      @autospec("bot.exts.filters.token_remover", "TOKEN_RE") -    def test_find_token_invalid_matches(self, token_re, token_cls, is_valid_id, is_valid_timestamp): -        """None should be returned if no matches have valid user IDs or timestamps.""" +    def test_find_token_invalid_matches( +        self, +        token_re, +        token_cls, +        extract_user_id, +        is_valid_timestamp, +        is_maybe_valid_hmac, +    ): +        """None should be returned if no matches have valid user IDs, HMACs, and timestamps."""          token_re.finditer.return_value = [mock.create_autospec(Match, spec_set=True, instance=True)]          token_cls.return_value = mock.create_autospec(Token, spec_set=True, instance=True) -        is_valid_id.return_value = False +        extract_user_id.return_value = None          is_valid_timestamp.return_value = False +        is_maybe_valid_hmac.return_value = False          return_value = TokenRemover.find_token_in_message(self.msg) @@ -234,7 +280,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase):      @autospec("bot.exts.filters.token_remover", "LOG_MESSAGE")      def test_format_log_message(self, log_message):          """Should correctly format the log message with info from the message and token.""" -        token = Token("NDY3MjIzMjMwNjUwNzc3NjQx", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") +        token = Token("NDcyMjY1OTQzMDYyNDEzMzMy", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4")          log_message.format.return_value = "Howdy"          return_value = TokenRemover.format_log_message(self.msg, token) @@ -248,18 +294,68 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase):              hmac="x" * len(token.hmac),          ) +    @autospec("bot.exts.filters.token_remover", "UNKNOWN_USER_LOG_MESSAGE") +    def test_format_userid_log_message_unknown(self, unknown_user_log_message): +        """Should correctly format the user ID portion when the actual user it belongs to is unknown.""" +        token = Token("NDcyMjY1OTQzMDYyNDEzMzMy", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") +        unknown_user_log_message.format.return_value = " Partner" +        msg = MockMessage(id=555, content="hello world") +        msg.guild.get_member.return_value = None + +        return_value = TokenRemover.format_userid_log_message(msg, token) + +        self.assertEqual(return_value, (unknown_user_log_message.format.return_value, False)) +        unknown_user_log_message.format.assert_called_once_with(user_id=472265943062413332) + +    @autospec("bot.exts.filters.token_remover", "KNOWN_USER_LOG_MESSAGE") +    def test_format_userid_log_message_bot(self, known_user_log_message): +        """Should correctly format the user ID portion when the ID belongs to a known bot.""" +        token = Token("NDcyMjY1OTQzMDYyNDEzMzMy", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") +        known_user_log_message.format.return_value = " Partner" +        msg = MockMessage(id=555, content="hello world") +        msg.guild.get_member.return_value.__str__.return_value = "Sam" +        msg.guild.get_member.return_value.bot = True + +        return_value = TokenRemover.format_userid_log_message(msg, token) + +        self.assertEqual(return_value, (known_user_log_message.format.return_value, False)) + +        known_user_log_message.format.assert_called_once_with( +            user_id=472265943062413332, +            user_name="Sam", +            kind="BOT", +        ) + +    @autospec("bot.exts.filters.token_remover", "KNOWN_USER_LOG_MESSAGE") +    def test_format_log_message_user_token_user(self, user_token_message): +        """Should correctly format the user ID portion when the ID belongs to a known user.""" +        token = Token("NDY3MjIzMjMwNjUwNzc3NjQx", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") +        user_token_message.format.return_value = "Partner" + +        return_value = TokenRemover.format_userid_log_message(self.msg, token) + +        self.assertEqual(return_value, (user_token_message.format.return_value, True)) +        user_token_message.format.assert_called_once_with( +            user_id=467223230650777641, +            user_name="Woody", +            kind="USER", +        ) +      @mock.patch.object(TokenRemover, "mod_log", new_callable=mock.PropertyMock)      @autospec("bot.exts.filters.token_remover", "log") -    @autospec(TokenRemover, "format_log_message") -    async def test_take_action(self, format_log_message, logger, mod_log_property): +    @autospec(TokenRemover, "format_log_message", "format_userid_log_message") +    async def test_take_action(self, format_log_message, format_userid_log_message, logger, mod_log_property):          """Should delete the message and send a mod log."""          cog = TokenRemover(self.bot)          mod_log = mock.create_autospec(ModLog, spec_set=True, instance=True)          token = mock.create_autospec(Token, spec_set=True, instance=True) +        token.user_id = "no-id"          log_msg = "testing123" +        userid_log_message = "userid-log-message"          mod_log_property.return_value = mod_log          format_log_message.return_value = log_msg +        format_userid_log_message.return_value = (userid_log_message, True)          await cog.take_action(self.msg, token) @@ -269,6 +365,7 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase):          )          format_log_message.assert_called_once_with(self.msg, token) +        format_userid_log_message.assert_called_once_with(self.msg, token)          logger.debug.assert_called_with(log_msg)          self.bot.stats.incr.assert_called_once_with("tokens.removed_tokens") @@ -277,9 +374,10 @@ class TokenRemoverTests(unittest.IsolatedAsyncioTestCase):              icon_url=constants.Icons.token_removed,              colour=Colour(constants.Colours.soft_red),              title="Token removed!", -            text=log_msg, +            text=log_msg + "\n" + userid_log_message,              thumbnail=self.msg.author.avatar_url_as.return_value, -            channel_id=constants.Channels.mod_alerts +            channel_id=constants.Channels.mod_alerts, +            ping_everyone=True,          )      @mock.patch.object(TokenRemover, "mod_log", new_callable=mock.PropertyMock) | 
