diff options
146 files changed, 11177 insertions, 5710 deletions
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..8760b35ec --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,32 @@ +name: "Code scanning - action" + +on: + push: + pull_request: + schedule: + - cron: '0 12 * * *' + +jobs: + CodeQL-Build: + + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + fetch-depth: 2 + + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: python + + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 @@ -4,26 +4,32 @@ verify_ssl = true name = "pypi" [packages] -discord-py = "~=1.3.2" +aio-pika = "~=6.1" aiodns = "~=2.0" aiohttp = "~=3.5" -sphinx = "~=2.2" -markdownify = "~=0.4" -lxml = "~=4.4" -pyyaml = "~=5.1" +aioping = "~=0.3.1" +aioredis = "~=1.3.1" +"async-rediscache[fakeredis]" = "~=0.1.2" +beautifulsoup4 = "~=4.9" +colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} +coloredlogs = "~=14.0" +deepdiff = "~=4.0" +discord.py = "~=1.4.0" +feedparser = "~=5.2" fuzzywuzzy = "~=0.17" -aio-pika = "~=6.1" +lxml = "~=4.4" +markdownify = "~=0.4" +more_itertools = "~=8.2" python-dateutil = "~=2.8" -deepdiff = "~=4.0" +pyyaml = "~=5.1" requests = "~=2.22" -more_itertools = "~=8.2" sentry-sdk = "~=0.14" -coloredlogs = "~=14.0" -colorama = {version = "~=0.4.3", sys_platform = "== 'win32'"} +sphinx = "~=2.2" +statsd = "~=3.3" [dev-packages] coverage = "~=5.0" -flake8 = "~=3.7" +flake8 = "~=3.8" flake8-annotations = "~=2.0" flake8-bugbear = "~=20.1" flake8-docstrings = "~=1.4" @@ -45,4 +51,5 @@ precommit = "pre-commit install" build = "docker build -t pythondiscord/bot:latest -f Dockerfile ." push = "docker push pythondiscord/bot:latest" test = "coverage run -m unittest" +html = "coverage html" report = "coverage report" diff --git a/Pipfile.lock b/Pipfile.lock index ad9a3173a..4c63277de 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "2d3ba484e8467a115126b2ba39fa5f36f103ea455477813dd658797875c79cc9" + "sha256": "644012a1c3fa3e3a30f8b8f8e672c468dfaa155d9e43d26e2be8713c8dc5ebb3" }, "pipfile-spec": 6, "requires": { @@ -18,11 +18,11 @@ "default": { "aio-pika": { "hashes": [ - "sha256:9e4614636296e0040055bd6b304e97a38cc9796669ef391fc9b36649831d43ee", - "sha256:c9d242b3c7142d64b185feb6c5cce4154962610e89ec2e9b52bd69ef01f89b2f" + "sha256:4a20d4d941e1f113a950ea529a90bd9159c8d7aafaa1c71e9c707c8c2b526ea6", + "sha256:7bf3f183df1eb348d007210a0c1a3c5c755f1b3def1a9a395e93f30b91da1daf" ], "index": "pypi", - "version": "==6.6.0" + "version": "==6.7.0" }, "aiodns": { "hashes": [ @@ -50,12 +50,29 @@ "index": "pypi", "version": "==3.6.2" }, + "aioping": { + "hashes": [ + "sha256:8900ef2f5a589ba0c12aaa9c2d586f5371820d468d21b374ddb47ef5fc8f297c", + "sha256:f983d86acab3a04c322731ce88d42c55d04d2842565fc8532fe10c838abfd275" + ], + "index": "pypi", + "version": "==0.3.1" + }, + "aioredis": { + "hashes": [ + "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a", + "sha256:b61808d7e97b7cd5a92ed574937a079c9387fdadd22bfbfa7ad2fd319ecc26e3" + ], + "index": "pypi", + "version": "==1.3.1" + }, "aiormq": { "hashes": [ - "sha256:286e0b0772075580466e45f98f051b9728a9316b9c36f0c14c7bc1409be375b0", - "sha256:7ed7d6df6b57af7f8bce7d1ebcbdfc32b676192e46703e81e9e217316e56b5bd" + "sha256:106695a836f19c1af6c46b58e8aac80e00f86c5b3287a3c6483a1ee369cc95c9", + "sha256:9f6dbf6155fe2b7a3d24bf68de97fb812db0fac0a54e96bc1af14ea95078ba7f" ], - "version": "==3.2.1" + "markers": "python_version >= '3.6'", + "version": "==3.2.3" }, "alabaster": { "hashes": [ @@ -64,74 +81,98 @@ ], "version": "==0.7.12" }, + "async-rediscache": { + "extras": [ + "fakeredis" + ], + "hashes": [ + "sha256:6be8a657d724ccbcfb1946d29a80c3478c5f9ecd2f78a0a26d2f4013a622258f", + "sha256:c25e4fff73f64d20645254783c3224a4c49e083e3fab67c44f17af944c5e26af" + ], + "index": "pypi", + "markers": "python_version ~= '3.7'", + "version": "==0.1.4" + }, "async-timeout": { "hashes": [ "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3" ], + "markers": "python_full_version >= '3.5.3'", "version": "==3.0.1" }, "attrs": { "hashes": [ - "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", - "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594", + "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc" ], - "version": "==19.3.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.2.0" }, "babel": { "hashes": [ "sha256:1aac2ae2d0d8ea368fa90906567f5c08463d98ade155c0c4bfedd6a0f7160e38", "sha256:d670ea0b10f8b723672d3a6abeb87b565b244da220d76b4dba1b66269ec152d4" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.0" }, "beautifulsoup4": { "hashes": [ - "sha256:05fd825eb01c290877657a56df4c6e4c311b3965bda790c613a3d6fb01a5462a", - "sha256:9fbb4d6e48ecd30bcacc5b63b94088192dcda178513b2ae3c394229f8911b887", - "sha256:e1505eeed31b0f4ce2dbb3bc8eb256c04cc2b3b72af7d551a4ab6efd5cbe5dae" + "sha256:1edf5e39f3a5bc6e38b235b369128416c7239b34f692acccececb040233032a1", + "sha256:5dfe44f8fddc89ac5453f02659d3ab1668f2c0d9684839f0785037e8c6d9ac8d", + "sha256:645d833a828722357038299b7f6879940c11dddd95b900fe5387c258b72bb883" ], - "version": "==4.8.2" + "index": "pypi", + "version": "==4.9.2" }, "certifi": { "hashes": [ - "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", - "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" + "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", + "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" ], - "version": "==2019.11.28" + "version": "==2020.6.20" }, "cffi": { "hashes": [ - "sha256:001bf3242a1bb04d985d63e138230802c6c8d4db3668fb545fb5005ddf5bb5ff", - "sha256:00789914be39dffba161cfc5be31b55775de5ba2235fe49aa28c148236c4e06b", - "sha256:028a579fc9aed3af38f4892bdcc7390508adabc30c6af4a6e4f611b0c680e6ac", - "sha256:14491a910663bf9f13ddf2bc8f60562d6bc5315c1f09c704937ef17293fb85b0", - "sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384", - "sha256:2089ed025da3919d2e75a4d963d008330c96751127dd6f73c8dc0c65041b4c26", - "sha256:2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6", - "sha256:337d448e5a725bba2d8293c48d9353fc68d0e9e4088d62a9571def317797522b", - "sha256:399aed636c7d3749bbed55bc907c3288cb43c65c4389964ad5ff849b6370603e", - "sha256:3b911c2dbd4f423b4c4fcca138cadde747abdb20d196c4a48708b8a2d32b16dd", - "sha256:3d311bcc4a41408cf5854f06ef2c5cab88f9fded37a3b95936c9879c1640d4c2", - "sha256:62ae9af2d069ea2698bf536dcfe1e4eed9090211dbaafeeedf5cb6c41b352f66", - "sha256:66e41db66b47d0d8672d8ed2708ba91b2f2524ece3dee48b5dfb36be8c2f21dc", - "sha256:675686925a9fb403edba0114db74e741d8181683dcf216be697d208857e04ca8", - "sha256:7e63cbcf2429a8dbfe48dcc2322d5f2220b77b2e17b7ba023d6166d84655da55", - "sha256:8a6c688fefb4e1cd56feb6c511984a6c4f7ec7d2a1ff31a10254f3c817054ae4", - "sha256:8c0ffc886aea5df6a1762d0019e9cb05f825d0eec1f520c51be9d198701daee5", - "sha256:95cd16d3dee553f882540c1ffe331d085c9e629499ceadfbda4d4fde635f4b7d", - "sha256:99f748a7e71ff382613b4e1acc0ac83bf7ad167fb3802e35e90d9763daba4d78", - "sha256:b8c78301cefcf5fd914aad35d3c04c2b21ce8629b5e4f4e45ae6812e461910fa", - "sha256:c420917b188a5582a56d8b93bdd8e0f6eca08c84ff623a4c16e809152cd35793", - "sha256:c43866529f2f06fe0edc6246eb4faa34f03fe88b64a0a9a942561c8e22f4b71f", - "sha256:cab50b8c2250b46fe738c77dbd25ce017d5e6fb35d3407606e7a4180656a5a6a", - "sha256:cef128cb4d5e0b3493f058f10ce32365972c554572ff821e175dbc6f8ff6924f", - "sha256:cf16e3cf6c0a5fdd9bc10c21687e19d29ad1fe863372b5543deaec1039581a30", - "sha256:e56c744aa6ff427a607763346e4170629caf7e48ead6921745986db3692f987f", - "sha256:e577934fc5f8779c554639376beeaa5657d54349096ef24abe8c74c5d9c117c3", - "sha256:f2b0fa0c01d8a0c7483afd9f31d7ecf2d71760ca24499c8697aeb5ca37dc090c" - ], - "version": "==1.14.0" + "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d", + "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b", + "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4", + "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f", + "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3", + "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579", + "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537", + "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e", + "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05", + "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171", + "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca", + "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522", + "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c", + "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc", + "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d", + "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808", + "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828", + "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869", + "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d", + "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9", + "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0", + "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc", + "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15", + "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c", + "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a", + "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3", + "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1", + "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768", + "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d", + "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b", + "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e", + "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d", + "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730", + "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394", + "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1", + "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591" + ], + "version": "==1.14.3" }, "chardet": { "hashes": [ @@ -145,7 +186,6 @@ "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff", "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1" ], - "index": "pypi", "markers": "sys_platform == 'win32'", "version": "==0.4.3" }, @@ -165,20 +205,47 @@ "index": "pypi", "version": "==4.3.2" }, - "discord-py": { + "discord": { "hashes": [ - "sha256:7424be26b07b37ecad4404d9383d685995a0e0b3df3f9c645bdd3a4d977b83b4" + "sha256:9d4debb4a37845543bd4b92cb195bc53a302797333e768e70344222857ff1559", + "sha256:ff6653655e342e7721dfb3f10421345fd852c2a33f2cca912b1c39b3778a9429" ], "index": "pypi", - "version": "==1.3.2" + "py": "~=1.4.0", + "version": "==1.0.1" + }, + "discord.py": { + "hashes": [ + "sha256:98ea3096a3585c9c379209926f530808f5fcf4930928d8cfb579d2562d119570", + "sha256:f9decb3bfa94613d922376288617e6a6f969260923643e2897f4540c34793442" + ], + "markers": "python_full_version >= '3.5.3'", + "version": "==1.4.1" }, "docutils": { "hashes": [ "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af", "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==0.16" }, + "fakeredis": { + "hashes": [ + "sha256:7ea0866ba5edb40fe2e9b1722535df0c7e6b91d518aa5f50d96c2fff3ea7f4c2", + "sha256:aad8836ffe0319ffbba66dcf872ac6e7e32d1f19790e31296ba58445efb0a5c7" + ], + "version": "==1.4.3" + }, + "feedparser": { + "hashes": [ + "sha256:bd030652c2d08532c034c27fcd7c85868e7fa3cb2b17f230a44a6bbc92519bf9", + "sha256:cd2485472e41471632ed3029d44033ee420ad0b57111db95c240c9160a85831c", + "sha256:ce875495c90ebd74b179855449040003a1beb40cd13d5f037a0654251e260b02" + ], + "index": "pypi", + "version": "==5.2.1" + }, "fuzzywuzzy": { "hashes": [ "sha256:45016e92264780e58972dca1b3d939ac864b78437422beecebb3095f8efd00e8", @@ -187,73 +254,134 @@ "index": "pypi", "version": "==0.18.0" }, + "hiredis": { + "hashes": [ + "sha256:06a039208f83744a702279b894c8cf24c14fd63c59cd917dcde168b79eef0680", + "sha256:0a909bf501459062aa1552be1461456518f367379fdc9fdb1f2ca5e4a1fdd7c0", + "sha256:18402d9e54fb278cb9a8c638df6f1550aca36a009d47ecf5aa263a38600f35b0", + "sha256:1e4cbbc3858ec7e680006e5ca590d89a5e083235988f26a004acf7244389ac01", + "sha256:23344e3c2177baf6975fbfa361ed92eb7d36d08f454636e5054b3faa7c2aff8a", + "sha256:289b31885b4996ce04cadfd5fc03d034dce8e2a8234479f7c9e23b9e245db06b", + "sha256:2c1c570ae7bf1bab304f29427e2475fe1856814312c4a1cf1cd0ee133f07a3c6", + "sha256:2c227c0ed371771ffda256034427320870e8ea2e4fd0c0a618c766e7c49aad73", + "sha256:3bb9b63d319402cead8bbd9dd55dca3b667d2997e9a0d8a1f9b6cc274db4baee", + "sha256:3ef2183de67b59930d2db8b8e8d4d58e00a50fcc5e92f4f678f6eed7a1c72d55", + "sha256:43b8ed3dbfd9171e44c554cb4acf4ee4505caa84c5e341858b50ea27dd2b6e12", + "sha256:47bcf3c5e6c1e87ceb86cdda2ee983fa0fe56a999e6185099b3c93a223f2fa9b", + "sha256:5263db1e2e1e8ae30500cdd75a979ff99dcc184201e6b4b820d0de74834d2323", + "sha256:5b1451727f02e7acbdf6aae4e06d75f66ee82966ff9114550381c3271a90f56c", + "sha256:6996883a8a6ff9117cbb3d6f5b0dcbbae6fb9e31e1a3e4e2f95e0214d9a1c655", + "sha256:6c96f64a54f030366657a54bb90b3093afc9c16c8e0dfa29fc0d6dbe169103a5", + "sha256:7332d5c3e35154cd234fd79573736ddcf7a0ade7a986db35b6196b9171493e75", + "sha256:7885b6f32c4a898e825bb7f56f36a02781ac4a951c63e4169f0afcf9c8c30dfb", + "sha256:7b0f63f10a166583ab744a58baad04e0f52cfea1ac27bfa1b0c21a48d1003c23", + "sha256:819f95d4eba3f9e484dd115ab7ab72845cf766b84286a00d4ecf76d33f1edca1", + "sha256:8968eeaa4d37a38f8ca1f9dbe53526b69628edc9c42229a5b2f56d98bb828c1f", + "sha256:89ebf69cb19a33d625db72d2ac589d26e936b8f7628531269accf4a3196e7872", + "sha256:8daecd778c1da45b8bd54fd41ffcd471a86beed3d8e57a43acf7a8d63bba4058", + "sha256:955ba8ea73cf3ed8bd2f963b4cb9f8f0dcb27becd2f4b3dd536fd24c45533454", + "sha256:964f18a59f5a64c0170f684c417f4fe3e695a536612e13074c4dd5d1c6d7c882", + "sha256:969843fbdfbf56cdb71da6f0bdf50f9985b8b8aeb630102945306cf10a9c6af2", + "sha256:996021ef33e0f50b97ff2d6b5f422a0fe5577de21a8873b58a779a5ddd1c3132", + "sha256:9e9c9078a7ce07e6fce366bd818be89365a35d2e4b163268f0ca9ba7e13bb2f6", + "sha256:a04901757cb0fb0f5602ac11dda48f5510f94372144d06c2563ba56c480b467c", + "sha256:a7bf1492429f18d205f3a818da3ff1f242f60aa59006e53dee00b4ef592a3363", + "sha256:aa0af2deb166a5e26e0d554b824605e660039b161e37ed4f01b8d04beec184f3", + "sha256:abfb15a6a7822f0fae681785cb38860e7a2cb1616a708d53df557b3d76c5bfd4", + "sha256:b253fe4df2afea4dfa6b1fa8c5fef212aff8bcaaeb4207e81eed05cb5e4a7919", + "sha256:b27f082f47d23cffc4cf1388b84fdc45c4ef6015f906cd7e0d988d9e35d36349", + "sha256:b33aea449e7f46738811fbc6f0b3177c6777a572207412bbbf6f525ffed001ae", + "sha256:b44f9421c4505c548435244d74037618f452844c5d3c67719d8a55e2613549da", + "sha256:bcc371151d1512201d0214c36c0c150b1dc64f19c2b1a8c9cb1d7c7c15ebd93f", + "sha256:c2851deeabd96d3f6283e9c6b26e0bfed4de2dc6fb15edf913e78b79fc5909ed", + "sha256:cdfd501c7ac5b198c15df800a3a34c38345f5182e5f80770caf362bccca65628", + "sha256:d2c0caffa47606d6d7c8af94ba42547bd2a441f06c74fd90a1ffe328524a6c64", + "sha256:dcb2db95e629962db5a355047fb8aefb012df6c8ae608930d391619dbd96fd86", + "sha256:e0eeb9c112fec2031927a1745788a181d0eecbacbed941fc5c4f7bc3f7b273bf", + "sha256:e154891263306200260d7f3051982774d7b9ef35af3509d5adbbe539afd2610c", + "sha256:e2e023a42dcbab8ed31f97c2bcdb980b7fbe0ada34037d87ba9d799664b58ded", + "sha256:e64be68255234bb489a574c4f2f8df7029c98c81ec4d160d6cd836e7f0679390", + "sha256:e82d6b930e02e80e5109b678c663a9ed210680ded81c1abaf54635d88d1da298" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.1.0" + }, "humanfriendly": { "hashes": [ - "sha256:25c2108a45cfd1e8fbe9cdb30b825d34ef5d5675c8e11e4775c9aedbfb0bdee2", - "sha256:3a831920e40e55ad49adb64c9179ed50c604cabca72cd300e7bd5b51310e4ebb" + "sha256:bf52ec91244819c780341a3438d5d7b09f431d3f113a475147ac9b7b167a3d12", + "sha256:e78960b31198511f45fd455534ae7645a6207d33e512d2e842c766d15d9c8080" ], - "version": "==8.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==8.2" }, "idna": { "hashes": [ - "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", - "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" ], - "version": "==2.9" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.10" }, "imagesize": { "hashes": [ "sha256:6965f19a6a2039c7d48bca7dba2473069ff854c36ae6f19d2cde309d998228a1", "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.2.0" }, "jinja2": { "hashes": [ - "sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250", - "sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49" + "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0", + "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035" ], - "version": "==2.11.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.11.2" }, "lxml": { "hashes": [ - "sha256:06d4e0bbb1d62e38ae6118406d7cdb4693a3fa34ee3762238bcb96c9e36a93cd", - "sha256:0701f7965903a1c3f6f09328c1278ac0eee8f56f244e66af79cb224b7ef3801c", - "sha256:1f2c4ec372bf1c4a2c7e4bb20845e8bcf8050365189d86806bad1e3ae473d081", - "sha256:4235bc124fdcf611d02047d7034164897ade13046bda967768836629bc62784f", - "sha256:5828c7f3e615f3975d48f40d4fe66e8a7b25f16b5e5705ffe1d22e43fb1f6261", - "sha256:585c0869f75577ac7a8ff38d08f7aac9033da2c41c11352ebf86a04652758b7a", - "sha256:5d467ce9c5d35b3bcc7172c06320dddb275fea6ac2037f72f0a4d7472035cea9", - "sha256:63dbc21efd7e822c11d5ddbedbbb08cd11a41e0032e382a0fd59b0b08e405a3a", - "sha256:7bc1b221e7867f2e7ff1933165c0cec7153dce93d0cdba6554b42a8beb687bdb", - "sha256:8620ce80f50d023d414183bf90cc2576c2837b88e00bea3f33ad2630133bbb60", - "sha256:8a0ebda56ebca1a83eb2d1ac266649b80af8dd4b4a3502b2c1e09ac2f88fe128", - "sha256:90ed0e36455a81b25b7034038e40880189169c308a3df360861ad74da7b68c1a", - "sha256:95e67224815ef86924fbc2b71a9dbd1f7262384bca4bc4793645794ac4200717", - "sha256:afdb34b715daf814d1abea0317b6d672476b498472f1e5aacbadc34ebbc26e89", - "sha256:b4b2c63cc7963aedd08a5f5a454c9f67251b1ac9e22fd9d72836206c42dc2a72", - "sha256:d068f55bda3c2c3fcaec24bd083d9e2eede32c583faf084d6e4b9daaea77dde8", - "sha256:d5b3c4b7edd2e770375a01139be11307f04341ec709cf724e0f26ebb1eef12c3", - "sha256:deadf4df349d1dcd7b2853a2c8796593cc346600726eff680ed8ed11812382a7", - "sha256:df533af6f88080419c5a604d0d63b2c33b1c0c4409aba7d0cb6de305147ea8c8", - "sha256:e4aa948eb15018a657702fee0b9db47e908491c64d36b4a90f59a64741516e77", - "sha256:e5d842c73e4ef6ed8c1bd77806bf84a7cb535f9c0cf9b2c74d02ebda310070e1", - "sha256:ebec08091a22c2be870890913bdadd86fcd8e9f0f22bcb398abd3af914690c15", - "sha256:edc15fcfd77395e24543be48871c251f38132bb834d9fdfdad756adb6ea37679", - "sha256:f2b74784ed7e0bc2d02bd53e48ad6ba523c9b36c194260b7a5045071abbb1012", - "sha256:fa071559f14bd1e92077b1b5f6c22cf09756c6de7139370249eb372854ce51e6", - "sha256:fd52e796fee7171c4361d441796b64df1acfceb51f29e545e812f16d023c4bbc", - "sha256:fe976a0f1ef09b3638778024ab9fb8cde3118f203364212c198f71341c0715ca" - ], - "index": "pypi", - "version": "==4.5.0" + "sha256:05a444b207901a68a6526948c7cc8f9fe6d6f24c70781488e32fd74ff5996e3f", + "sha256:08fc93257dcfe9542c0a6883a25ba4971d78297f63d7a5a26ffa34861ca78730", + "sha256:107781b213cf7201ec3806555657ccda67b1fccc4261fb889ef7fc56976db81f", + "sha256:121b665b04083a1e85ff1f5243d4a93aa1aaba281bc12ea334d5a187278ceaf1", + "sha256:1fa21263c3aba2b76fd7c45713d4428dbcc7644d73dcf0650e9d344e433741b3", + "sha256:2b30aa2bcff8e958cd85d907d5109820b01ac511eae5b460803430a7404e34d7", + "sha256:4b4a111bcf4b9c948e020fd207f915c24a6de3f1adc7682a2d92660eb4e84f1a", + "sha256:5591c4164755778e29e69b86e425880f852464a21c7bb53c7ea453bbe2633bbe", + "sha256:59daa84aef650b11bccd18f99f64bfe44b9f14a08a28259959d33676554065a1", + "sha256:5a9c8d11aa2c8f8b6043d845927a51eb9102eb558e3f936df494e96393f5fd3e", + "sha256:5dd20538a60c4cc9a077d3b715bb42307239fcd25ef1ca7286775f95e9e9a46d", + "sha256:74f48ec98430e06c1fa8949b49ebdd8d27ceb9df8d3d1c92e1fdc2773f003f20", + "sha256:786aad2aa20de3dbff21aab86b2fb6a7be68064cbbc0219bde414d3a30aa47ae", + "sha256:7ad7906e098ccd30d8f7068030a0b16668ab8aa5cda6fcd5146d8d20cbaa71b5", + "sha256:80a38b188d20c0524fe8959c8ce770a8fdf0e617c6912d23fc97c68301bb9aba", + "sha256:8f0ec6b9b3832e0bd1d57af41f9238ea7709bbd7271f639024f2fc9d3bb01293", + "sha256:92282c83547a9add85ad658143c76a64a8d339028926d7dc1998ca029c88ea6a", + "sha256:94150231f1e90c9595ccc80d7d2006c61f90a5995db82bccbca7944fd457f0f6", + "sha256:9dc9006dcc47e00a8a6a029eb035c8f696ad38e40a27d073a003d7d1443f5d88", + "sha256:a76979f728dd845655026ab991df25d26379a1a8fc1e9e68e25c7eda43004bed", + "sha256:aa8eba3db3d8761db161003e2d0586608092e217151d7458206e243be5a43843", + "sha256:bea760a63ce9bba566c23f726d72b3c0250e2fa2569909e2d83cda1534c79443", + "sha256:c3f511a3c58676147c277eff0224c061dd5a6a8e1373572ac817ac6324f1b1e0", + "sha256:c9d317efde4bafbc1561509bfa8a23c5cab66c44d49ab5b63ff690f5159b2304", + "sha256:cc411ad324a4486b142c41d9b2b6a722c534096963688d879ea6fa8a35028258", + "sha256:cdc13a1682b2a6241080745b1953719e7fe0850b40a5c71ca574f090a1391df6", + "sha256:cfd7c5dd3c35c19cec59c63df9571c67c6d6e5c92e0fe63517920e97f61106d1", + "sha256:e1cacf4796b20865789083252186ce9dc6cc59eca0c2e79cca332bdff24ac481", + "sha256:e70d4e467e243455492f5de463b72151cc400710ac03a0678206a5f27e79ddef", + "sha256:ecc930ae559ea8a43377e8b60ca6f8d61ac532fc57efb915d899de4a67928efd", + "sha256:f161af26f596131b63b236372e4ce40f3167c1b5b5d459b29d2514bd8c9dc9ee" + ], + "index": "pypi", + "version": "==4.5.2" }, "markdownify": { "hashes": [ - "sha256:28ce67d1888e4908faaab7b04d2193cda70ea4f902f156a21d0aaea55e63e0a1" + "sha256:30be8340724e706c9e811c27fe8c1542cf74a15b46827924fff5c54b40dd9b0d", + "sha256:a69588194fd76634f0139d6801b820fd652dc5eeba9530e90d323dfdc0155252" ], "index": "pypi", - "version": "==0.4.1" + "version": "==0.5.3" }, "markupsafe": { "hashes": [ @@ -291,50 +419,54 @@ "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.1" }, "more-itertools": { "hashes": [ - "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c", - "sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507" + "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20", + "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c" ], "index": "pypi", - "version": "==8.2.0" + "version": "==8.5.0" }, "multidict": { "hashes": [ - "sha256:317f96bc0950d249e96d8d29ab556d01dd38888fbe68324f46fd834b430169f1", - "sha256:42f56542166040b4474c0c608ed051732033cd821126493cf25b6c276df7dd35", - "sha256:4b7df040fb5fe826d689204f9b544af469593fb3ff3a069a6ad3409f742f5928", - "sha256:544fae9261232a97102e27a926019100a9db75bec7b37feedd74b3aa82f29969", - "sha256:620b37c3fea181dab09267cd5a84b0f23fa043beb8bc50d8474dd9694de1fa6e", - "sha256:6e6fef114741c4d7ca46da8449038ec8b1e880bbe68674c01ceeb1ac8a648e78", - "sha256:7774e9f6c9af3f12f296131453f7b81dabb7ebdb948483362f5afcaac8a826f1", - "sha256:85cb26c38c96f76b7ff38b86c9d560dea10cf3459bb5f4caf72fc1bb932c7136", - "sha256:a326f4240123a2ac66bb163eeba99578e9d63a8654a59f4688a79198f9aa10f8", - "sha256:ae402f43604e3b2bc41e8ea8b8526c7fa7139ed76b0d64fc48e28125925275b2", - "sha256:aee283c49601fa4c13adc64c09c978838a7e812f85377ae130a24d7198c0331e", - "sha256:b51249fdd2923739cd3efc95a3d6c363b67bbf779208e9f37fd5e68540d1a4d4", - "sha256:bb519becc46275c594410c6c28a8a0adc66fe24fef154a9addea54c1adb006f5", - "sha256:c2c37185fb0af79d5c117b8d2764f4321eeb12ba8c141a95d0aa8c2c1d0a11dd", - "sha256:dc561313279f9d05a3d0ffa89cd15ae477528ea37aa9795c4654588a3287a9ab", - "sha256:e439c9a10a95cb32abd708bb8be83b2134fa93790a4fb0535ca36db3dda94d20", - "sha256:fc3b4adc2ee8474cb3cd2a155305d5f8eda0a9c91320f83e55748e1fcb68f8e3" - ], - "version": "==4.7.5" + "sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a", + "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000", + "sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2", + "sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507", + "sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5", + "sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7", + "sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d", + "sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463", + "sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19", + "sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3", + "sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b", + "sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c", + "sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87", + "sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7", + "sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430", + "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255", + "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d" + ], + "markers": "python_version >= '3.5'", + "version": "==4.7.6" }, "ordered-set": { "hashes": [ - "sha256:a7bfa858748c73b096e43db14eb23e2bc714a503f990c89fac8fab9b0ee79724" + "sha256:ba93b2df055bca202116ec44b9bead3df33ea63a7d5827ff8e16738b97f33a95" ], - "version": "==3.1.1" + "markers": "python_version >= '3.5'", + "version": "==4.0.2" }, "packaging": { "hashes": [ - "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3", - "sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752" + "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", + "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" ], - "version": "==20.3" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.4" }, "pamqp": { "hashes": [ @@ -382,28 +514,24 @@ "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.20" }, "pygments": { "hashes": [ - "sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44", - "sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324" + "sha256:307543fe65c0947b126e83dd5a61bd8acbd84abec11f43caebaf5534cbc17998", + "sha256:926c3f319eda178d1bd90851e4317e6d8cdb5e292a3386aac9bd75eca29cf9c7" ], - "version": "==2.6.1" + "markers": "python_version >= '3.5'", + "version": "==2.7.1" }, "pyparsing": { "hashes": [ - "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", - "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" - ], - "version": "==2.4.6" - }, - "pyreadline": { - "hashes": [ - "sha256:4530592fc2e85b25b1a9f79664433da09237c1a270e4d78ea5aa3a2c7229e2d1" + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "markers": "sys_platform == 'win32'", - "version": "==2.1" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.7" }, "python-dateutil": { "hashes": [ @@ -415,10 +543,10 @@ }, "pytz": { "hashes": [ - "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", - "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" + "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", + "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048" ], - "version": "==2019.3" + "version": "==2020.1" }, "pyyaml": { "hashes": [ @@ -437,28 +565,37 @@ "index": "pypi", "version": "==5.3.1" }, + "redis": { + "hashes": [ + "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2", + "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==3.5.3" + }, "requests": { "hashes": [ - "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", - "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6" + "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", + "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" ], "index": "pypi", - "version": "==2.23.0" + "version": "==2.24.0" }, "sentry-sdk": { "hashes": [ - "sha256:23808d571d2461a4ce3784ec12bbee5bdb8c026c143fe79d36cef8a6d653e71f", - "sha256:bb90a4e19c7233a580715fc986cc44be2c48fc10b31e71580a2037e1c94b6950" + "sha256:c9c0fa1412bad87104c4eee8dd36c7bbf60b0d92ae917ab519094779b22e6d9a", + "sha256:e159f7c919d19ae86e5a4ff370fccc45149fab461fbeb93fb5a735a0b33a9cb1" ], "index": "pypi", - "version": "==0.14.3" + "version": "==0.17.8" }, "six": { "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "version": "==1.14.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.15.0" }, "snowballstemmer": { "hashes": [ @@ -467,12 +604,20 @@ ], "version": "==2.0.0" }, + "sortedcontainers": { + "hashes": [ + "sha256:4e73a757831fc3ca4de2859c422564239a31d8213d09a2a666e375807034d2ba", + "sha256:c633ebde8580f241f274c1f8994a665c0e54a17724fecd0cae2f079e09c36d3f" + ], + "version": "==2.2.2" + }, "soupsieve": { "hashes": [ - "sha256:e914534802d7ffd233242b785229d5ba0766a7f487385e3f714446a07bf540ae", - "sha256:fcd71e08c0aee99aca1b73f45478549ee7e7fc006d51b37bec9e9def7dc22b69" + "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", + "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232" ], - "version": "==2.0" + "markers": "python_version >= '3.0'", + "version": "==2.0.1" }, "sphinx": { "hashes": [ @@ -487,6 +632,7 @@ "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a", "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58" ], + "markers": "python_version >= '3.5'", "version": "==1.0.2" }, "sphinxcontrib-devhelp": { @@ -494,6 +640,7 @@ "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e", "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4" ], + "markers": "python_version >= '3.5'", "version": "==1.0.2" }, "sphinxcontrib-htmlhelp": { @@ -501,6 +648,7 @@ "sha256:3c0bc24a2c41e340ac37c85ced6dafc879ab485c095b1d65d2461ac2f7cca86f", "sha256:e8f5bb7e31b2dbb25b9cc435c8ab7a79787ebf7f906155729338f3156d93659b" ], + "markers": "python_version >= '3.5'", "version": "==1.0.3" }, "sphinxcontrib-jsmath": { @@ -508,6 +656,7 @@ "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8" ], + "markers": "python_version >= '3.5'", "version": "==1.0.1" }, "sphinxcontrib-qthelp": { @@ -515,6 +664,7 @@ "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72", "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6" ], + "markers": "python_version >= '3.5'", "version": "==1.0.3" }, "sphinxcontrib-serializinghtml": { @@ -522,136 +672,119 @@ "sha256:eaa0eccc86e982a9b939b2b82d12cc5d013385ba5eadcc7e4fed23f4405f77bc", "sha256:f242a81d423f59617a8e5cf16f5d4d74e28ee9a66f9e5b637a18082991db5a9a" ], + "markers": "python_version >= '3.5'", "version": "==1.1.4" }, + "statsd": { + "hashes": [ + "sha256:c610fb80347fca0ef62666d241bce64184bd7cc1efe582f9690e045c25535eaa", + "sha256:e3e6db4c246f7c59003e51c9720a51a7f39a396541cb9b147ff4b14d15b5dd1f" + ], + "index": "pypi", + "version": "==3.3.0" + }, "urllib3": { "hashes": [ - "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", - "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" - ], - "version": "==1.25.8" - }, - "websockets": { - "hashes": [ - "sha256:0e4fb4de42701340bd2353bb2eee45314651caa6ccee80dbd5f5d5978888fed5", - "sha256:1d3f1bf059d04a4e0eb4985a887d49195e15ebabc42364f4eb564b1d065793f5", - "sha256:20891f0dddade307ffddf593c733a3fdb6b83e6f9eef85908113e628fa5a8308", - "sha256:295359a2cc78736737dd88c343cd0747546b2174b5e1adc223824bcaf3e164cb", - "sha256:2db62a9142e88535038a6bcfea70ef9447696ea77891aebb730a333a51ed559a", - "sha256:3762791ab8b38948f0c4d281c8b2ddfa99b7e510e46bd8dfa942a5fff621068c", - "sha256:3db87421956f1b0779a7564915875ba774295cc86e81bc671631379371af1170", - "sha256:3ef56fcc7b1ff90de46ccd5a687bbd13a3180132268c4254fc0fa44ecf4fc422", - "sha256:4f9f7d28ce1d8f1295717c2c25b732c2bc0645db3215cf757551c392177d7cb8", - "sha256:5c01fd846263a75bc8a2b9542606927cfad57e7282965d96b93c387622487485", - "sha256:5c65d2da8c6bce0fca2528f69f44b2f977e06954c8512a952222cea50dad430f", - "sha256:751a556205d8245ff94aeef23546a1113b1dd4f6e4d102ded66c39b99c2ce6c8", - "sha256:7ff46d441db78241f4c6c27b3868c9ae71473fe03341340d2dfdbe8d79310acc", - "sha256:965889d9f0e2a75edd81a07592d0ced54daa5b0785f57dc429c378edbcffe779", - "sha256:9b248ba3dd8a03b1a10b19efe7d4f7fa41d158fdaa95e2cf65af5a7b95a4f989", - "sha256:9bef37ee224e104a413f0780e29adb3e514a5b698aabe0d969a6ba426b8435d1", - "sha256:c1ec8db4fac31850286b7cd3b9c0e1b944204668b8eb721674916d4e28744092", - "sha256:c8a116feafdb1f84607cb3b14aa1418424ae71fee131642fc568d21423b51824", - "sha256:ce85b06a10fc65e6143518b96d3dca27b081a740bae261c2fb20375801a9d56d", - "sha256:d705f8aeecdf3262379644e4b55107a3b55860eb812b673b28d0fbc347a60c55", - "sha256:e898a0863421650f0bebac8ba40840fc02258ef4714cb7e1fd76b6a6354bda36", - "sha256:f8a7bff6e8664afc4e6c28b983845c5bc14965030e3fb98789734d416af77c4b" - ], - "version": "==8.1" + "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a", + "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.25.10" }, "yarl": { "hashes": [ - "sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce", - "sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6", - "sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce", - "sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae", - "sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d", - "sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f", - "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b", - "sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b", - "sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb", - "sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462", - "sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea", - "sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70", - "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1", - "sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a", - "sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b", - "sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080", - "sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2" - ], - "version": "==1.4.2" + "sha256:04a54f126a0732af75e5edc9addeaa2113e2ca7c6fce8974a63549a70a25e50e", + "sha256:3cc860d72ed989f3b1f3abbd6ecf38e412de722fb38b8f1b1a086315cf0d69c5", + "sha256:5d84cc36981eb5a8533be79d6c43454c8e6a39ee3118ceaadbd3c029ab2ee580", + "sha256:5e447e7f3780f44f890360ea973418025e8c0cdcd7d6a1b221d952600fd945dc", + "sha256:61d3ea3c175fe45f1498af868879c6ffeb989d4143ac542163c45538ba5ec21b", + "sha256:67c5ea0970da882eaf9efcf65b66792557c526f8e55f752194eff8ec722c75c2", + "sha256:6f6898429ec3c4cfbef12907047136fd7b9e81a6ee9f105b45505e633427330a", + "sha256:7ce35944e8e61927a8f4eb78f5bc5d1e6da6d40eadd77e3f79d4e9399e263921", + "sha256:b7c199d2cbaf892ba0f91ed36d12ff41ecd0dde46cbf64ff4bfe997a3ebc925e", + "sha256:c15d71a640fb1f8e98a1423f9c64d7f1f6a3a168f803042eaf3a5b5022fde0c1", + "sha256:c22607421f49c0cb6ff3ed593a49b6a99c6ffdeaaa6c944cdda83c2393c8864d", + "sha256:c604998ab8115db802cc55cb1b91619b2831a6128a62ca7eea577fc8ea4d3131", + "sha256:d088ea9319e49273f25b1c96a3763bf19a882cff774d1792ae6fba34bd40550a", + "sha256:db9eb8307219d7e09b33bcb43287222ef35cbcf1586ba9472b0a4b833666ada1", + "sha256:e31fef4e7b68184545c3d68baec7074532e077bd1906b040ecfba659737df188", + "sha256:e32f0fb443afcfe7f01f95172b66f279938fbc6bdaebe294b0ff6747fb6db020", + "sha256:fcbe419805c9b20db9a51d33b942feddbf6e7fb468cb20686fd7089d4164c12a" + ], + "markers": "python_version >= '3.5'", + "version": "==1.6.0" } }, "develop": { "appdirs": { "hashes": [ - "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", - "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" + "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", + "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" ], - "version": "==1.4.3" + "version": "==1.4.4" }, "attrs": { "hashes": [ - "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", - "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594", + "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc" ], - "version": "==19.3.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.2.0" }, "cfgv": { "hashes": [ - "sha256:1ccf53320421aeeb915275a196e23b3b8ae87dea8ac6698b1638001d4a486d53", - "sha256:c8e8f552ffcc6194f4e18dd4f68d9aef0c0d58ae7e7be8c82bee3c5e9edfa513" + "sha256:32e43d604bbe7896fe7c248a9c2276447dbef840feb28fe20494f62af110211d", + "sha256:cf22deb93d4bcf92f345a5c3cd39d3d41d6340adc60c78bbbd6588c384fda6a1" ], - "version": "==3.1.0" + "markers": "python_full_version >= '3.6.1'", + "version": "==3.2.0" }, "coverage": { "hashes": [ - "sha256:03f630aba2b9b0d69871c2e8d23a69b7fe94a1e2f5f10df5049c0df99db639a0", - "sha256:046a1a742e66d065d16fb564a26c2a15867f17695e7f3d358d7b1ad8a61bca30", - "sha256:0a907199566269e1cfa304325cc3b45c72ae341fbb3253ddde19fa820ded7a8b", - "sha256:165a48268bfb5a77e2d9dbb80de7ea917332a79c7adb747bd005b3a07ff8caf0", - "sha256:1b60a95fc995649464e0cd48cecc8288bac5f4198f21d04b8229dc4097d76823", - "sha256:1f66cf263ec77af5b8fe14ef14c5e46e2eb4a795ac495ad7c03adc72ae43fafe", - "sha256:2e08c32cbede4a29e2a701822291ae2bc9b5220a971bba9d1e7615312efd3037", - "sha256:3844c3dab800ca8536f75ae89f3cf566848a3eb2af4d9f7b1103b4f4f7a5dad6", - "sha256:408ce64078398b2ee2ec08199ea3fcf382828d2f8a19c5a5ba2946fe5ddc6c31", - "sha256:443be7602c790960b9514567917af538cac7807a7c0c0727c4d2bbd4014920fd", - "sha256:4482f69e0701139d0f2c44f3c395d1d1d37abd81bfafbf9b6efbe2542679d892", - "sha256:4a8a259bf990044351baf69d3b23e575699dd60b18460c71e81dc565f5819ac1", - "sha256:513e6526e0082c59a984448f4104c9bf346c2da9961779ede1fc458e8e8a1f78", - "sha256:5f587dfd83cb669933186661a351ad6fc7166273bc3e3a1531ec5c783d997aac", - "sha256:62061e87071497951155cbccee487980524d7abea647a1b2a6eb6b9647df9006", - "sha256:641e329e7f2c01531c45c687efcec8aeca2a78a4ff26d49184dce3d53fc35014", - "sha256:65a7e00c00472cd0f59ae09d2fb8a8aaae7f4a0cf54b2b74f3138d9f9ceb9cb2", - "sha256:6ad6ca45e9e92c05295f638e78cd42bfaaf8ee07878c9ed73e93190b26c125f7", - "sha256:73aa6e86034dad9f00f4bbf5a666a889d17d79db73bc5af04abd6c20a014d9c8", - "sha256:7c9762f80a25d8d0e4ab3cb1af5d9dffbddb3ee5d21c43e3474c84bf5ff941f7", - "sha256:85596aa5d9aac1bf39fe39d9fa1051b0f00823982a1de5766e35d495b4a36ca9", - "sha256:86a0ea78fd851b313b2e712266f663e13b6bc78c2fb260b079e8b67d970474b1", - "sha256:8a620767b8209f3446197c0e29ba895d75a1e272a36af0786ec70fe7834e4307", - "sha256:922fb9ef2c67c3ab20e22948dcfd783397e4c043a5c5fa5ff5e9df5529074b0a", - "sha256:9fad78c13e71546a76c2f8789623eec8e499f8d2d799f4b4547162ce0a4df435", - "sha256:a37c6233b28e5bc340054cf6170e7090a4e85069513320275a4dc929144dccf0", - "sha256:c3fc325ce4cbf902d05a80daa47b645d07e796a80682c1c5800d6ac5045193e5", - "sha256:cda33311cb9fb9323958a69499a667bd728a39a7aa4718d7622597a44c4f1441", - "sha256:db1d4e38c9b15be1521722e946ee24f6db95b189d1447fa9ff18dd16ba89f732", - "sha256:eda55e6e9ea258f5e4add23bcf33dc53b2c319e70806e180aecbff8d90ea24de", - "sha256:f372cdbb240e09ee855735b9d85e7f50730dcfb6296b74b95a3e5dea0615c4c1" - ], - "index": "pypi", - "version": "==5.0.4" - }, - "distlib": { - "hashes": [ - "sha256:2e166e231a26b36d6dfe35a48c4464346620f8645ed0ace01ee31822b288de21" + "sha256:0203acd33d2298e19b57451ebb0bed0ab0c602e5cf5a818591b4918b1f97d516", + "sha256:0f313707cdecd5cd3e217fc68c78a960b616604b559e9ea60cc16795c4304259", + "sha256:1c6703094c81fa55b816f5ae542c6ffc625fec769f22b053adb42ad712d086c9", + "sha256:1d44bb3a652fed01f1f2c10d5477956116e9b391320c94d36c6bf13b088a1097", + "sha256:280baa8ec489c4f542f8940f9c4c2181f0306a8ee1a54eceba071a449fb870a0", + "sha256:29a6272fec10623fcbe158fdf9abc7a5fa032048ac1d8631f14b50fbfc10d17f", + "sha256:2b31f46bf7b31e6aa690d4c7a3d51bb262438c6dcb0d528adde446531d0d3bb7", + "sha256:2d43af2be93ffbad25dd959899b5b809618a496926146ce98ee0b23683f8c51c", + "sha256:381ead10b9b9af5f64646cd27107fb27b614ee7040bb1226f9c07ba96625cbb5", + "sha256:47a11bdbd8ada9b7ee628596f9d97fbd3851bd9999d398e9436bd67376dbece7", + "sha256:4d6a42744139a7fa5b46a264874a781e8694bb32f1d76d8137b68138686f1729", + "sha256:50691e744714856f03a86df3e2bff847c2acede4c191f9a1da38f088df342978", + "sha256:530cc8aaf11cc2ac7430f3614b04645662ef20c348dce4167c22d99bec3480e9", + "sha256:582ddfbe712025448206a5bc45855d16c2e491c2dd102ee9a2841418ac1c629f", + "sha256:63808c30b41f3bbf65e29f7280bf793c79f54fb807057de7e5238ffc7cc4d7b9", + "sha256:71b69bd716698fa62cd97137d6f2fdf49f534decb23a2c6fc80813e8b7be6822", + "sha256:7858847f2d84bf6e64c7f66498e851c54de8ea06a6f96a32a1d192d846734418", + "sha256:78e93cc3571fd928a39c0b26767c986188a4118edc67bc0695bc7a284da22e82", + "sha256:7f43286f13d91a34fadf61ae252a51a130223c52bfefb50310d5b2deb062cf0f", + "sha256:86e9f8cd4b0cdd57b4ae71a9c186717daa4c5a99f3238a8723f416256e0b064d", + "sha256:8f264ba2701b8c9f815b272ad568d555ef98dfe1576802ab3149c3629a9f2221", + "sha256:9342dd70a1e151684727c9c91ea003b2fb33523bf19385d4554f7897ca0141d4", + "sha256:9361de40701666b034c59ad9e317bae95c973b9ff92513dd0eced11c6adf2e21", + "sha256:9669179786254a2e7e57f0ecf224e978471491d660aaca833f845b72a2df3709", + "sha256:aac1ba0a253e17889550ddb1b60a2063f7474155465577caa2a3b131224cfd54", + "sha256:aef72eae10b5e3116bac6957de1df4d75909fc76d1499a53fb6387434b6bcd8d", + "sha256:bd3166bb3b111e76a4f8e2980fa1addf2920a4ca9b2b8ca36a3bc3dedc618270", + "sha256:c1b78fb9700fc961f53386ad2fd86d87091e06ede5d118b8a50dea285a071c24", + "sha256:c3888a051226e676e383de03bf49eb633cd39fc829516e5334e69b8d81aae751", + "sha256:c5f17ad25d2c1286436761b462e22b5020d83316f8e8fcb5deb2b3151f8f1d3a", + "sha256:c851b35fc078389bc16b915a0a7c1d5923e12e2c5aeec58c52f4aa8085ac8237", + "sha256:cb7df71de0af56000115eafd000b867d1261f786b5eebd88a0ca6360cccfaca7", + "sha256:cedb2f9e1f990918ea061f28a0f0077a07702e3819602d3507e2ff98c8d20636", + "sha256:e8caf961e1b1a945db76f1b5fa9c91498d15f545ac0ababbe575cfab185d3bd8" ], - "version": "==0.3.0" + "index": "pypi", + "version": "==5.3" }, - "entrypoints": { + "distlib": { "hashes": [ - "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", - "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" + "sha256:8c09de2c67b3e7deef7184574fc060ab8a793e7adbb183d942c389c8b13c52fb", + "sha256:edf6116872c863e1aa9d5bb7cb5e05a022c519a4594dc703843343a9ddd9bff1" ], - "version": "==0.3" + "version": "==0.3.1" }, "filelock": { "hashes": [ @@ -662,19 +795,19 @@ }, "flake8": { "hashes": [ - "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb", - "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca" + "sha256:15e351d19611c887e482fb960eae4d44845013cc142d42896e9862f775d8cf5c", + "sha256:f04b9fcbac03b0a3e58c0ab3a0ecc462e023a9faf046d57794184028123aa208" ], "index": "pypi", - "version": "==3.7.9" + "version": "==3.8.3" }, "flake8-annotations": { "hashes": [ - "sha256:a38b44d01abd480586a92a02a2b0a36231ec42dcc5e114de78fa5db016d8d3f9", - "sha256:d5b0e8704e4e7728b352fa1464e23539ff2341ba11cc153b536fa2cf921ee659" + "sha256:09fe1aa3f40cb8fef632a0ab3614050a7584bb884b6134e70cf1fc9eeee642fa", + "sha256:5bda552f074fd6e34276c7761756fa07d824ffac91ce9c0a8555eb2bc5b92d7a" ], "index": "pypi", - "version": "==2.0.1" + "version": "==2.4.0" }, "flake8-bugbear": { "hashes": [ @@ -717,11 +850,11 @@ }, "flake8-tidy-imports": { "hashes": [ - "sha256:5b6e75cec6d751e66534c522fbdce7dac1c2738b1216b0f6b10453995932e188", - "sha256:cf26fbb3ab31a398f265d53b6f711d80006450c19221e41b2b7b0e0b14ac39c5" + "sha256:62059ca07d8a4926b561d392cbab7f09ee042350214a25cf12823384a45d27dd", + "sha256:c30b40337a2e6802ba3bb611c26611154a27e94c53fc45639e3e282169574fd3" ], "index": "pypi", - "version": "==4.0.1" + "version": "==4.1.0" }, "flake8-todo": { "hashes": [ @@ -732,10 +865,11 @@ }, "identify": { "hashes": [ - "sha256:a7577a1f55cee1d21953a5cf11a3c839ab87f5ef909a4cba6cf52ed72b4c6059", - "sha256:ab246293e6585a1c6361a505b68d5b501a0409310932b7de2c2ead667b564d89" + "sha256:7c22c384a2c9b32c5cc891d13f923f6b2653aa83e2d75d8f79be240d6c86c4f4", + "sha256:da683bfb7669fa749fc7731f378229e2dbf29a1d1337cbde04106f02236eb29d" ], - "version": "==1.4.13" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.5.5" }, "mccabe": { "hashes": [ @@ -746,46 +880,50 @@ }, "nodeenv": { "hashes": [ - "sha256:5b2438f2e42af54ca968dd1b374d14a1194848955187b0e5e4be1f73813a5212" + "sha256:5304d424c529c997bc888453aeaa6362d242b6b4631e90f3d4bf1b290f1c84a9", + "sha256:ab45090ae383b716c4ef89e690c41ff8c2b257b85b309f01f3654df3d084bd7c" ], - "version": "==1.3.5" + "version": "==1.5.0" }, "pep8-naming": { "hashes": [ - "sha256:5d9f1056cb9427ce344e98d1a7f5665710e2f20f748438e308995852cfa24164", - "sha256:f3b4a5f9dd72b991bf7d8e2a341d2e1aa3a884a769b5aaac4f56825c1763bf3a" + "sha256:a1dd47dd243adfe8a83616e27cf03164960b507530f155db94e10b36a6cd6724", + "sha256:f43bfe3eea7e0d73e8b5d07d6407ab47f2476ccaeff6937c84275cd30b016738" ], "index": "pypi", - "version": "==0.10.0" + "version": "==0.11.1" }, "pre-commit": { "hashes": [ - "sha256:487c675916e6f99d355ec5595ad77b325689d423ef4839db1ed2f02f639c9522", - "sha256:c0aa11bce04a7b46c5544723aedf4e81a4d5f64ad1205a30a9ea12d5e81969e1" + "sha256:810aef2a2ba4f31eed1941fc270e72696a1ad5590b9751839c90807d0fff6b9a", + "sha256:c54fd3e574565fe128ecc5e7d2f91279772ddb03f8729645fa812fe809084a70" ], "index": "pypi", - "version": "==2.2.0" + "version": "==2.7.1" }, "pycodestyle": { "hashes": [ - "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", - "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" + "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367", + "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e" ], - "version": "==2.5.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.6.0" }, "pydocstyle": { "hashes": [ - "sha256:da7831660b7355307b32778c4a0dbfb137d89254ef31a2b2978f50fc0b4d7586", - "sha256:f4f5d210610c2d153fae39093d44224c17429e2ad7da12a8b419aba5c2f614b5" + "sha256:19b86fa8617ed916776a11cd8bc0197e5b9856d5433b777f51a3defe13075325", + "sha256:aca749e190a01726a4fb472dd4ef23b5c9da7b9205c0a7857c06533de13fd678" ], - "version": "==5.0.2" + "markers": "python_version >= '3.5'", + "version": "==5.1.1" }, "pyflakes": { "hashes": [ - "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", - "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" + "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92", + "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8" ], - "version": "==2.1.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.2.0" }, "pyyaml": { "hashes": [ @@ -806,10 +944,11 @@ }, "six": { "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "version": "==1.14.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.15.0" }, "snowballstemmer": { "hashes": [ @@ -820,25 +959,26 @@ }, "toml": { "hashes": [ - "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", - "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" + "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", + "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88" ], - "version": "==0.10.0" + "version": "==0.10.1" }, "unittest-xml-reporting": { "hashes": [ - "sha256:74eaf7739a7957a74f52b8187c5616f61157372189bef0a32ba5c30bbc00e58a", - "sha256:e09b8ae70cce9904cdd331f53bf929150962869a5324ab7ff3dd6c8b87e01f7d" + "sha256:7bf515ea8cb244255a25100cd29db611a73f8d3d0aaf672ed3266307e14cc1ca", + "sha256:984cebba69e889401bfe3adb9088ca376b3a1f923f0590d005126c1bffd1a695" ], "index": "pypi", - "version": "==3.0.2" + "version": "==3.0.4" }, "virtualenv": { "hashes": [ - "sha256:87831f1070534b636fea2241dd66f3afe37ac9041bcca6d0af3215cdcfbf7d82", - "sha256:f3128d882383c503003130389bf892856341c1da12c881ae24d6358c82561b55" + "sha256:43add625c53c596d38f971a465553f6318decc39d98512bc100fa1b1e839c8dc", + "sha256:e0305af10299a7fb0d69393d8f04cb2965dda9351140d11ac8db4e5e3970451b" ], - "version": "==20.0.13" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.0.31" } } } @@ -1,6 +1,6 @@ # Python Utility Bot -[](https://discord.gg/2B963hn) +[](https://discord.gg/2B963hn) [](https://dev.azure.com/python-discord/Python%20Discord/_build/latest?definitionId=1&branchName=master) [](https://dev.azure.com/python-discord/Python%20Discord/_apis/build/status/Bot?branchName=master) [](https://dev.azure.com/python-discord/Python%20Discord/_apis/build/status/Bot?branchName=master) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 16d1b7a2a..4500cb6e8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,9 +1,13 @@ # https://aka.ms/yaml variables: + PIP_NO_CACHE_DIR: false + PIP_USER: 1 PIPENV_HIDE_EMOJIS: 1 PIPENV_IGNORE_VIRTUALENVS: 1 PIPENV_NOSPIN: 1 + PRE_COMMIT_HOME: $(Pipeline.Workspace)/pre-commit-cache + PYTHONUSERBASE: $(Pipeline.Workspace)/py-user-base jobs: - job: test @@ -12,22 +16,39 @@ jobs: vmImage: ubuntu-18.04 variables: - PIP_CACHE_DIR: ".cache/pip" - PRE_COMMIT_HOME: $(Pipeline.Workspace)/pre-commit-cache + BOT_API_KEY: foo + BOT_SENTRY_DSN: blah + BOT_TOKEN: bar + REDDIT_CLIENT_ID: spam + REDDIT_SECRET: ham + WOLFRAM_API_KEY: baz + REDIS_PASSWORD: '' steps: - task: UsePythonVersion@0 displayName: 'Set Python version' - name: PythonVersion + name: python inputs: versionSpec: '3.8.x' addToPath: true + - task: Cache@2 + displayName: 'Restore Python environment' + inputs: + key: python | $(Agent.OS) | "$(python.pythonLocation)" | 0 | ./Pipfile | ./Pipfile.lock + cacheHitVar: PY_ENV_RESTORED + path: $(PYTHONUSERBASE) + + - script: echo '##vso[task.prependpath]$(PYTHONUSERBASE)/bin' + displayName: 'Prepend PATH' + - script: pip install pipenv displayName: 'Install pipenv' + condition: and(succeeded(), ne(variables.PY_ENV_RESTORED, 'true')) - script: pipenv install --dev --deploy --system displayName: 'Install project using pipenv' + condition: and(succeeded(), ne(variables.PY_ENV_RESTORED, 'true')) # Create an executable shell script which replaces the original pipenv binary. # The shell script ignores the first argument and executes the rest of the args as a command. @@ -35,22 +56,21 @@ jobs: # pipenv entirely, which is too dumb to know it should use the system interpreter rather than # creating a new venv. - script: | - printf '%s\n%s' '#!/bin/bash' '"${@:2}"' > $(PythonVersion.pythonLocation)/bin/pipenv \ - && chmod +x $(PythonVersion.pythonLocation)/bin/pipenv + printf '%s\n%s' '#!/bin/bash' '"${@:2}"' > $(python.pythonLocation)/bin/pipenv \ + && chmod +x $(python.pythonLocation)/bin/pipenv displayName: 'Mock pipenv binary' - task: Cache@2 displayName: 'Restore pre-commit environment' inputs: - key: pre-commit | "$(PythonVersion.pythonLocation)" | .pre-commit-config.yaml - restoreKeys: | - pre-commit | "$(PythonVersion.pythonLocation)" + key: pre-commit | "$(python.pythonLocation)" | 0 | .pre-commit-config.yaml path: $(PRE_COMMIT_HOME) - - script: pre-commit run --all-files + # pre-commit's venv doesn't allow user installs - not that they're really needed anyway. + - script: export PIP_USER=0; pre-commit run --all-files displayName: 'Run pre-commit hooks' - - script: BOT_API_KEY=foo BOT_SENTRY_DSN=blah BOT_TOKEN=bar WOLFRAM_API_KEY=baz REDDIT_CLIENT_ID=spam REDDIT_SECRET=ham coverage run -m xmlrunner + - script: coverage run -m xmlrunner displayName: Run tests - script: coverage report -m && coverage xml -o coverage.xml diff --git a/bot/__init__.py b/bot/__init__.py index c9dbc3f40..3ee70c4e9 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -1,10 +1,15 @@ +import asyncio import logging import os import sys +from functools import partial, partialmethod from logging import Logger, handlers from pathlib import Path import coloredlogs +from discord.ext import commands + +from bot.command import Command TRACE_LEVEL = logging.TRACE = 5 logging.addLevelName(TRACE_LEVEL, "TRACE") @@ -33,7 +38,7 @@ log_format = logging.Formatter(format_string) log_file = Path("logs", "bot.log") log_file.parent.mkdir(exist_ok=True) -file_handler = handlers.RotatingFileHandler(log_file, maxBytes=5242880, backupCount=7) +file_handler = handlers.RotatingFileHandler(log_file, maxBytes=5242880, backupCount=7, encoding="utf8") file_handler.setFormatter(log_format) root_log = logging.getLogger() @@ -58,4 +63,16 @@ coloredlogs.install(logger=root_log, stream=sys.stdout) logging.getLogger("discord").setLevel(logging.WARNING) logging.getLogger("websockets").setLevel(logging.WARNING) +logging.getLogger("chardet").setLevel(logging.WARNING) logging.getLogger(__name__) + + +# On Windows, the selector event loop is required for aiodns. +if os.name == "nt": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + + +# Monkey-patch discord.py decorators to use the Command subclass which supports root aliases. +# Must be patched before any cogs are added. +commands.command = partial(commands.command, cls=Command) +commands.GroupMixin.command = partialmethod(commands.GroupMixin.command, cls=Command) diff --git a/bot/__main__.py b/bot/__main__.py index 0ae869d0d..152ddbf92 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -1,68 +1,68 @@ +import asyncio import logging import discord import sentry_sdk +from async_rediscache import RedisSession from discord.ext.commands import when_mentioned_or +from sentry_sdk.integrations.aiohttp import AioHttpIntegration from sentry_sdk.integrations.logging import LoggingIntegration +from sentry_sdk.integrations.redis import RedisIntegration +from bot import constants from bot.bot import Bot -from bot.constants import Bot as BotConfig +from bot.utils.extensions import EXTENSIONS +# Set up Sentry. sentry_logging = LoggingIntegration( level=logging.DEBUG, event_level=logging.WARNING ) sentry_sdk.init( - dsn=BotConfig.sentry_dsn, - integrations=[sentry_logging] + dsn=constants.Bot.sentry_dsn, + integrations=[ + sentry_logging, + AioHttpIntegration(), + RedisIntegration(), + ] ) +# Create the redis session instance. +redis_session = RedisSession( + address=(constants.Redis.host, constants.Redis.port), + password=constants.Redis.password, + minsize=1, + maxsize=20, + use_fakeredis=constants.Redis.use_fakeredis, + global_namespace="bot", +) + +# Connect redis session to ensure it's connected before we try to access Redis +# from somewhere within the bot. We create the event loop in the same way +# discord.py normally does and pass it to the bot's __init__. +loop = asyncio.get_event_loop() +loop.run_until_complete(redis_session.connect()) + + +# Instantiate the bot. +allowed_roles = [discord.Object(id_) for id_ in constants.MODERATION_ROLES] bot = Bot( - command_prefix=when_mentioned_or(BotConfig.prefix), + redis_session=redis_session, + loop=loop, + command_prefix=when_mentioned_or(constants.Bot.prefix), activity=discord.Game(name="Commands: !help"), case_insensitive=True, max_messages=10_000, + allowed_mentions=discord.AllowedMentions(everyone=False, roles=allowed_roles) ) -# Internal/debug -bot.load_extension("bot.cogs.error_handler") -bot.load_extension("bot.cogs.filtering") -bot.load_extension("bot.cogs.logging") -bot.load_extension("bot.cogs.security") -bot.load_extension("bot.cogs.config_verifier") - -# Commands, etc -bot.load_extension("bot.cogs.antimalware") -bot.load_extension("bot.cogs.antispam") -bot.load_extension("bot.cogs.bot") -bot.load_extension("bot.cogs.clean") -bot.load_extension("bot.cogs.extensions") -bot.load_extension("bot.cogs.help") - -bot.load_extension("bot.cogs.doc") -bot.load_extension("bot.cogs.verification") +# Load extensions. +extensions = set(EXTENSIONS) # Create a mutable copy. +if not constants.HelpChannels.enable: + extensions.remove("bot.exts.help_channels") -# Feature cogs -bot.load_extension("bot.cogs.alias") -bot.load_extension("bot.cogs.defcon") -bot.load_extension("bot.cogs.eval") -bot.load_extension("bot.cogs.duck_pond") -bot.load_extension("bot.cogs.free") -bot.load_extension("bot.cogs.information") -bot.load_extension("bot.cogs.jams") -bot.load_extension("bot.cogs.moderation") -bot.load_extension("bot.cogs.off_topic_names") -bot.load_extension("bot.cogs.reddit") -bot.load_extension("bot.cogs.reminders") -bot.load_extension("bot.cogs.site") -bot.load_extension("bot.cogs.snekbox") -bot.load_extension("bot.cogs.sync") -bot.load_extension("bot.cogs.tags") -bot.load_extension("bot.cogs.token_remover") -bot.load_extension("bot.cogs.utils") -bot.load_extension("bot.cogs.watchchannels") -bot.load_extension("bot.cogs.webhook_remover") -bot.load_extension("bot.cogs.wolfram") +for extension in extensions: + bot.load_extension(extension) -bot.run(BotConfig.token) +bot.run(constants.Bot.token) diff --git a/bot/async_stats.py b/bot/async_stats.py new file mode 100644 index 000000000..58a80f528 --- /dev/null +++ b/bot/async_stats.py @@ -0,0 +1,39 @@ +import asyncio +import socket + +from statsd.client.base import StatsClientBase + + +class AsyncStatsClient(StatsClientBase): + """An async transport method for statsd communication.""" + + def __init__( + self, + loop: asyncio.AbstractEventLoop, + host: str = 'localhost', + port: int = 8125, + prefix: str = None + ): + """Create a new client.""" + family, _, _, _, addr = socket.getaddrinfo( + host, port, socket.AF_INET, socket.SOCK_DGRAM)[0] + self._addr = addr + self._prefix = prefix + self._loop = loop + self._transport = None + + async def create_socket(self) -> None: + """Use the loop.create_datagram_endpoint method to create a socket.""" + self._transport, _ = await self._loop.create_datagram_endpoint( + asyncio.DatagramProtocol, + family=socket.AF_INET, + remote_addr=self._addr + ) + + def _send(self, data: str) -> None: + """Start an async task to send data to statsd.""" + self._loop.create_task(self._async_send(data)) + + async def _async_send(self, data: str) -> None: + """Send data to the statsd server using the async transport.""" + self._transport.sendto(data.encode('ascii'), self._addr) diff --git a/bot/bot.py b/bot/bot.py index 950ac6751..b2e5237fe 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -2,14 +2,17 @@ import asyncio import logging import socket import warnings -from typing import Optional +from collections import defaultdict +from typing import Dict, Optional import aiohttp import discord +from async_rediscache import RedisSession from discord.ext import commands +from sentry_sdk import push_scope -from bot import api -from bot import constants +from bot import DEBUG_MODE, api, constants +from bot.async_stats import AsyncStatsClient log = logging.getLogger('bot') @@ -17,7 +20,7 @@ log = logging.getLogger('bot') class Bot(commands.Bot): """A subclass of `discord.ext.commands.Bot` with an aiohttp session and an API client.""" - def __init__(self, *args, **kwargs): + def __init__(self, *args, redis_session: RedisSession, **kwargs): if "connector" in kwargs: warnings.warn( "If login() is called (or the bot is started), the connector will be overwritten " @@ -27,50 +30,33 @@ class Bot(commands.Bot): super().__init__(*args, **kwargs) self.http_session: Optional[aiohttp.ClientSession] = None + self.redis_session = redis_session self.api_client = api.APIClient(loop=self.loop) + self.filter_list_cache = defaultdict(dict) self._connector = None self._resolver = None self._guild_available = asyncio.Event() - def add_cog(self, cog: commands.Cog) -> None: - """Adds a "cog" to the bot and logs the operation.""" - super().add_cog(cog) - log.info(f"Cog loaded: {cog.qualified_name}") - - def clear(self) -> None: - """ - Clears the internal state of the bot and recreates the connector and sessions. - - Will cause a DeprecationWarning if called outside a coroutine. - """ - # Because discord.py recreates the HTTPClient session, may as well follow suit and recreate - # our own stuff here too. - self._recreate() - super().clear() + statsd_url = constants.Stats.statsd_host - async def close(self) -> None: - """Close the Discord connection and the aiohttp session, connector, and resolver.""" - await super().close() + if DEBUG_MODE: + # Since statsd is UDP, there are no errors for sending to a down port. + # For this reason, setting the statsd host to 127.0.0.1 for development + # will effectively disable stats. + statsd_url = "127.0.0.1" - await self.api_client.close() + self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") - if self.http_session: - await self.http_session.close() + async def cache_filter_list_data(self) -> None: + """Cache all the data in the FilterList on the site.""" + full_cache = await self.api_client.get('bot/filter-lists') - if self._connector: - await self._connector.close() - - if self._resolver: - await self._resolver.close() - - async def login(self, *args, **kwargs) -> None: - """Re-create the connector and set up sessions before logging into Discord.""" - self._recreate() - await super().login(*args, **kwargs) + for item in full_cache: + self.insert_item_into_filter_list_cache(item) def _recreate(self) -> None: - """Re-create the connector, aiohttp session, and the APIClient.""" + """Re-create the connector, aiohttp session, the APIClient and the Redis session.""" # Use asyncio for DNS resolution instead of threads so threads aren't spammed. # Doesn't seem to have any state with regards to being closed, so no need to worry? self._resolver = aiohttp.AsyncResolver() @@ -81,6 +67,11 @@ class Bot(commands.Bot): "The previous connector was not closed; it will remain open and be overwritten" ) + if self.redis_session.closed: + # If the RedisSession was somehow closed, we try to reconnect it + # here. Normally, this shouldn't happen. + self.loop.create_task(self.redis_session.connect()) + # Use AF_INET as its socket family to prevent HTTPS related problems both locally # and in production. self._connector = aiohttp.TCPConnector( @@ -101,6 +92,85 @@ class Bot(commands.Bot): self.http_session = aiohttp.ClientSession(connector=self._connector) self.api_client.recreate(force=True, connector=self._connector) + # Build the FilterList cache + self.loop.create_task(self.cache_filter_list_data()) + + def add_cog(self, cog: commands.Cog) -> None: + """Adds a "cog" to the bot and logs the operation.""" + super().add_cog(cog) + log.info(f"Cog loaded: {cog.qualified_name}") + + def add_command(self, command: commands.Command) -> None: + """Add `command` as normal and then add its root aliases to the bot.""" + super().add_command(command) + self._add_root_aliases(command) + + def remove_command(self, name: str) -> Optional[commands.Command]: + """ + Remove a command/alias as normal and then remove its root aliases from the bot. + + Individual root aliases cannot be removed by this function. + To remove them, either remove the entire command or manually edit `bot.all_commands`. + """ + command = super().remove_command(name) + if command is None: + # Even if it's a root alias, there's no way to get the Bot instance to remove the alias. + return + + self._remove_root_aliases(command) + return command + + def clear(self) -> None: + """ + Clears the internal state of the bot and recreates the connector and sessions. + + Will cause a DeprecationWarning if called outside a coroutine. + """ + # Because discord.py recreates the HTTPClient session, may as well follow suit and recreate + # our own stuff here too. + self._recreate() + super().clear() + + async def close(self) -> None: + """Close the Discord connection and the aiohttp session, connector, statsd client, and resolver.""" + await super().close() + + await self.api_client.close() + + if self.http_session: + await self.http_session.close() + + if self._connector: + await self._connector.close() + + if self._resolver: + await self._resolver.close() + + if self.stats._transport: + self.stats._transport.close() + + if self.redis_session: + await self.redis_session.close() + + def insert_item_into_filter_list_cache(self, item: Dict[str, str]) -> None: + """Add an item to the bots filter_list_cache.""" + type_ = item["type"] + allowed = item["allowed"] + content = item["content"] + + self.filter_list_cache[f"{type_}.{allowed}"][content] = { + "id": item["id"], + "comment": item["comment"], + "created_at": item["created_at"], + "updated_at": item["updated_at"], + } + + async def login(self, *args, **kwargs) -> None: + """Re-create the connector and set up sessions before logging into Discord.""" + self._recreate() + await self.stats.create_socket() + await super().login(*args, **kwargs) + async def on_guild_available(self, guild: discord.Guild) -> None: """ Set the internal guild available event when constants.Guild.id becomes available. @@ -141,3 +211,35 @@ class Bot(commands.Bot): gateway event before giving up and thus not populating the cache for unavailable guilds. """ await self._guild_available.wait() + + async def on_error(self, event: str, *args, **kwargs) -> None: + """Log errors raised in event listeners rather than printing them to stderr.""" + self.stats.incr(f"errors.event.{event}") + + with push_scope() as scope: + scope.set_tag("event", event) + scope.set_extra("args", args) + scope.set_extra("kwargs", kwargs) + + log.exception(f"Unhandled exception in {event}.") + + def _add_root_aliases(self, command: commands.Command) -> None: + """Recursively add root aliases for `command` and any of its subcommands.""" + if isinstance(command, commands.Group): + for subcommand in command.commands: + self._add_root_aliases(subcommand) + + for alias in getattr(command, "root_aliases", ()): + if alias in self.all_commands: + raise commands.CommandRegistrationError(alias, alias_conflict=True) + + self.all_commands[alias] = command + + def _remove_root_aliases(self, command: commands.Command) -> None: + """Recursively remove root aliases for `command` and any of its subcommands.""" + if isinstance(command, commands.Group): + for subcommand in command.commands: + self._remove_root_aliases(subcommand) + + for alias in getattr(command, "root_aliases", ()): + self.all_commands.pop(alias, None) diff --git a/bot/cogs/alias.py b/bot/cogs/alias.py deleted file mode 100644 index d7e49b390..000000000 --- a/bot/cogs/alias.py +++ /dev/null @@ -1,153 +0,0 @@ -import inspect -import logging - -from discord import Colour, Embed -from discord.ext.commands import ( - Cog, Command, Context, Greedy, - clean_content, command, group, -) - -from bot.bot import Bot -from bot.cogs.extensions import Extension -from bot.converters import FetchedMember, TagNameConverter -from bot.pagination import LinePaginator - -log = logging.getLogger(__name__) - - -class Alias (Cog): - """Aliases for commonly used commands.""" - - def __init__(self, bot: Bot): - self.bot = bot - - async def invoke(self, ctx: Context, cmd_name: str, *args, **kwargs) -> None: - """Invokes a command with args and kwargs.""" - log.debug(f"{cmd_name} was invoked through an alias") - cmd = self.bot.get_command(cmd_name) - if not cmd: - return log.info(f'Did not find command "{cmd_name}" to invoke.') - elif not await cmd.can_run(ctx): - return log.info( - f'{str(ctx.author)} tried to run the command "{cmd_name}" but lacks permission.' - ) - - await cmd(ctx, *args, **kwargs) - - @command(name='aliases') - async def aliases_command(self, ctx: Context) -> None: - """Show configured aliases on the bot.""" - embed = Embed( - title='Configured aliases', - colour=Colour.blue() - ) - await LinePaginator.paginate( - ( - f"• `{ctx.prefix}{value.name}` " - f"=> `{ctx.prefix}{name[:-len('_alias')].replace('_', ' ')}`" - for name, value in inspect.getmembers(self) - if isinstance(value, Command) and name.endswith('_alias') - ), - ctx, embed, empty=False, max_lines=20 - ) - - @command(name="resources", aliases=("resource",), hidden=True) - async def site_resources_alias(self, ctx: Context) -> None: - """Alias for invoking <prefix>site resources.""" - await self.invoke(ctx, "site resources") - - @command(name="tools", hidden=True) - async def site_tools_alias(self, ctx: Context) -> None: - """Alias for invoking <prefix>site tools.""" - await self.invoke(ctx, "site tools") - - @command(name="watch", hidden=True) - async def bigbrother_watch_alias(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: - """Alias for invoking <prefix>bigbrother watch [user] [reason].""" - await self.invoke(ctx, "bigbrother watch", user, reason=reason) - - @command(name="unwatch", hidden=True) - async def bigbrother_unwatch_alias(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: - """Alias for invoking <prefix>bigbrother unwatch [user] [reason].""" - await self.invoke(ctx, "bigbrother unwatch", user, reason=reason) - - @command(name="home", hidden=True) - async def site_home_alias(self, ctx: Context) -> None: - """Alias for invoking <prefix>site home.""" - await self.invoke(ctx, "site home") - - @command(name="faq", hidden=True) - async def site_faq_alias(self, ctx: Context) -> None: - """Alias for invoking <prefix>site faq.""" - await self.invoke(ctx, "site faq") - - @command(name="rules", aliases=("rule",), hidden=True) - async def site_rules_alias(self, ctx: Context, rules: Greedy[int], *_: str) -> None: - """Alias for invoking <prefix>site rules.""" - await self.invoke(ctx, "site rules", *rules) - - @command(name="reload", hidden=True) - async def extensions_reload_alias(self, ctx: Context, *extensions: Extension) -> None: - """Alias for invoking <prefix>extensions reload [extensions...].""" - await self.invoke(ctx, "extensions reload", *extensions) - - @command(name="defon", hidden=True) - async def defcon_enable_alias(self, ctx: Context) -> None: - """Alias for invoking <prefix>defcon enable.""" - await self.invoke(ctx, "defcon enable") - - @command(name="defoff", hidden=True) - async def defcon_disable_alias(self, ctx: Context) -> None: - """Alias for invoking <prefix>defcon disable.""" - await self.invoke(ctx, "defcon disable") - - @command(name="exception", hidden=True) - async def tags_get_traceback_alias(self, ctx: Context) -> None: - """Alias for invoking <prefix>tags get traceback.""" - await self.invoke(ctx, "tags get", tag_name="traceback") - - @group(name="get", - aliases=("show", "g"), - hidden=True, - invoke_without_command=True) - async def get_group_alias(self, ctx: Context) -> None: - """Group for reverse aliases for commands like `tags get`, allowing for `get tags` or `get docs`.""" - pass - - @get_group_alias.command(name="tags", aliases=("tag", "t"), hidden=True) - async def tags_get_alias( - self, ctx: Context, *, tag_name: TagNameConverter = None - ) -> None: - """ - Alias for invoking <prefix>tags get [tag_name]. - - tag_name: str - tag to be viewed. - """ - await self.invoke(ctx, "tags get", tag_name=tag_name) - - @get_group_alias.command(name="docs", aliases=("doc", "d"), hidden=True) - async def docs_get_alias( - self, ctx: Context, symbol: clean_content = None - ) -> None: - """Alias for invoking <prefix>docs get [symbol].""" - await self.invoke(ctx, "docs get", symbol) - - @command(name="nominate", hidden=True) - async def nomination_add_alias(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: - """Alias for invoking <prefix>talentpool add [user] [reason].""" - await self.invoke(ctx, "talentpool add", user, reason=reason) - - @command(name="unnominate", hidden=True) - async def nomination_end_alias(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: - """Alias for invoking <prefix>nomination end [user] [reason].""" - await self.invoke(ctx, "nomination end", user, reason=reason) - - @command(name="nominees", hidden=True) - async def nominees_alias(self, ctx: Context) -> None: - """Alias for invoking <prefix>tp watched.""" - await self.invoke(ctx, "talentpool watched") - - -def setup(bot: Bot) -> None: - """Load the Alias cog.""" - bot.add_cog(Alias(bot)) diff --git a/bot/cogs/antimalware.py b/bot/cogs/antimalware.py deleted file mode 100644 index 79bf486a4..000000000 --- a/bot/cogs/antimalware.py +++ /dev/null @@ -1,67 +0,0 @@ -import logging -from os.path import splitext - -from discord import Embed, Message, NotFound -from discord.ext.commands import Cog - -from bot.bot import Bot -from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES, URLs - -log = logging.getLogger(__name__) - - -class AntiMalware(Cog): - """Delete messages which contain attachments with non-whitelisted file extensions.""" - - def __init__(self, bot: Bot): - self.bot = bot - - @Cog.listener() - async def on_message(self, message: Message) -> None: - """Identify messages with prohibited attachments.""" - # Return when message don't have attachment and don't moderate DMs - if not message.attachments or not message.guild: - return - - # Check if user is staff, if is, return - # Since we only care that roles exist to iterate over, check for the attr rather than a User/Member instance - if hasattr(message.author, "roles") and any(role.id in STAFF_ROLES for role in message.author.roles): - return - - embed = Embed() - file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} - extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) - blocked_extensions_str = ', '.join(extensions_blocked) - if ".py" in extensions_blocked: - # Short-circuit on *.py files to provide a pastebin link - embed.description = ( - "It looks like you tried to attach a Python file - " - f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" - ) - elif extensions_blocked: - whitelisted_types = ', '.join(AntiMalwareConfig.whitelist) - meta_channel = self.bot.get_channel(Channels.meta) - embed.description = ( - f"It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " - f"We currently allow the following file types: **{whitelisted_types}**.\n\n" - f"Feel free to ask in {meta_channel.mention} if you think this is a mistake." - ) - - if embed.description: - log.info( - f"User '{message.author}' ({message.author.id}) uploaded blacklisted file(s): {blocked_extensions_str}", - extra={"attachment_list": [attachment.filename for attachment in message.attachments]} - ) - - await message.channel.send(f"Hey {message.author.mention}!", embed=embed) - - # Delete the offending message: - try: - await message.delete() - except NotFound: - log.info(f"Tried to delete message `{message.id}`, but message could not be found.") - - -def setup(bot: Bot) -> None: - """Load the AntiMalware cog.""" - bot.add_cog(AntiMalware(bot)) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py deleted file mode 100644 index 3f3dbb853..000000000 --- a/bot/cogs/filtering.py +++ /dev/null @@ -1,369 +0,0 @@ -import logging -import re -from typing import Optional, Union - -import discord.errors -from dateutil.relativedelta import relativedelta -from discord import Colour, DMChannel, Member, Message, TextChannel -from discord.ext.commands import Cog -from discord.utils import escape_markdown - -from bot.bot import Bot -from bot.cogs.moderation import ModLog -from bot.constants import ( - Channels, Colours, - Filter, Icons, URLs -) - -log = logging.getLogger(__name__) - -INVITE_RE = re.compile( - r"(?:discord(?:[\.,]|dot)gg|" # Could be discord.gg/ - r"discord(?:[\.,]|dot)com(?:\/|slash)invite|" # or discord.com/invite/ - r"discordapp(?:[\.,]|dot)com(?:\/|slash)invite|" # or discordapp.com/invite/ - r"discord(?:[\.,]|dot)me|" # or discord.me - r"discord(?:[\.,]|dot)io" # or discord.io. - r")(?:[\/]|slash)" # / or 'slash' - r"([a-zA-Z0-9]+)", # the invite code itself - flags=re.IGNORECASE -) - -SPOILER_RE = re.compile(r"(\|\|.+?\|\|)", re.DOTALL) -URL_RE = re.compile(r"(https?://[^\s]+)", flags=re.IGNORECASE) -ZALGO_RE = re.compile(r"[\u0300-\u036F\u0489]") - -WORD_WATCHLIST_PATTERNS = [ - re.compile(fr'\b{expression}\b', flags=re.IGNORECASE) for expression in Filter.word_watchlist -] -TOKEN_WATCHLIST_PATTERNS = [ - re.compile(fr'{expression}', flags=re.IGNORECASE) for expression in Filter.token_watchlist -] -WATCHLIST_PATTERNS = WORD_WATCHLIST_PATTERNS + TOKEN_WATCHLIST_PATTERNS - - -def expand_spoilers(text: str) -> str: - """Return a string containing all interpretations of a spoilered message.""" - split_text = SPOILER_RE.split(text) - return ''.join( - split_text[0::2] + split_text[1::2] + split_text - ) - - -class Filtering(Cog): - """Filtering out invites, blacklisting domains, and warning us of certain regular expressions.""" - - def __init__(self, bot: Bot): - self.bot = bot - - staff_mistake_str = "If you believe this was a mistake, please let staff know!" - self.filters = { - "filter_zalgo": { - "enabled": Filter.filter_zalgo, - "function": self._has_zalgo, - "type": "filter", - "content_only": True, - "user_notification": Filter.notify_user_zalgo, - "notification_msg": ( - "Your post has been removed for abusing Unicode character rendering (aka Zalgo text). " - f"{staff_mistake_str}" - ) - }, - "filter_invites": { - "enabled": Filter.filter_invites, - "function": self._has_invites, - "type": "filter", - "content_only": True, - "user_notification": Filter.notify_user_invites, - "notification_msg": ( - f"Per Rule 6, your invite link has been removed. {staff_mistake_str}\n\n" - r"Our server rules can be found here: <https://pythondiscord.com/pages/rules>" - ) - }, - "filter_domains": { - "enabled": Filter.filter_domains, - "function": self._has_urls, - "type": "filter", - "content_only": True, - "user_notification": Filter.notify_user_domains, - "notification_msg": ( - f"Your URL has been removed because it matched a blacklisted domain. {staff_mistake_str}" - ) - }, - "watch_regex": { - "enabled": Filter.watch_regex, - "function": self._has_watch_regex_match, - "type": "watchlist", - "content_only": True, - }, - "watch_rich_embeds": { - "enabled": Filter.watch_rich_embeds, - "function": self._has_rich_embed, - "type": "watchlist", - "content_only": False, - }, - } - - @property - def mod_log(self) -> ModLog: - """Get currently loaded ModLog cog instance.""" - return self.bot.get_cog("ModLog") - - @Cog.listener() - async def on_message(self, msg: Message) -> None: - """Invoke message filter for new messages.""" - await self._filter_message(msg) - - @Cog.listener() - async def on_message_edit(self, before: Message, after: Message) -> None: - """ - Invoke message filter for message edits. - - If there have been multiple edits, calculate the time delta from the previous edit. - """ - if not before.edited_at: - delta = relativedelta(after.edited_at, before.created_at).microseconds - else: - delta = relativedelta(after.edited_at, before.edited_at).microseconds - await self._filter_message(after, delta) - - async def _filter_message(self, msg: Message, delta: Optional[int] = None) -> None: - """Filter the input message to see if it violates any of our rules, and then respond accordingly.""" - # Should we filter this message? - role_whitelisted = False - - if type(msg.author) is Member: # Only Member has roles, not User. - for role in msg.author.roles: - if role.id in Filter.role_whitelist: - role_whitelisted = True - - filter_message = ( - msg.channel.id not in Filter.channel_whitelist # Channel not in whitelist - and not role_whitelisted # Role not in whitelist - and not msg.author.bot # Author not a bot - ) - - # If none of the above, we can start filtering. - if filter_message: - for filter_name, _filter in self.filters.items(): - # Is this specific filter enabled in the config? - if _filter["enabled"]: - # Double trigger check for the embeds filter - if filter_name == "watch_rich_embeds": - # If the edit delta is less than 0.001 seconds, then we're probably dealing - # with a double filter trigger. - if delta is not None and delta < 100: - continue - - # Does the filter only need the message content or the full message? - if _filter["content_only"]: - match = await _filter["function"](msg.content) - else: - match = await _filter["function"](msg) - - if match: - # If this is a filter (not a watchlist), we should delete the message. - if _filter["type"] == "filter": - try: - # Embeds (can?) trigger both the `on_message` and `on_message_edit` - # event handlers, triggering filtering twice for the same message. - # - # If `on_message`-triggered filtering already deleted the message - # then `on_message_edit`-triggered filtering will raise exception - # since the message no longer exists. - # - # In addition, to avoid sending two notifications to the user, the - # logs, and mod_alert, we return if the message no longer exists. - await msg.delete() - except discord.errors.NotFound: - return - - # Notify the user if the filter specifies - if _filter["user_notification"]: - await self.notify_member(msg.author, _filter["notification_msg"], msg.channel) - - if isinstance(msg.channel, DMChannel): - channel_str = "via DM" - else: - channel_str = f"in {msg.channel.mention}" - - # Word and match stats for watch_regex - if filter_name == "watch_regex": - surroundings = match.string[max(match.start() - 10, 0): match.end() + 10] - message_content = ( - f"**Match:** '{match[0]}'\n" - f"**Location:** '...{escape_markdown(surroundings)}...'\n" - f"\n**Original Message:**\n{escape_markdown(msg.content)}" - ) - else: # Use content of discord Message - message_content = msg.content - - message = ( - f"The {filter_name} {_filter['type']} was triggered " - f"by **{msg.author}** " - f"(`{msg.author.id}`) {channel_str} with [the " - f"following message]({msg.jump_url}):\n\n" - f"{message_content}" - ) - - log.debug(message) - - additional_embeds = None - additional_embeds_msg = None - - if filter_name == "filter_invites": - additional_embeds = [] - for invite, data in match.items(): - embed = discord.Embed(description=( - f"**Members:**\n{data['members']}\n" - f"**Active:**\n{data['active']}" - )) - embed.set_author(name=data["name"]) - embed.set_thumbnail(url=data["icon"]) - embed.set_footer(text=f"Guild Invite Code: {invite}") - additional_embeds.append(embed) - additional_embeds_msg = "For the following guild(s):" - - elif filter_name == "watch_rich_embeds": - additional_embeds = msg.embeds - additional_embeds_msg = "With the following embed(s):" - - # Send pretty mod log embed to mod-alerts - await self.mod_log.send_log_message( - icon_url=Icons.filtering, - colour=Colour(Colours.soft_red), - title=f"{_filter['type'].title()} triggered!", - text=message, - thumbnail=msg.author.avatar_url_as(static_format="png"), - channel_id=Channels.mod_alerts, - ping_everyone=Filter.ping_everyone, - additional_embeds=additional_embeds, - additional_embeds_msg=additional_embeds_msg - ) - - break # We don't want multiple filters to trigger - - @staticmethod - async def _has_watch_regex_match(text: str) -> Union[bool, re.Match]: - """ - Return True if `text` matches any regex from `word_watchlist` or `token_watchlist` configs. - - `word_watchlist`'s patterns are placed between word boundaries while `token_watchlist` is - matched as-is. Spoilers are expanded, if any, and URLs are ignored. - """ - if SPOILER_RE.search(text): - text = expand_spoilers(text) - - # Make sure it's not a URL - if URL_RE.search(text): - return False - - for pattern in WATCHLIST_PATTERNS: - match = pattern.search(text) - if match: - return match - - @staticmethod - async def _has_urls(text: str) -> bool: - """Returns True if the text contains one of the blacklisted URLs from the config file.""" - if not URL_RE.search(text): - return False - - text = text.lower() - - for url in Filter.domain_blacklist: - if url.lower() in text: - return True - - return False - - @staticmethod - async def _has_zalgo(text: str) -> bool: - """ - Returns True if the text contains zalgo characters. - - Zalgo range is \u0300 – \u036F and \u0489. - """ - return bool(ZALGO_RE.search(text)) - - async def _has_invites(self, text: str) -> Union[dict, bool]: - """ - Checks if there's any invites in the text content that aren't in the guild whitelist. - - If any are detected, a dictionary of invite data is returned, with a key per invite. - If none are detected, False is returned. - - Attempts to catch some of common ways to try to cheat the system. - """ - # Remove backslashes to prevent escape character aroundfuckery like - # discord\.gg/gdudes-pony-farm - text = text.replace("\\", "") - - invites = INVITE_RE.findall(text) - invite_data = dict() - for invite in invites: - if invite in invite_data: - continue - - response = await self.bot.http_session.get( - f"{URLs.discord_invite_api}/{invite}", params={"with_counts": "true"} - ) - response = await response.json() - guild = response.get("guild") - if guild is None: - # Lack of a "guild" key in the JSON response indicates either an group DM invite, an - # expired invite, or an invalid invite. The API does not currently differentiate - # between invalid and expired invites - return True - - guild_id = int(guild.get("id")) - - if guild_id not in Filter.guild_invite_whitelist: - guild_icon_hash = guild["icon"] - guild_icon = ( - "https://cdn.discordapp.com/icons/" - f"{guild_id}/{guild_icon_hash}.png?size=512" - ) - - invite_data[invite] = { - "name": guild["name"], - "icon": guild_icon, - "members": response["approximate_member_count"], - "active": response["approximate_presence_count"] - } - - return invite_data if invite_data else False - - @staticmethod - async def _has_rich_embed(msg: Message) -> bool: - """Determines if `msg` contains any rich embeds not auto-generated from a URL.""" - if msg.embeds: - for embed in msg.embeds: - if embed.type == "rich": - urls = URL_RE.findall(msg.content) - if not embed.url or embed.url not in urls: - # If `embed.url` does not exist or if `embed.url` is not part of the content - # of the message, it's unlikely to be an auto-generated embed by Discord. - return True - else: - log.trace( - "Found a rich embed sent by a regular user account, " - "but it was likely just an automatic URL embed." - ) - return False - return False - - async def notify_member(self, filtered_member: Member, reason: str, channel: TextChannel) -> None: - """ - Notify filtered_member about a moderation action with the reason str. - - First attempts to DM the user, fall back to in-channel notification if user has DMs disabled - """ - try: - await filtered_member.send(reason) - except discord.errors.Forbidden: - await channel.send(f"{filtered_member.mention} {reason}") - - -def setup(bot: Bot) -> None: - """Load the Filtering cog.""" - bot.add_cog(Filtering(bot)) diff --git a/bot/cogs/free.py b/bot/cogs/free.py deleted file mode 100644 index 33b55e79a..000000000 --- a/bot/cogs/free.py +++ /dev/null @@ -1,103 +0,0 @@ -import logging -from datetime import datetime -from operator import itemgetter - -from discord import Colour, Embed, Member, utils -from discord.ext.commands import Cog, Context, command - -from bot.bot import Bot -from bot.constants import Categories, Channels, Free, STAFF_ROLES -from bot.decorators import redirect_output - -log = logging.getLogger(__name__) - -TIMEOUT = Free.activity_timeout -RATE = Free.cooldown_rate -PER = Free.cooldown_per - - -class Free(Cog): - """Tries to figure out which help channels are free.""" - - PYTHON_HELP_ID = Categories.python_help - - @command(name="free", aliases=('f',)) - @redirect_output(destination_channel=Channels.bot_commands, bypass_roles=STAFF_ROLES) - async def free(self, ctx: Context, user: Member = None, seek: int = 2) -> None: - """ - Lists free help channels by likeliness of availability. - - seek is used only when this command is invoked in a help channel. - You cannot override seek without mentioning a user first. - - When seek is 2, we are avoiding considering the last active message - in a channel to be the one that invoked this command. - - When seek is 3 or more, a user has been mentioned on the assumption - that they asked if the channel is free or they asked their question - in an active channel, and we want the message before that happened. - """ - free_channels = [] - python_help = utils.get(ctx.guild.categories, id=self.PYTHON_HELP_ID) - - if user is not None and seek == 2: - seek = 3 - elif not 0 < seek < 10: - seek = 3 - - # Iterate through all the help channels - # to check latest activity - for channel in python_help.channels: - # Seek further back in the help channel - # the command was invoked in - if channel.id == ctx.channel.id: - messages = await channel.history(limit=seek).flatten() - msg = messages[seek - 1] - # Otherwise get last message - else: - msg = await channel.history(limit=1).next() # noqa: B305 - - inactive = (datetime.utcnow() - msg.created_at).seconds - if inactive > TIMEOUT: - free_channels.append((inactive, channel)) - - embed = Embed() - embed.colour = Colour.blurple() - embed.title = "**Looking for a free help channel?**" - - if user is not None: - embed.description = f"**Hey {user.mention}!**\n\n" - else: - embed.description = "" - - # Display all potentially inactive channels - # in descending order of inactivity - if free_channels: - # Sort channels in descending order by seconds - # Get position in list, inactivity, and channel object - # For each channel, add to embed.description - sorted_channels = sorted(free_channels, key=itemgetter(0), reverse=True) - - for (inactive, channel) in sorted_channels[:3]: - minutes, seconds = divmod(inactive, 60) - if minutes > 59: - hours, minutes = divmod(minutes, 60) - embed.description += f"{channel.mention} **{hours}h {minutes}m {seconds}s** inactive\n" - else: - embed.description += f"{channel.mention} **{minutes}m {seconds}s** inactive\n" - - embed.set_footer(text="Please confirm these channels are free before posting") - else: - embed.description = ( - "Doesn't look like any channels are available right now. " - "You're welcome to check for yourself to be sure. " - "If all channels are truly busy, please be patient " - "as one will likely be available soon." - ) - - await ctx.send(embed=embed) - - -def setup(bot: Bot) -> None: - """Load the Free cog.""" - bot.add_cog(Free()) diff --git a/bot/cogs/help.py b/bot/cogs/help.py deleted file mode 100644 index 744722220..000000000 --- a/bot/cogs/help.py +++ /dev/null @@ -1,568 +0,0 @@ -import asyncio -import itertools -from collections import namedtuple -from contextlib import suppress -from typing import Union - -from discord import Colour, Embed, HTTPException, Message, Reaction, User -from discord.ext import commands -from discord.ext.commands import CheckFailure, Cog as DiscordCog, Command, Context -from fuzzywuzzy import fuzz, process - -from bot import constants -from bot.bot import Bot -from bot.constants import Channels, Emojis, STAFF_ROLES -from bot.decorators import redirect_output -from bot.pagination import ( - FIRST_EMOJI, LAST_EMOJI, - LEFT_EMOJI, LinePaginator, RIGHT_EMOJI, -) - -DELETE_EMOJI = Emojis.trashcan - -REACTIONS = { - FIRST_EMOJI: 'first', - LEFT_EMOJI: 'back', - RIGHT_EMOJI: 'next', - LAST_EMOJI: 'end', - DELETE_EMOJI: 'stop', -} - -Cog = namedtuple('Cog', ['name', 'description', 'commands']) - - -class HelpQueryNotFound(ValueError): - """ - Raised when a HelpSession Query doesn't match a command or cog. - - Contains the custom attribute of ``possible_matches``. - - Instances of this object contain a dictionary of any command(s) that were close to matching the - query, where keys are the possible matched command names and values are the likeness match scores. - """ - - def __init__(self, arg: str, possible_matches: dict = None): - super().__init__(arg) - self.possible_matches = possible_matches - - -class HelpSession: - """ - An interactive session for bot and command help output. - - Expected attributes include: - * title: str - The title of the help message. - * query: Union[discord.ext.commands.Bot, discord.ext.commands.Command] - * description: str - The description of the query. - * pages: list[str] - A list of the help content split into manageable pages. - * message: `discord.Message` - The message object that's showing the help contents. - * destination: `discord.abc.Messageable` - Where the help message is to be sent to. - - Cogs can be grouped into custom categories. All cogs with the same category will be displayed - under a single category name in the help output. Custom categories are defined inside the cogs - as a class attribute named `category`. A description can also be specified with the attribute - `category_description`. If a description is not found in at least one cog, the default will be - the regular description (class docstring) of the first cog found in the category. - """ - - def __init__( - self, - ctx: Context, - *command, - cleanup: bool = False, - only_can_run: bool = True, - show_hidden: bool = False, - max_lines: int = 15 - ): - """Creates an instance of the HelpSession class.""" - self._ctx = ctx - self._bot = ctx.bot - self.title = "Command Help" - - # set the query details for the session - if command: - query_str = ' '.join(command) - self.query = self._get_query(query_str) - self.description = self.query.description or self.query.help - else: - self.query = ctx.bot - self.description = self.query.description - self.author = ctx.author - self.destination = ctx.channel - - # set the config for the session - self._cleanup = cleanup - self._only_can_run = only_can_run - self._show_hidden = show_hidden - self._max_lines = max_lines - - # init session states - self._pages = None - self._current_page = 0 - self.message = None - self._timeout_task = None - self.reset_timeout() - - def _get_query(self, query: str) -> Union[Command, Cog]: - """Attempts to match the provided query with a valid command or cog.""" - command = self._bot.get_command(query) - if command: - return command - - # Find all cog categories that match. - cog_matches = [] - description = None - for cog in self._bot.cogs.values(): - if hasattr(cog, "category") and cog.category == query: - cog_matches.append(cog) - if hasattr(cog, "category_description"): - description = cog.category_description - - # Try to search by cog name if no categories match. - if not cog_matches: - cog = self._bot.cogs.get(query) - - # Don't consider it a match if the cog has a category. - if cog and not hasattr(cog, "category"): - cog_matches = [cog] - - if cog_matches: - cog = cog_matches[0] - cmds = (cog.get_commands() for cog in cog_matches) # Commands of all cogs - - return Cog( - name=cog.category if hasattr(cog, "category") else cog.qualified_name, - description=description or cog.description, - commands=tuple(itertools.chain.from_iterable(cmds)) # Flatten the list - ) - - self._handle_not_found(query) - - def _handle_not_found(self, query: str) -> None: - """ - Handles when a query does not match a valid command or cog. - - Will pass on possible close matches along with the `HelpQueryNotFound` exception. - """ - # Combine command and cog names - choices = list(self._bot.all_commands) + list(self._bot.cogs) - - result = process.extractBests(query, choices, scorer=fuzz.ratio, score_cutoff=90) - - raise HelpQueryNotFound(f'Query "{query}" not found.', dict(result)) - - async def timeout(self, seconds: int = 30) -> None: - """Waits for a set number of seconds, then stops the help session.""" - await asyncio.sleep(seconds) - await self.stop() - - def reset_timeout(self) -> None: - """Cancels the original timeout task and sets it again from the start.""" - # cancel original if it exists - if self._timeout_task: - if not self._timeout_task.cancelled(): - self._timeout_task.cancel() - - # recreate the timeout task - self._timeout_task = self._bot.loop.create_task(self.timeout()) - - async def on_reaction_add(self, reaction: Reaction, user: User) -> None: - """Event handler for when reactions are added on the help message.""" - # ensure it was the relevant session message - if reaction.message.id != self.message.id: - return - - # ensure it was the session author who reacted - if user.id != self.author.id: - return - - emoji = str(reaction.emoji) - - # check if valid action - if emoji not in REACTIONS: - return - - self.reset_timeout() - - # Run relevant action method - action = getattr(self, f'do_{REACTIONS[emoji]}', None) - if action: - await action() - - # remove the added reaction to prep for re-use - with suppress(HTTPException): - await self.message.remove_reaction(reaction, user) - - async def on_message_delete(self, message: Message) -> None: - """Closes the help session when the help message is deleted.""" - if message.id == self.message.id: - await self.stop() - - async def prepare(self) -> None: - """Sets up the help session pages, events, message and reactions.""" - # create paginated content - await self.build_pages() - - # setup listeners - self._bot.add_listener(self.on_reaction_add) - self._bot.add_listener(self.on_message_delete) - - # Send the help message - await self.update_page() - self.add_reactions() - - def add_reactions(self) -> None: - """Adds the relevant reactions to the help message based on if pagination is required.""" - # if paginating - if len(self._pages) > 1: - for reaction in REACTIONS: - self._bot.loop.create_task(self.message.add_reaction(reaction)) - - # if single-page - else: - self._bot.loop.create_task(self.message.add_reaction(DELETE_EMOJI)) - - def _category_key(self, cmd: Command) -> str: - """ - Returns a cog name of a given command for use as a key for `sorted` and `groupby`. - - A zero width space is used as a prefix for results with no cogs to force them last in ordering. - """ - if cmd.cog: - try: - if cmd.cog.category: - return f'**{cmd.cog.category}**' - except AttributeError: - pass - - return f'**{cmd.cog_name}**' - else: - return "**\u200bNo Category:**" - - def _get_command_params(self, cmd: Command) -> str: - """ - Returns the command usage signature. - - This is a custom implementation of `command.signature` in order to format the command - signature without aliases. - """ - results = [] - for name, param in cmd.clean_params.items(): - - # if argument has a default value - if param.default is not param.empty: - - if isinstance(param.default, str): - show_default = param.default - else: - show_default = param.default is not None - - # if default is not an empty string or None - if show_default: - results.append(f'[{name}={param.default}]') - else: - results.append(f'[{name}]') - - # if variable length argument - elif param.kind == param.VAR_POSITIONAL: - results.append(f'[{name}...]') - - # if required - else: - results.append(f'<{name}>') - - return f"{cmd.name} {' '.join(results)}" - - async def build_pages(self) -> None: - """Builds the list of content pages to be paginated through in the help message, as a list of str.""" - # Use LinePaginator to restrict embed line height - paginator = LinePaginator(prefix='', suffix='', max_lines=self._max_lines) - - prefix = constants.Bot.prefix - - # show signature if query is a command - if isinstance(self.query, commands.Command): - signature = self._get_command_params(self.query) - parent = self.query.full_parent_name + ' ' if self.query.parent else '' - paginator.add_line(f'**```{prefix}{parent}{signature}```**') - - # show command aliases - aliases = ', '.join(f'`{a}`' for a in self.query.aliases) - if aliases: - paginator.add_line(f'**Can also use:** {aliases}\n') - - if not await self.query.can_run(self._ctx): - paginator.add_line('***You cannot run this command.***\n') - - # show name if query is a cog - if isinstance(self.query, Cog): - paginator.add_line(f'**{self.query.name}**') - - if self.description: - paginator.add_line(f'*{self.description}*') - - # list all children commands of the queried object - if isinstance(self.query, (commands.GroupMixin, Cog)): - - # remove hidden commands if session is not wanting hiddens - if not self._show_hidden: - filtered = [c for c in self.query.commands if not c.hidden] - else: - filtered = self.query.commands - - # if after filter there are no commands, finish up - if not filtered: - self._pages = paginator.pages - return - - # set category to Commands if cog - if isinstance(self.query, Cog): - grouped = (('**Commands:**', self.query.commands),) - - # set category to Subcommands if command - elif isinstance(self.query, commands.Command): - grouped = (('**Subcommands:**', self.query.commands),) - - # don't show prefix for subcommands - prefix = '' - - # otherwise sort and organise all commands into categories - else: - cat_sort = sorted(filtered, key=self._category_key) - grouped = itertools.groupby(cat_sort, key=self._category_key) - - # process each category - for category, cmds in grouped: - cmds = sorted(cmds, key=lambda c: c.name) - - # if there are no commands, skip category - if len(cmds) == 0: - continue - - cat_cmds = [] - - # format details for each child command - for command in cmds: - - # skip if hidden and hide if session is set to - if command.hidden and not self._show_hidden: - continue - - # see if the user can run the command - strikeout = '' - - # Patch to make the !help command work outside of #bot-commands again - # This probably needs a proper rewrite, but this will make it work in - # the mean time. - try: - can_run = await command.can_run(self._ctx) - except CheckFailure: - can_run = False - - if not can_run: - # skip if we don't show commands they can't run - if self._only_can_run: - continue - strikeout = '~~' - - signature = self._get_command_params(command) - info = f"{strikeout}**`{prefix}{signature}`**{strikeout}" - - # handle if the command has no docstring - if command.short_doc: - cat_cmds.append(f'{info}\n*{command.short_doc}*') - else: - cat_cmds.append(f'{info}\n*No details provided.*') - - # state var for if the category should be added next - print_cat = 1 - new_page = True - - for details in cat_cmds: - - # keep details together, paginating early if it won't fit - lines_adding = len(details.split('\n')) + print_cat - if paginator._linecount + lines_adding > self._max_lines: - paginator._linecount = 0 - new_page = True - paginator.close_page() - - # new page so print category title again - print_cat = 1 - - if print_cat: - if new_page: - paginator.add_line('') - paginator.add_line(category) - print_cat = 0 - - paginator.add_line(details) - - # save organised pages to session - self._pages = paginator.pages - - def embed_page(self, page_number: int = 0) -> Embed: - """Returns an Embed with the requested page formatted within.""" - embed = Embed() - - # if command or cog, add query to title for pages other than first - if isinstance(self.query, (commands.Command, Cog)) and page_number > 0: - title = f'Command Help | "{self.query.name}"' - else: - title = self.title - - embed.set_author(name=title, icon_url=constants.Icons.questionmark) - embed.description = self._pages[page_number] - - # add page counter to footer if paginating - page_count = len(self._pages) - if page_count > 1: - embed.set_footer(text=f'Page {self._current_page+1} / {page_count}') - - return embed - - async def update_page(self, page_number: int = 0) -> None: - """Sends the intial message, or changes the existing one to the given page number.""" - self._current_page = page_number - embed_page = self.embed_page(page_number) - - if not self.message: - self.message = await self.destination.send(embed=embed_page) - else: - await self.message.edit(embed=embed_page) - - @classmethod - async def start(cls, ctx: Context, *command, **options) -> "HelpSession": - """ - Create and begin a help session based on the given command context. - - Available options kwargs: - * cleanup: Optional[bool] - Set to `True` to have the message deleted on session end. Defaults to `False`. - * only_can_run: Optional[bool] - Set to `True` to hide commands the user can't run. Defaults to `False`. - * show_hidden: Optional[bool] - Set to `True` to include hidden commands. Defaults to `False`. - * max_lines: Optional[int] - Sets the max number of lines the paginator will add to a single page. Defaults to 20. - """ - session = cls(ctx, *command, **options) - await session.prepare() - - return session - - async def stop(self) -> None: - """Stops the help session, removes event listeners and attempts to delete the help message.""" - self._bot.remove_listener(self.on_reaction_add) - self._bot.remove_listener(self.on_message_delete) - - # ignore if permission issue, or the message doesn't exist - with suppress(HTTPException, AttributeError): - if self._cleanup: - await self.message.delete() - else: - await self.message.clear_reactions() - - @property - def is_first_page(self) -> bool: - """Check if session is currently showing the first page.""" - return self._current_page == 0 - - @property - def is_last_page(self) -> bool: - """Check if the session is currently showing the last page.""" - return self._current_page == (len(self._pages)-1) - - async def do_first(self) -> None: - """Event that is called when the user requests the first page.""" - if not self.is_first_page: - await self.update_page(0) - - async def do_back(self) -> None: - """Event that is called when the user requests the previous page.""" - if not self.is_first_page: - await self.update_page(self._current_page-1) - - async def do_next(self) -> None: - """Event that is called when the user requests the next page.""" - if not self.is_last_page: - await self.update_page(self._current_page+1) - - async def do_end(self) -> None: - """Event that is called when the user requests the last page.""" - if not self.is_last_page: - await self.update_page(len(self._pages)-1) - - async def do_stop(self) -> None: - """Event that is called when the user requests to stop the help session.""" - await self.message.delete() - - -class Help(DiscordCog): - """Custom Embed Pagination Help feature.""" - - @commands.command('help') - @redirect_output(destination_channel=Channels.bot_commands, bypass_roles=STAFF_ROLES) - async def new_help(self, ctx: Context, *commands) -> None: - """Shows Command Help.""" - try: - await HelpSession.start(ctx, *commands) - except HelpQueryNotFound as error: - embed = Embed() - embed.colour = Colour.red() - embed.title = str(error) - - if error.possible_matches: - matches = '\n'.join(error.possible_matches.keys()) - embed.description = f'**Did you mean:**\n`{matches}`' - - await ctx.send(embed=embed) - - -def unload(bot: Bot) -> None: - """ - Reinstates the original help command. - - This is run if the cog raises an exception on load, or if the extension is unloaded. - """ - bot.remove_command('help') - bot.add_command(bot._old_help) - - -def setup(bot: Bot) -> None: - """ - The setup for the help extension. - - This is called automatically on `bot.load_extension` being run. - - Stores the original help command instance on the `bot._old_help` attribute for later - reinstatement, before removing it from the command registry so the new help command can be - loaded successfully. - - If an exception is raised during the loading of the cog, `unload` will be called in order to - reinstate the original help command. - """ - bot._old_help = bot.get_command('help') - bot.remove_command('help') - - try: - bot.add_cog(Help()) - except Exception: - unload(bot) - raise - - -def teardown(bot: Bot) -> None: - """ - The teardown for the help extension. - - This is called automatically on `bot.unload_extension` being run. - - Calls `unload` in order to reinstate the original help command. - """ - unload(bot) diff --git a/bot/cogs/moderation/__init__.py b/bot/cogs/moderation/__init__.py deleted file mode 100644 index 6880ca1bd..000000000 --- a/bot/cogs/moderation/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from bot.bot import Bot -from .infractions import Infractions -from .management import ModManagement -from .modlog import ModLog -from .silence import Silence -from .superstarify import Superstarify - - -def setup(bot: Bot) -> None: - """Load the Infractions, ModManagement, ModLog, Silence, and Superstarify cogs.""" - bot.add_cog(Infractions(bot)) - bot.add_cog(ModLog(bot)) - bot.add_cog(ModManagement(bot)) - bot.add_cog(Silence(bot)) - bot.add_cog(Superstarify(bot)) diff --git a/bot/cogs/sync/__init__.py b/bot/cogs/sync/__init__.py deleted file mode 100644 index fe7df4e9b..000000000 --- a/bot/cogs/sync/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from bot.bot import Bot -from .cog import Sync - - -def setup(bot: Bot) -> None: - """Load the Sync cog.""" - bot.add_cog(Sync(bot)) diff --git a/bot/cogs/sync/syncers.py b/bot/cogs/sync/syncers.py deleted file mode 100644 index 003bf3727..000000000 --- a/bot/cogs/sync/syncers.py +++ /dev/null @@ -1,342 +0,0 @@ -import abc -import logging -import typing as t -from collections import namedtuple -from functools import partial - -from discord import Guild, HTTPException, Member, Message, Reaction, User -from discord.ext.commands import Context - -from bot import constants -from bot.api import ResponseCodeError -from bot.bot import Bot - -log = logging.getLogger(__name__) - -# These objects are declared as namedtuples because tuples are hashable, -# something that we make use of when diffing site roles against guild roles. -_Role = namedtuple('Role', ('id', 'name', 'colour', 'permissions', 'position')) -_User = namedtuple('User', ('id', 'name', 'discriminator', 'avatar_hash', 'roles', 'in_guild')) -_Diff = namedtuple('Diff', ('created', 'updated', 'deleted')) - - -class Syncer(abc.ABC): - """Base class for synchronising the database with objects in the Discord cache.""" - - _CORE_DEV_MENTION = f"<@&{constants.Roles.core_developers}> " - _REACTION_EMOJIS = (constants.Emojis.check_mark, constants.Emojis.cross_mark) - - def __init__(self, bot: Bot) -> None: - self.bot = bot - - @property - @abc.abstractmethod - def name(self) -> str: - """The name of the syncer; used in output messages and logging.""" - raise NotImplementedError # pragma: no cover - - async def _send_prompt(self, message: t.Optional[Message] = None) -> t.Optional[Message]: - """ - Send a prompt to confirm or abort a sync using reactions and return the sent message. - - If a message is given, it is edited to display the prompt and reactions. Otherwise, a new - message is sent to the dev-core channel and mentions the core developers role. If the - channel cannot be retrieved, return None. - """ - log.trace(f"Sending {self.name} sync confirmation prompt.") - - msg_content = ( - f'Possible cache issue while syncing {self.name}s. ' - f'More than {constants.Sync.max_diff} {self.name}s were changed. ' - f'React to confirm or abort the sync.' - ) - - # Send to core developers if it's an automatic sync. - if not message: - log.trace("Message not provided for confirmation; creating a new one in dev-core.") - channel = self.bot.get_channel(constants.Channels.dev_core) - - if not channel: - log.debug("Failed to get the dev-core channel from cache; attempting to fetch it.") - try: - channel = await self.bot.fetch_channel(constants.Channels.dev_core) - except HTTPException: - log.exception( - f"Failed to fetch channel for sending sync confirmation prompt; " - f"aborting {self.name} sync." - ) - return None - - message = await channel.send(f"{self._CORE_DEV_MENTION}{msg_content}") - else: - await message.edit(content=msg_content) - - # Add the initial reactions. - log.trace(f"Adding reactions to {self.name} syncer confirmation prompt.") - for emoji in self._REACTION_EMOJIS: - await message.add_reaction(emoji) - - return message - - def _reaction_check( - self, - author: Member, - message: Message, - reaction: Reaction, - user: t.Union[Member, User] - ) -> bool: - """ - Return True if the `reaction` is a valid confirmation or abort reaction on `message`. - - If the `author` of the prompt is a bot, then a reaction by any core developer will be - considered valid. Otherwise, the author of the reaction (`user`) will have to be the - `author` of the prompt. - """ - # For automatic syncs, check for the core dev role instead of an exact author - has_role = any(constants.Roles.core_developers == role.id for role in user.roles) - return ( - reaction.message.id == message.id - and not user.bot - and (has_role if author.bot else user == author) - and str(reaction.emoji) in self._REACTION_EMOJIS - ) - - async def _wait_for_confirmation(self, author: Member, message: Message) -> bool: - """ - Wait for a confirmation reaction by `author` on `message` and return True if confirmed. - - Uses the `_reaction_check` function to determine if a reaction is valid. - - If there is no reaction within `bot.constants.Sync.confirm_timeout` seconds, return False. - To acknowledge the reaction (or lack thereof), `message` will be edited. - """ - # Preserve the core-dev role mention in the message edits so users aren't confused about - # where notifications came from. - mention = self._CORE_DEV_MENTION if author.bot else "" - - reaction = None - try: - log.trace(f"Waiting for a reaction to the {self.name} syncer confirmation prompt.") - reaction, _ = await self.bot.wait_for( - 'reaction_add', - check=partial(self._reaction_check, author, message), - timeout=constants.Sync.confirm_timeout - ) - except TimeoutError: - # reaction will remain none thus sync will be aborted in the finally block below. - log.debug(f"The {self.name} syncer confirmation prompt timed out.") - - if str(reaction) == constants.Emojis.check_mark: - log.trace(f"The {self.name} syncer was confirmed.") - await message.edit(content=f':ok_hand: {mention}{self.name} sync will proceed.') - return True - else: - log.info(f"The {self.name} syncer was aborted or timed out!") - await message.edit( - content=f':warning: {mention}{self.name} sync aborted or timed out!' - ) - return False - - @abc.abstractmethod - async def _get_diff(self, guild: Guild) -> _Diff: - """Return the difference between the cache of `guild` and the database.""" - raise NotImplementedError # pragma: no cover - - @abc.abstractmethod - async def _sync(self, diff: _Diff) -> None: - """Perform the API calls for synchronisation.""" - raise NotImplementedError # pragma: no cover - - async def _get_confirmation_result( - self, - diff_size: int, - author: Member, - message: t.Optional[Message] = None - ) -> t.Tuple[bool, t.Optional[Message]]: - """ - Prompt for confirmation and return a tuple of the result and the prompt message. - - `diff_size` is the size of the diff of the sync. If it is greater than - `bot.constants.Sync.max_diff`, the prompt will be sent. The `author` is the invoked of the - sync and the `message` is an extant message to edit to display the prompt. - - If confirmed or no confirmation was needed, the result is True. The returned message will - either be the given `message` or a new one which was created when sending the prompt. - """ - log.trace(f"Determining if confirmation prompt should be sent for {self.name} syncer.") - if diff_size > constants.Sync.max_diff: - message = await self._send_prompt(message) - if not message: - return False, None # Couldn't get channel. - - confirmed = await self._wait_for_confirmation(author, message) - if not confirmed: - return False, message # Sync aborted. - - return True, message - - async def sync(self, guild: Guild, ctx: t.Optional[Context] = None) -> None: - """ - Synchronise the database with the cache of `guild`. - - If the differences between the cache and the database are greater than - `bot.constants.Sync.max_diff`, then a confirmation prompt will be sent to the dev-core - channel. The confirmation can be optionally redirect to `ctx` instead. - """ - log.info(f"Starting {self.name} syncer.") - - message = None - author = self.bot.user - if ctx: - message = await ctx.send(f"📊 Synchronising {self.name}s.") - author = ctx.author - - diff = await self._get_diff(guild) - diff_dict = diff._asdict() # Ugly method for transforming the NamedTuple into a dict - totals = {k: len(v) for k, v in diff_dict.items() if v is not None} - diff_size = sum(totals.values()) - - confirmed, message = await self._get_confirmation_result(diff_size, author, message) - if not confirmed: - return - - # Preserve the core-dev role mention in the message edits so users aren't confused about - # where notifications came from. - mention = self._CORE_DEV_MENTION if author.bot else "" - - try: - await self._sync(diff) - except ResponseCodeError as e: - log.exception(f"{self.name} syncer failed!") - - # Don't show response text because it's probably some really long HTML. - results = f"status {e.status}\n```{e.response_json or 'See log output for details'}```" - content = f":x: {mention}Synchronisation of {self.name}s failed: {results}" - else: - results = ", ".join(f"{name} `{total}`" for name, total in totals.items()) - log.info(f"{self.name} syncer finished: {results}.") - content = f":ok_hand: {mention}Synchronisation of {self.name}s complete: {results}" - - if message: - await message.edit(content=content) - - -class RoleSyncer(Syncer): - """Synchronise the database with roles in the cache.""" - - name = "role" - - async def _get_diff(self, guild: Guild) -> _Diff: - """Return the difference of roles between the cache of `guild` and the database.""" - log.trace("Getting the diff for roles.") - roles = await self.bot.api_client.get('bot/roles') - - # Pack DB roles and guild roles into one common, hashable format. - # They're hashable so that they're easily comparable with sets later. - db_roles = {_Role(**role_dict) for role_dict in roles} - guild_roles = { - _Role( - id=role.id, - name=role.name, - colour=role.colour.value, - permissions=role.permissions.value, - position=role.position, - ) - for role in guild.roles - } - - guild_role_ids = {role.id for role in guild_roles} - api_role_ids = {role.id for role in db_roles} - new_role_ids = guild_role_ids - api_role_ids - deleted_role_ids = api_role_ids - guild_role_ids - - # New roles are those which are on the cached guild but not on the - # DB guild, going by the role ID. We need to send them in for creation. - roles_to_create = {role for role in guild_roles if role.id in new_role_ids} - roles_to_update = guild_roles - db_roles - roles_to_create - roles_to_delete = {role for role in db_roles if role.id in deleted_role_ids} - - return _Diff(roles_to_create, roles_to_update, roles_to_delete) - - async def _sync(self, diff: _Diff) -> None: - """Synchronise the database with the role cache of `guild`.""" - log.trace("Syncing created roles...") - for role in diff.created: - await self.bot.api_client.post('bot/roles', json=role._asdict()) - - log.trace("Syncing updated roles...") - for role in diff.updated: - await self.bot.api_client.put(f'bot/roles/{role.id}', json=role._asdict()) - - log.trace("Syncing deleted roles...") - for role in diff.deleted: - await self.bot.api_client.delete(f'bot/roles/{role.id}') - - -class UserSyncer(Syncer): - """Synchronise the database with users in the cache.""" - - name = "user" - - async def _get_diff(self, guild: Guild) -> _Diff: - """Return the difference of users between the cache of `guild` and the database.""" - log.trace("Getting the diff for users.") - users = await self.bot.api_client.get('bot/users') - - # Pack DB roles and guild roles into one common, hashable format. - # They're hashable so that they're easily comparable with sets later. - db_users = { - user_dict['id']: _User( - roles=tuple(sorted(user_dict.pop('roles'))), - **user_dict - ) - for user_dict in users - } - guild_users = { - member.id: _User( - id=member.id, - name=member.name, - discriminator=int(member.discriminator), - avatar_hash=member.avatar, - roles=tuple(sorted(role.id for role in member.roles)), - in_guild=True - ) - for member in guild.members - } - - users_to_create = set() - users_to_update = set() - - for db_user in db_users.values(): - guild_user = guild_users.get(db_user.id) - if guild_user is not None: - if db_user != guild_user: - users_to_update.add(guild_user) - - elif db_user.in_guild: - # The user is known in the DB but not the guild, and the - # DB currently specifies that the user is a member of the guild. - # This means that the user has left since the last sync. - # Update the `in_guild` attribute of the user on the site - # to signify that the user left. - new_api_user = db_user._replace(in_guild=False) - users_to_update.add(new_api_user) - - new_user_ids = set(guild_users.keys()) - set(db_users.keys()) - for user_id in new_user_ids: - # The user is known on the guild but not on the API. This means - # that the user has joined since the last sync. Create it. - new_user = guild_users[user_id] - users_to_create.add(new_user) - - return _Diff(users_to_create, users_to_update, None) - - async def _sync(self, diff: _Diff) -> None: - """Synchronise the database with the user cache of `guild`.""" - log.trace("Syncing created users...") - for user in diff.created: - await self.bot.api_client.post('bot/users', json=user._asdict()) - - log.trace("Syncing updated users...") - for user in diff.updated: - await self.bot.api_client.put(f'bot/users/{user.id}', json=user._asdict()) diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py deleted file mode 100644 index 421ad23e2..000000000 --- a/bot/cogs/token_remover.py +++ /dev/null @@ -1,158 +0,0 @@ -import base64 -import binascii -import logging -import re -import struct -import typing as t -from datetime import datetime - -from discord import Colour, Message -from discord.ext.commands import Cog -from discord.utils import snowflake_time - -from bot.bot import Bot -from bot.cogs.moderation import ModLog -from bot.constants import Channels, Colours, Event, Icons - -log = logging.getLogger(__name__) - -DELETION_MESSAGE_TEMPLATE = ( - "Hey {mention}! I noticed you posted a seemingly valid Discord API " - "token in your message and have removed your message. " - "This means that your token has been **compromised**. " - "Please change your token **immediately** at: " - "<https://discordapp.com/developers/applications/me>\n\n" - "Feel free to re-post it with the token removed. " - "If you believe this was a mistake, please let us know!" -) -DISCORD_EPOCH_TIMESTAMP = datetime(2017, 1, 1) -TOKEN_EPOCH = 1_293_840_000 -TOKEN_RE = re.compile( - r"[^\s\.()\"']+" # Matches token part 1: The user ID string, encoded as base64 - r"\." # Matches a literal dot between the token parts - r"[^\s\.()\"']+" # Matches token part 2: The creation timestamp, as an integer - r"\." # Matches a literal dot between the token parts - r"[^\s\.()\"']+" # Matches token part 3: The HMAC, unused by us, but check that it isn't empty -) - - -class TokenRemover(Cog): - """Scans messages for potential discord.py bot tokens and removes them.""" - - def __init__(self, bot: Bot): - self.bot = bot - - @property - def mod_log(self) -> ModLog: - """Get currently loaded ModLog cog instance.""" - return self.bot.get_cog("ModLog") - - @Cog.listener() - async def on_message(self, msg: Message) -> None: - """ - Check each message for a string that matches Discord's token pattern. - - See: https://discordapp.com/developers/docs/reference#snowflakes - """ - found_token = self.find_token_in_message(msg) - if found_token: - await self.take_action(msg, found_token) - - @Cog.listener() - async def on_message_edit(self, before: Message, after: Message) -> None: - """ - Check each edit for a string that matches Discord's token pattern. - - See: https://discordapp.com/developers/docs/reference#snowflakes - """ - found_token = self.find_token_in_message(after) - if found_token: - await self.take_action(after, found_token) - - async def take_action(self, msg: Message, found_token: str) -> None: - """Remove the `msg` containing a token an send a mod_log message.""" - user_id, creation_timestamp, hmac = found_token.split('.') - self.mod_log.ignore(Event.message_delete, msg.id) - await msg.delete() - await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention)) - - message = ( - "Censored a seemingly valid token sent by " - f"{msg.author} (`{msg.author.id}`) in {msg.channel.mention}, token was " - f"`{user_id}.{creation_timestamp}.{'x' * len(hmac)}`" - ) - log.debug(message) - - # Send pretty mod log embed to mod-alerts - await self.mod_log.send_log_message( - icon_url=Icons.token_removed, - colour=Colour(Colours.soft_red), - title="Token removed!", - text=message, - thumbnail=msg.author.avatar_url_as(static_format="png"), - channel_id=Channels.mod_alerts, - ) - - @classmethod - def find_token_in_message(cls, msg: Message) -> t.Optional[str]: - """Return a seemingly valid token found in `msg` or `None` if no token is found.""" - if msg.author.bot: - return - - # Use findall rather than search to guard against method calls prematurely returning the - # token check (e.g. `message.channel.send` also matches our token pattern) - maybe_matches = TOKEN_RE.findall(msg.content) - for substr in maybe_matches: - if cls.is_maybe_token(substr): - # Short-circuit on first match - return substr - - # No matching substring - return - - @classmethod - def is_maybe_token(cls, test_str: str) -> bool: - """Check the provided string to see if it is a seemingly valid token.""" - try: - user_id, creation_timestamp, hmac = test_str.split('.') - except ValueError: - return False - - if cls.is_valid_user_id(user_id) and cls.is_valid_timestamp(creation_timestamp): - return True - - @staticmethod - def is_valid_user_id(b64_content: str) -> bool: - """ - Check potential token to see if it contains a valid Discord user ID. - - See: https://discordapp.com/developers/docs/reference#snowflakes - """ - b64_content += '=' * (-len(b64_content) % 4) - - try: - content: bytes = base64.b64decode(b64_content) - return content.decode('utf-8').isnumeric() - except (binascii.Error, UnicodeDecodeError): - return False - - @staticmethod - def is_valid_timestamp(b64_content: str) -> bool: - """ - Check potential token to see if it contains a valid timestamp. - - See: https://discordapp.com/developers/docs/reference#snowflakes - """ - b64_content += '=' * (-len(b64_content) % 4) - - try: - content = base64.urlsafe_b64decode(b64_content) - snowflake = struct.unpack('i', content)[0] - except (binascii.Error, struct.error): - return False - return snowflake_time(snowflake + TOKEN_EPOCH) < DISCORD_EPOCH_TIMESTAMP - - -def setup(bot: Bot) -> None: - """Load the TokenRemover cog.""" - bot.add_cog(TokenRemover(bot)) diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py deleted file mode 100644 index b0a493e68..000000000 --- a/bot/cogs/verification.py +++ /dev/null @@ -1,233 +0,0 @@ -import logging -from contextlib import suppress -from datetime import datetime - -from discord import Colour, Forbidden, Message, NotFound, Object -from discord.ext import tasks -from discord.ext.commands import Cog, Context, command - -from bot import constants -from bot.bot import Bot -from bot.cogs.moderation import ModLog -from bot.decorators import InChannelCheckFailure, in_channel, without_role -from bot.utils.checks import without_role_check - -log = logging.getLogger(__name__) - -WELCOME_MESSAGE = f""" -Hello! Welcome to the server, and thanks for verifying yourself! - -For your records, these are the documents you accepted: - -`1)` Our rules, here: <https://pythondiscord.com/pages/rules> -`2)` Our privacy policy, here: <https://pythondiscord.com/pages/privacy> - you can find information on how to have \ -your information removed here as well. - -Feel free to review them at any point! - -Additionally, if you'd like to receive notifications for the announcements \ -we post in <#{constants.Channels.announcements}> -from time to time, you can send `!subscribe` to <#{constants.Channels.bot_commands}> at any time \ -to assign yourself the **Announcements** role. We'll mention this role every time we make an announcement. - -If you'd like to unsubscribe from the announcement notifications, simply send `!unsubscribe` to \ -<#{constants.Channels.bot_commands}>. -""" - -if constants.DEBUG_MODE: - PERIODIC_PING = "Periodic checkpoint message successfully sent." -else: - PERIODIC_PING = ( - f"@everyone To verify that you have read our rules, please type `{constants.Bot.prefix}accept`." - " If you encounter any problems during the verification process, " - f"ping the <@&{constants.Roles.admins}> role in this channel." - ) -BOT_MESSAGE_DELETE_DELAY = 10 - - -class Verification(Cog): - """User verification and role self-management.""" - - def __init__(self, bot: Bot): - self.bot = bot - self.periodic_ping.start() - - @property - def mod_log(self) -> ModLog: - """Get currently loaded ModLog cog instance.""" - return self.bot.get_cog("ModLog") - - @Cog.listener() - async def on_message(self, message: Message) -> None: - """Check new message event for messages to the checkpoint channel & process.""" - if message.channel.id != constants.Channels.verification: - return # Only listen for #checkpoint messages - - if message.author.bot: - # They're a bot, delete their message after the delay. - # But not the periodic ping; we like that one. - if message.content != PERIODIC_PING: - await message.delete(delay=BOT_MESSAGE_DELETE_DELAY) - return - - # if a user mentions a role or guild member - # alert the mods in mod-alerts channel - if message.mentions or message.role_mentions: - log.debug( - f"{message.author} mentioned one or more users " - f"and/or roles in {message.channel.name}" - ) - - embed_text = ( - f"{message.author.mention} sent a message in " - f"{message.channel.mention} that contained user and/or role mentions." - f"\n\n**Original message:**\n>>> {message.content}" - ) - - # Send pretty mod log embed to mod-alerts - await self.mod_log.send_log_message( - icon_url=constants.Icons.filtering, - colour=Colour(constants.Colours.soft_red), - title=f"User/Role mentioned in {message.channel.name}", - text=embed_text, - thumbnail=message.author.avatar_url_as(static_format="png"), - channel_id=constants.Channels.mod_alerts, - ping_everyone=constants.Filter.ping_everyone, - ) - - ctx: Context = await self.bot.get_context(message) - if ctx.command is not None and ctx.command.name == "accept": - return - - if any(r.id == constants.Roles.verified for r in ctx.author.roles): - log.info( - f"{ctx.author} posted '{ctx.message.content}' " - "in the verification channel, but is already verified." - ) - return - - log.debug( - f"{ctx.author} posted '{ctx.message.content}' in the verification " - "channel. We are providing instructions how to verify." - ) - await ctx.send( - f"{ctx.author.mention} Please type `!accept` to verify that you accept our rules, " - f"and gain access to the rest of the server.", - delete_after=20 - ) - - log.trace(f"Deleting the message posted by {ctx.author}") - with suppress(NotFound): - await ctx.message.delete() - - @command(name='accept', aliases=('verify', 'verified', 'accepted'), hidden=True) - @without_role(constants.Roles.verified) - @in_channel(constants.Channels.verification) - async def accept_command(self, ctx: Context, *_) -> None: # We don't actually care about the args - """Accept our rules and gain access to the rest of the server.""" - log.debug(f"{ctx.author} called !accept. Assigning the 'Developer' role.") - await ctx.author.add_roles(Object(constants.Roles.verified), reason="Accepted the rules") - try: - await ctx.author.send(WELCOME_MESSAGE) - except Forbidden: - log.info(f"Sending welcome message failed for {ctx.author}.") - finally: - log.trace(f"Deleting accept message by {ctx.author}.") - with suppress(NotFound): - self.mod_log.ignore(constants.Event.message_delete, ctx.message.id) - await ctx.message.delete() - - @command(name='subscribe') - @in_channel(constants.Channels.bot_commands) - async def subscribe_command(self, ctx: Context, *_) -> None: # We don't actually care about the args - """Subscribe to announcement notifications by assigning yourself the role.""" - has_role = False - - for role in ctx.author.roles: - if role.id == constants.Roles.announcements: - has_role = True - break - - if has_role: - await ctx.send(f"{ctx.author.mention} You're already subscribed!") - return - - log.debug(f"{ctx.author} called !subscribe. Assigning the 'Announcements' role.") - await ctx.author.add_roles(Object(constants.Roles.announcements), reason="Subscribed to announcements") - - log.trace(f"Deleting the message posted by {ctx.author}.") - - await ctx.send( - f"{ctx.author.mention} Subscribed to <#{constants.Channels.announcements}> notifications.", - ) - - @command(name='unsubscribe') - @in_channel(constants.Channels.bot_commands) - async def unsubscribe_command(self, ctx: Context, *_) -> None: # We don't actually care about the args - """Unsubscribe from announcement notifications by removing the role from yourself.""" - has_role = False - - for role in ctx.author.roles: - if role.id == constants.Roles.announcements: - has_role = True - break - - if not has_role: - await ctx.send(f"{ctx.author.mention} You're already unsubscribed!") - return - - log.debug(f"{ctx.author} called !unsubscribe. Removing the 'Announcements' role.") - await ctx.author.remove_roles(Object(constants.Roles.announcements), reason="Unsubscribed from announcements") - - log.trace(f"Deleting the message posted by {ctx.author}.") - - await ctx.send( - f"{ctx.author.mention} Unsubscribed from <#{constants.Channels.announcements}> notifications." - ) - - # This cannot be static (must have a __func__ attribute). - async def cog_command_error(self, ctx: Context, error: Exception) -> None: - """Check for & ignore any InChannelCheckFailure.""" - if isinstance(error, InChannelCheckFailure): - error.handled = True - - @staticmethod - def bot_check(ctx: Context) -> bool: - """Block any command within the verification channel that is not !accept.""" - if ctx.channel.id == constants.Channels.verification and without_role_check(ctx, *constants.MODERATION_ROLES): - return ctx.command.name == "accept" - else: - return True - - @tasks.loop(hours=12) - async def periodic_ping(self) -> None: - """Every week, mention @everyone to remind them to verify.""" - messages = self.bot.get_channel(constants.Channels.verification).history(limit=10) - need_to_post = True # True if a new message needs to be sent. - - async for message in messages: - if message.author == self.bot.user and message.content == PERIODIC_PING: - delta = datetime.utcnow() - message.created_at # Time since last message. - if delta.days >= 7: # Message is older than a week. - await message.delete() - else: - need_to_post = False - - break - - if need_to_post: - await self.bot.get_channel(constants.Channels.verification).send(PERIODIC_PING) - - @periodic_ping.before_loop - async def before_ping(self) -> None: - """Only start the loop when the bot is ready.""" - await self.bot.wait_until_guild_available() - - def cog_unload(self) -> None: - """Cancel the periodic ping task when the cog is unloaded.""" - self.periodic_ping.cancel() - - -def setup(bot: Bot) -> None: - """Load the Verification cog.""" - bot.add_cog(Verification(bot)) diff --git a/bot/cogs/watchchannels/__init__.py b/bot/cogs/watchchannels/__init__.py deleted file mode 100644 index 69d118df6..000000000 --- a/bot/cogs/watchchannels/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from bot.bot import Bot -from .bigbrother import BigBrother -from .talentpool import TalentPool - - -def setup(bot: Bot) -> None: - """Load the BigBrother and TalentPool cogs.""" - bot.add_cog(BigBrother(bot)) - bot.add_cog(TalentPool(bot)) diff --git a/bot/cogs/wolfram.py b/bot/cogs/wolfram.py deleted file mode 100644 index 5d6b4630b..000000000 --- a/bot/cogs/wolfram.py +++ /dev/null @@ -1,272 +0,0 @@ -import logging -from io import BytesIO -from typing import Callable, List, Optional, Tuple -from urllib import parse - -import discord -from dateutil.relativedelta import relativedelta -from discord import Embed -from discord.ext import commands -from discord.ext.commands import BucketType, Cog, Context, check, group - -from bot.bot import Bot -from bot.constants import Colours, STAFF_ROLES, Wolfram -from bot.pagination import ImagePaginator -from bot.utils.time import humanize_delta - -log = logging.getLogger(__name__) - -APPID = Wolfram.key -DEFAULT_OUTPUT_FORMAT = "JSON" -QUERY = "http://api.wolframalpha.com/v2/{request}?{data}" -WOLF_IMAGE = "https://www.symbols.com/gi.php?type=1&id=2886&i=1" - -MAX_PODS = 20 - -# Allows for 10 wolfram calls pr user pr day -usercd = commands.CooldownMapping.from_cooldown(Wolfram.user_limit_day, 60*60*24, BucketType.user) - -# Allows for max api requests / days in month per day for the entire guild (Temporary) -guildcd = commands.CooldownMapping.from_cooldown(Wolfram.guild_limit_day, 60*60*24, BucketType.guild) - - -async def send_embed( - ctx: Context, - message_txt: str, - colour: int = Colours.soft_red, - footer: str = None, - img_url: str = None, - f: discord.File = None -) -> None: - """Generate & send a response embed with Wolfram as the author.""" - embed = Embed(colour=colour) - embed.description = message_txt - embed.set_author(name="Wolfram Alpha", - icon_url=WOLF_IMAGE, - url="https://www.wolframalpha.com/") - if footer: - embed.set_footer(text=footer) - - if img_url: - embed.set_image(url=img_url) - - await ctx.send(embed=embed, file=f) - - -def custom_cooldown(*ignore: List[int]) -> Callable: - """ - Implement per-user and per-guild cooldowns for requests to the Wolfram API. - - A list of roles may be provided to ignore the per-user cooldown - """ - async def predicate(ctx: Context) -> bool: - user_bucket = usercd.get_bucket(ctx.message) - - if all(role.id not in ignore for role in ctx.author.roles): - user_rate = user_bucket.update_rate_limit() - - if user_rate: - # Can't use api; cause: member limit - delta = relativedelta(seconds=int(user_rate)) - cooldown = humanize_delta(delta) - message = ( - "You've used up your limit for Wolfram|Alpha requests.\n" - f"Cooldown: {cooldown}" - ) - await send_embed(ctx, message) - return False - - guild_bucket = guildcd.get_bucket(ctx.message) - guild_rate = guild_bucket.update_rate_limit() - - # Repr has a token attribute to read requests left - log.debug(guild_bucket) - - if guild_rate: - # Can't use api; cause: guild limit - message = ( - "The max limit of requests for the server has been reached for today.\n" - f"Cooldown: {int(guild_rate)}" - ) - await send_embed(ctx, message) - return False - - return True - return check(predicate) - - -async def get_pod_pages(ctx: Context, bot: Bot, query: str) -> Optional[List[Tuple]]: - """Get the Wolfram API pod pages for the provided query.""" - async with ctx.channel.typing(): - url_str = parse.urlencode({ - "input": query, - "appid": APPID, - "output": DEFAULT_OUTPUT_FORMAT, - "format": "image,plaintext" - }) - request_url = QUERY.format(request="query", data=url_str) - - async with bot.http_session.get(request_url) as response: - json = await response.json(content_type='text/plain') - - result = json["queryresult"] - - if result["error"]: - # API key not set up correctly - if result["error"]["msg"] == "Invalid appid": - message = "Wolfram API key is invalid or missing." - log.warning( - "API key seems to be missing, or invalid when " - f"processing a wolfram request: {url_str}, Response: {json}" - ) - await send_embed(ctx, message) - return - - message = "Something went wrong internally with your request, please notify staff!" - log.warning(f"Something went wrong getting a response from wolfram: {url_str}, Response: {json}") - await send_embed(ctx, message) - return - - if not result["success"]: - message = f"I couldn't find anything for {query}." - await send_embed(ctx, message) - return - - if not result["numpods"]: - message = "Could not find any results." - await send_embed(ctx, message) - return - - pods = result["pods"] - pages = [] - for pod in pods[:MAX_PODS]: - subs = pod.get("subpods") - - for sub in subs: - title = sub.get("title") or sub.get("plaintext") or sub.get("id", "") - img = sub["img"]["src"] - pages.append((title, img)) - return pages - - -class Wolfram(Cog): - """Commands for interacting with the Wolfram|Alpha API.""" - - def __init__(self, bot: Bot): - self.bot = bot - - @group(name="wolfram", aliases=("wolf", "wa"), invoke_without_command=True) - @custom_cooldown(*STAFF_ROLES) - async def wolfram_command(self, ctx: Context, *, query: str) -> None: - """Requests all answers on a single image, sends an image of all related pods.""" - url_str = parse.urlencode({ - "i": query, - "appid": APPID, - }) - query = QUERY.format(request="simple", data=url_str) - - # Give feedback that the bot is working. - async with ctx.channel.typing(): - async with self.bot.http_session.get(query) as response: - status = response.status - image_bytes = await response.read() - - f = discord.File(BytesIO(image_bytes), filename="image.png") - image_url = "attachment://image.png" - - if status == 501: - message = "Failed to get response" - footer = "" - color = Colours.soft_red - elif status == 400: - message = "No input found" - footer = "" - color = Colours.soft_red - elif status == 403: - message = "Wolfram API key is invalid or missing." - footer = "" - color = Colours.soft_red - else: - message = "" - footer = "View original for a bigger picture." - color = Colours.soft_orange - - # Sends a "blank" embed if no request is received, unsure how to fix - await send_embed(ctx, message, color, footer=footer, img_url=image_url, f=f) - - @wolfram_command.command(name="page", aliases=("pa", "p")) - @custom_cooldown(*STAFF_ROLES) - async def wolfram_page_command(self, ctx: Context, *, query: str) -> None: - """ - Requests a drawn image of given query. - - Keywords worth noting are, "like curve", "curve", "graph", "pokemon", etc. - """ - pages = await get_pod_pages(ctx, self.bot, query) - - if not pages: - return - - embed = Embed() - embed.set_author(name="Wolfram Alpha", - icon_url=WOLF_IMAGE, - url="https://www.wolframalpha.com/") - embed.colour = Colours.soft_orange - - await ImagePaginator.paginate(pages, ctx, embed) - - @wolfram_command.command(name="cut", aliases=("c",)) - @custom_cooldown(*STAFF_ROLES) - async def wolfram_cut_command(self, ctx: Context, *, query: str) -> None: - """ - Requests a drawn image of given query. - - Keywords worth noting are, "like curve", "curve", "graph", "pokemon", etc. - """ - pages = await get_pod_pages(ctx, self.bot, query) - - if not pages: - return - - if len(pages) >= 2: - page = pages[1] - else: - page = pages[0] - - await send_embed(ctx, page[0], colour=Colours.soft_orange, img_url=page[1]) - - @wolfram_command.command(name="short", aliases=("sh", "s")) - @custom_cooldown(*STAFF_ROLES) - async def wolfram_short_command(self, ctx: Context, *, query: str) -> None: - """Requests an answer to a simple question.""" - url_str = parse.urlencode({ - "i": query, - "appid": APPID, - }) - query = QUERY.format(request="result", data=url_str) - - # Give feedback that the bot is working. - async with ctx.channel.typing(): - async with self.bot.http_session.get(query) as response: - status = response.status - response_text = await response.text() - - if status == 501: - message = "Failed to get response" - color = Colours.soft_red - elif status == 400: - message = "No input found" - color = Colours.soft_red - elif response_text == "Error 1: Invalid appid": - message = "Wolfram API key is invalid or missing." - color = Colours.soft_red - else: - message = response_text - color = Colours.soft_orange - - await send_embed(ctx, message, color) - - -def setup(bot: Bot) -> None: - """Load the Wolfram cog.""" - bot.add_cog(Wolfram(bot)) diff --git a/bot/command.py b/bot/command.py new file mode 100644 index 000000000..0fb900f7b --- /dev/null +++ b/bot/command.py @@ -0,0 +1,18 @@ +from discord.ext import commands + + +class Command(commands.Command): + """ + A `discord.ext.commands.Command` subclass which supports root aliases. + + A `root_aliases` keyword argument is added, which is a sequence of alias names that will act as + top-level commands rather than being aliases of the command's group. It's stored as an attribute + also named `root_aliases`. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.root_aliases = kwargs.get("root_aliases", []) + + if not isinstance(self.root_aliases, (list, tuple)): + raise TypeError("Root aliases of a command must be a list or a tuple of strings.") diff --git a/bot/constants.py b/bot/constants.py index 549e69c8f..c21fd52e0 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -15,7 +15,7 @@ import os from collections.abc import Mapping from enum import Enum from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional import yaml @@ -198,7 +198,18 @@ class Bot(metaclass=YAMLGetter): prefix: str token: str - sentry_dsn: str + sentry_dsn: Optional[str] + + +class Redis(metaclass=YAMLGetter): + section = "bot" + subsection = "redis" + + host: str + port: int + password: Optional[str] + use_fakeredis: bool # If this is True, Bot will use fakeredis.aioredis + class Filter(metaclass=YAMLGetter): section = "filter" @@ -206,6 +217,7 @@ class Filter(metaclass=YAMLGetter): filter_zalgo: bool filter_invites: bool filter_domains: bool + filter_everyone_ping: bool watch_regex: bool watch_rich_embeds: bool @@ -213,12 +225,10 @@ class Filter(metaclass=YAMLGetter): notify_user_zalgo: bool notify_user_invites: bool notify_user_domains: bool + notify_user_everyone_ping: bool ping_everyone: bool - guild_invite_whitelist: List[int] - domain_blacklist: List[str] - word_watchlist: List[str] - token_watchlist: List[str] + offensive_msg_delete_days: int channel_whitelist: List[int] role_whitelist: List[int] @@ -244,7 +254,7 @@ class DuckPond(metaclass=YAMLGetter): section = "duck_pond" threshold: int - custom_emojis: List[int] + channel_blacklist: List[int] class Emojis(metaclass=YAMLGetter): @@ -260,6 +270,21 @@ class Emojis(metaclass=YAMLGetter): status_idle: str status_dnd: str + badge_staff: str + badge_partner: str + badge_hypesquad: str + badge_bug_hunter: str + badge_hypesquad_bravery: str + badge_hypesquad_brilliance: str + badge_hypesquad_balance: str + badge_early_supporter: str + badge_bug_hunter_level_2: str + badge_verified_bot_developer: str + + incident_actioned: str + incident_unactioned: str + incident_investigating: str + failmail: str trashcan: str @@ -269,20 +294,6 @@ class Emojis(metaclass=YAMLGetter): cross_mark: str check_mark: str - ducky_yellow: int - ducky_blurple: int - ducky_regal: int - ducky_camo: int - ducky_ninja: int - ducky_devil: int - ducky_tube: int - ducky_hunt: int - ducky_wizard: int - ducky_party: int - ducky_angel: int - ducky_maul: int - ducky_santa: int - upvotes: str comments: str user: str @@ -350,41 +361,53 @@ class CleanMessages(metaclass=YAMLGetter): message_limit: int +class Stats(metaclass=YAMLGetter): + section = "bot" + subsection = "stats" + + presence_update_timeout: int + statsd_host: str + class Categories(metaclass=YAMLGetter): section = "guild" subsection = "categories" - python_help: int + help_available: int + help_in_use: int + help_dormant: int + modmail: int class Channels(metaclass=YAMLGetter): section = "guild" subsection = "channels" + admin_announcements: int admin_spam: int admins: int announcements: int attachment_log: int big_brother_logs: int bot_commands: int + change_log: int + code_help_voice: int + cooldown: int defcon: int dev_contrib: int dev_core: int dev_log: int + dm_log: int esoteric: int - help_0: int - help_1: int - help_2: int - help_3: int - help_4: int - help_5: int - help_6: int - help_7: int helpers: int + how_to_get_help: int + incidents: int + incidents_archive: int + mailing_lists: int message_log: int meta: int mod_alerts: int + mod_announcements: int mod_log: int mod_spam: int mods: int @@ -393,7 +416,10 @@ class Channels(metaclass=YAMLGetter): off_topic_2: int organisation: int python_discussion: int + python_events: int + python_news: int reddit: int + staff_announcements: int talent_pool: int user_event_announcements: int user_log: int @@ -405,11 +431,13 @@ class Webhooks(metaclass=YAMLGetter): section = "guild" subsection = "webhooks" - talent_pool: int big_brother: int - reddit: int - duck_pond: int dev_log: int + dm_log: int + duck_pond: int + incidents_archive: int + reddit: int + talent_pool: int class Roles(metaclass=YAMLGetter): @@ -420,6 +448,7 @@ class Roles(metaclass=YAMLGetter): announcements: int contributors: int core_developers: int + help_cooldown: int helpers: int jammers: int moderators: int @@ -428,6 +457,7 @@ class Roles(metaclass=YAMLGetter): partners: int python_community: int team_leaders: int + unverified: int verified: int # This is the Developers role on PyDis, here named verified for readability reasons. @@ -435,17 +465,18 @@ class Guild(metaclass=YAMLGetter): section = "guild" id: int + invite: str # Discord invite, gets embedded in chat moderation_channels: List[int] moderation_roles: List[int] modlog_blacklist: List[int] reminder_whitelist: List[int] - staff_channels: List[int] staff_roles: List[int] + class Keys(metaclass=YAMLGetter): section = "keys" - site_api: str + site_api: Optional[str] class URLs(metaclass=YAMLGetter): @@ -462,25 +493,13 @@ class URLs(metaclass=YAMLGetter): bot_avatar: str github_bot_repo: str - # Site endpoints + # Base site vars site: str site_api: str - site_superstarify_api: str - site_logs_api: str - site_logs_view: str - site_reminders_api: str - site_reminders_user_api: str site_schema: str - site_settings_api: str - site_tags_api: str - site_user_api: str - site_user_complete_api: str - site_infractions: str - site_infractions_user: str - site_infractions_type: str - site_infractions_by_id: str - site_infractions_user_type_current: str - site_infractions_user_type: str + + # Site endpoints + site_logs_view: str paste_service: str @@ -488,16 +507,8 @@ class Reddit(metaclass=YAMLGetter): section = "reddit" subreddits: list - client_id: str - secret: str - - -class Wolfram(metaclass=YAMLGetter): - section = "wolfram" - - user_limit_day: int - guild_limit_day: int - key: str + client_id: Optional[str] + secret: Optional[str] class AntiSpam(metaclass=YAMLGetter): @@ -510,12 +521,6 @@ class AntiSpam(metaclass=YAMLGetter): rules: Dict[str, Dict[str, int]] -class AntiMalware(metaclass=YAMLGetter): - section = "anti_malware" - - whitelist: list - - class BigBrother(metaclass=YAMLGetter): section = 'big_brother' @@ -531,11 +536,21 @@ class Free(metaclass=YAMLGetter): cooldown_per: float -class Mention(metaclass=YAMLGetter): - section = 'mention' +class HelpChannels(metaclass=YAMLGetter): + section = 'help_channels' - message_timeout: int - reset_delay: int + enable: bool + claim_minutes: int + cmd_whitelist: List[int] + idle_minutes: int + deleted_idle_minutes: int + max_available: int + max_total_channels: int + name_prefix: str + notify: bool + notify_channel: int + notify_minutes: int + notify_roles: List[int] class RedirectOutput(metaclass=YAMLGetter): @@ -545,11 +560,22 @@ class RedirectOutput(metaclass=YAMLGetter): delete_delay: int -class Sync(metaclass=YAMLGetter): - section = 'sync' +class PythonNews(metaclass=YAMLGetter): + section = 'python_news' + + mail_lists: List[str] + channel: int + webhook: int - confirm_timeout: int - max_diff: int + +class Verification(metaclass=YAMLGetter): + section = "verification" + + unverified_after: int + kicked_after: int + reminder_frequency: int + bot_message_delete_delay: int + kick_confirmation_threshold: float class Event(Enum): @@ -589,13 +615,9 @@ PROJECT_ROOT = os.path.abspath(os.path.join(BOT_DIR, os.pardir)) MODERATION_ROLES = Guild.moderation_roles STAFF_ROLES = Guild.staff_roles -# Roles combinations -STAFF_CHANNELS = Guild.staff_channels - -# Default Channel combinations +# Channel combinations MODERATION_CHANNELS = Guild.moderation_channels - # Bot replies NEGATIVE_REPLIES = [ "Noooooo!!", diff --git a/bot/converters.py b/bot/converters.py index 72c46fdf0..2e118d476 100644 --- a/bot/converters.py +++ b/bot/converters.py @@ -2,6 +2,7 @@ import logging import re import typing as t from datetime import datetime +from functools import partial from ssl import CertificateError import dateutil.parser @@ -9,11 +10,18 @@ import dateutil.tz import discord from aiohttp import ClientConnectorError from dateutil.relativedelta import relativedelta -from discord.ext.commands import BadArgument, Context, Converter, UserConverter +from discord.ext.commands import BadArgument, Bot, Context, Converter, IDConverter, UserConverter +from discord.utils import DISCORD_EPOCH, snowflake_time +from bot.api import ResponseCodeError +from bot.constants import URLs +from bot.utils.regex import INVITE_RE log = logging.getLogger(__name__) +DISCORD_EPOCH_DT = datetime.utcfromtimestamp(DISCORD_EPOCH / 1000) +RE_USER_MENTION = re.compile(r"<@!?([0-9]+)>$") + def allowed_strings(*values, preserve_case: bool = False) -> t.Callable[[str], str]: """ @@ -34,6 +42,90 @@ def allowed_strings(*values, preserve_case: bool = False) -> t.Callable[[str], s return converter +class ValidDiscordServerInvite(Converter): + """ + A converter that validates whether a given string is a valid Discord server invite. + + Raises 'BadArgument' if: + - The string is not a valid Discord server invite. + - The string is valid, but is an invite for a group DM. + - The string is valid, but is expired. + + Returns a (partial) guild object if: + - The string is a valid vanity + - The string is a full invite URI + - The string contains the invite code (the stuff after discord.gg/) + + See the Discord API docs for documentation on the guild object: + https://discord.com/developers/docs/resources/guild#guild-object + """ + + async def convert(self, ctx: Context, server_invite: str) -> dict: + """Check whether the string is a valid Discord server invite.""" + invite_code = INVITE_RE.search(server_invite) + if invite_code: + response = await ctx.bot.http_session.get( + f"{URLs.discord_invite_api}/{invite_code[1]}" + ) + if response.status != 404: + invite_data = await response.json() + return invite_data.get("guild") + + id_converter = IDConverter() + if id_converter._get_id_match(server_invite): + raise BadArgument("Guild IDs are not supported, only invites.") + + raise BadArgument("This does not appear to be a valid Discord server invite.") + + +class ValidFilterListType(Converter): + """ + A converter that checks whether the given string is a valid FilterList type. + + Raises `BadArgument` if the argument is not a valid FilterList type, and simply + passes through the given argument otherwise. + """ + + @staticmethod + async def get_valid_types(bot: Bot) -> list: + """ + Try to get a list of valid filter list types. + + Raise a BadArgument if the API can't respond. + """ + try: + valid_types = await bot.api_client.get('bot/filter-lists/get-types') + except ResponseCodeError: + raise BadArgument("Cannot validate list_type: Unable to fetch valid types from API.") + + return [enum for enum, classname in valid_types] + + async def convert(self, ctx: Context, list_type: str) -> str: + """Checks whether the given string is a valid FilterList type.""" + valid_types = await self.get_valid_types(ctx.bot) + list_type = list_type.upper() + + if list_type not in valid_types: + + # Maybe the user is using the plural form of this type, + # e.g. "guild_invites" instead of "guild_invite". + # + # This code will support the simple plural form (a single 's' at the end), + # which works for all current list types, but if a list type is added in the future + # which has an irregular plural form (like 'ies'), this code will need to be + # refactored to support this. + if list_type.endswith("S") and list_type[:-1] in valid_types: + list_type = list_type[:-1] + + else: + valid_types_list = '\n'.join([f"• {type_.lower()}" for type_ in valid_types]) + raise BadArgument( + f"You have provided an invalid list type!\n\n" + f"Please provide one of the following: \n{valid_types_list}" + ) + return list_type + + class ValidPythonIdentifier(Converter): """ A converter that checks whether the given string is a valid Python identifier. @@ -85,17 +177,42 @@ class ValidURL(Converter): return url -class InfractionSearchQuery(Converter): - """A converter that checks if the argument is a Discord user, and if not, falls back to a string.""" +class Snowflake(IDConverter): + """ + Converts to an int if the argument is a valid Discord snowflake. + + A snowflake is valid if: + + * It consists of 15-21 digits (0-9) + * Its parsed datetime is after the Discord epoch + * Its parsed datetime is less than 1 day after the current time + """ + + async def convert(self, ctx: Context, arg: str) -> int: + """ + Ensure `arg` matches the ID pattern and its timestamp is in range. + + Return `arg` as an int if it's a valid snowflake. + """ + error = f"Invalid snowflake {arg!r}" + + if not self._get_id_match(arg): + raise BadArgument(error) + + snowflake = int(arg) - @staticmethod - async def convert(ctx: Context, arg: str) -> t.Union[discord.Member, str]: - """Check if the argument is a Discord user, and if not, falls back to a string.""" try: - maybe_snowflake = arg.strip("<@!>") - return await ctx.bot.fetch_user(maybe_snowflake) - except (discord.NotFound, discord.HTTPException): - return arg + time = snowflake_time(snowflake) + except (OverflowError, OSError) as e: + # Not sure if this can ever even happen, but let's be safe. + raise BadArgument(f"{error}: {e}") + + if time < DISCORD_EPOCH_DT: + raise BadArgument(f"{error}: timestamp is before the Discord epoch.") + elif (datetime.utcnow() - time).days < -1: + raise BadArgument(f"{error}: timestamp is too far into the future.") + + return snowflake class Subreddit(Converter): @@ -181,8 +298,8 @@ class TagContentConverter(Converter): return tag_content -class Duration(Converter): - """Convert duration strings into UTC datetime.datetime objects.""" +class DurationDelta(Converter): + """Convert duration strings into dateutil.relativedelta.relativedelta objects.""" duration_parser = re.compile( r"((?P<years>\d+?) ?(years|year|Y|y) ?)?" @@ -194,9 +311,9 @@ class Duration(Converter): r"((?P<seconds>\d+?) ?(seconds|second|S|s))?" ) - async def convert(self, ctx: Context, duration: str) -> datetime: + async def convert(self, ctx: Context, duration: str) -> relativedelta: """ - Converts a `duration` string to a datetime object that's `duration` in the future. + Converts a `duration` string to a relativedelta object. The converter supports the following symbols for each unit of time: - years: `Y`, `y`, `year`, `years` @@ -215,9 +332,52 @@ class Duration(Converter): duration_dict = {unit: int(amount) for unit, amount in match.groupdict(default=0).items()} delta = relativedelta(**duration_dict) + + return delta + + +class Duration(DurationDelta): + """Convert duration strings into UTC datetime.datetime objects.""" + + async def convert(self, ctx: Context, duration: str) -> datetime: + """ + Converts a `duration` string to a datetime object that's `duration` in the future. + + The converter supports the same symbols for each unit of time as its parent class. + """ + delta = await super().convert(ctx, duration) now = datetime.utcnow() - return now + delta + try: + return now + delta + except ValueError: + raise BadArgument(f"`{duration}` results in a datetime outside the supported range.") + + +class OffTopicName(Converter): + """A converter that ensures an added off-topic name is valid.""" + + async def convert(self, ctx: Context, argument: str) -> str: + """Attempt to replace any invalid characters with their approximate Unicode equivalent.""" + allowed_characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ!?'`-" + + # Chain multiple words to a single one + argument = "-".join(argument.split()) + + if not (2 <= len(argument) <= 96): + raise BadArgument("Channel name must be between 2 and 96 chars long") + + elif not all(c.isalnum() or c in allowed_characters for c in argument): + raise BadArgument( + "Channel name must only consist of " + "alphanumeric characters, minus signs or apostrophes." + ) + + # Replace invalid characters with unicode alternatives. + table = str.maketrans( + allowed_characters, '𝖠𝖡𝖢𝖣𝖤𝖥𝖦𝖧𝖨𝖩𝖪𝖫𝖬𝖭𝖮𝖯𝖰𝖱𝖲𝖳𝖴𝖵𝖶𝖷𝖸𝖹ǃ？’’-' + ) + return argument.translate(table) class ISODateTime(Converter): @@ -313,6 +473,24 @@ def proxy_user(user_id: str) -> discord.Object: return user +class UserMentionOrID(UserConverter): + """ + Converts to a `discord.User`, but only if a mention or userID is provided. + + Unlike the default `UserConverter`, it doesn't allow conversion from a name or name#descrim. + This is useful in cases where that lookup strategy would lead to ambiguity. + """ + + async def convert(self, ctx: Context, argument: str) -> discord.User: + """Convert the `arg` to a `discord.User`.""" + match = self._get_id_match(argument) or RE_USER_MENTION.match(argument) + + if match is not None: + return await super().convert(ctx, argument) + else: + raise BadArgument(f"`{argument}` is not a User mention or a User ID.") + + class FetchedUser(UserConverter): """ Converts to a `discord.User` or, if it fails, a `discord.Object`. @@ -358,5 +536,19 @@ class FetchedUser(UserConverter): raise BadArgument(f"User `{arg}` does not exist") +def _snowflake_from_regex(pattern: t.Pattern, arg: str) -> int: + """ + Extract the snowflake from `arg` using a regex `pattern` and return it as an int. + + The snowflake is expected to be within the first capture group in `pattern`. + """ + match = pattern.match(arg) + if not match: + raise BadArgument(f"Mention {str!r} is invalid.") + + return int(match.group(1)) + + Expiry = t.Union[Duration, ISODateTime] FetchedMember = t.Union[discord.Member, FetchedUser] +UserMention = partial(_snowflake_from_regex, RE_USER_MENTION) diff --git a/bot/decorators.py b/bot/decorators.py index 2d18eaa6a..2518124da 100644 --- a/bot/decorators.py +++ b/bot/decorators.py @@ -1,80 +1,65 @@ import logging import random -from asyncio import Lock, sleep +from asyncio import Lock, create_task, sleep from contextlib import suppress from functools import wraps -from typing import Callable, Container, Union +from typing import Callable, Container, Optional, Union from weakref import WeakValueDictionary -from discord import Colour, Embed, Member -from discord.errors import NotFound +from discord import Colour, Embed, Member, NotFound from discord.ext import commands -from discord.ext.commands import CheckFailure, Cog, Context +from discord.ext.commands import Cog, Context -from bot.constants import ERROR_REPLIES, RedirectOutput -from bot.utils.checks import with_role_check, without_role_check +from bot.constants import Channels, ERROR_REPLIES, RedirectOutput +from bot.utils.checks import in_whitelist_check log = logging.getLogger(__name__) -class InChannelCheckFailure(CheckFailure): - """Raised when a check fails for a message being sent in a whitelisted channel.""" - - def __init__(self, *channels: int): - self.channels = channels - channels_str = ', '.join(f"<#{c_id}>" for c_id in channels) - - super().__init__(f"Sorry, but you may only use this command within {channels_str}.") - - -def in_channel( - *channels: int, - hidden_channels: Container[int] = None, - bypass_roles: Container[int] = None +def in_whitelist( + *, + channels: Container[int] = (), + categories: Container[int] = (), + roles: Container[int] = (), + redirect: Optional[int] = Channels.bot_commands, + fail_silently: bool = False, ) -> Callable: """ - Checks that the message is in a whitelisted channel or optionally has a bypass role. - - Hidden channels are channels which will not be displayed in the InChannelCheckFailure error - message. - """ - hidden_channels = hidden_channels or [] - bypass_roles = bypass_roles or [] + Check if a command was issued in a whitelisted context. - def predicate(ctx: Context) -> bool: - """In-channel checker predicate.""" - if ctx.channel.id in channels or ctx.channel.id in hidden_channels: - log.debug(f"{ctx.author} tried to call the '{ctx.command.name}' command. " - f"The command was used in a whitelisted channel.") - return True + The whitelists that can be provided are: - if bypass_roles: - if any(r.id in bypass_roles for r in ctx.author.roles): - log.debug(f"{ctx.author} tried to call the '{ctx.command.name}' command. " - f"The command was not used in a whitelisted channel, " - f"but the author had a role to bypass the in_channel check.") - return True + - `channels`: a container with channel ids for whitelisted channels + - `categories`: a container with category ids for whitelisted categories + - `roles`: a container with with role ids for whitelisted roles - log.debug(f"{ctx.author} tried to call the '{ctx.command.name}' command. " - f"The in_channel check failed.") - - raise InChannelCheckFailure(*channels) + If the command was invoked in a context that was not whitelisted, the member is either + redirected to the `redirect` channel that was passed (default: #bot-commands) or simply + told that they're not allowed to use this particular command (if `None` was passed). + """ + def predicate(ctx: Context) -> bool: + """Check if command was issued in a whitelisted context.""" + return in_whitelist_check(ctx, channels, categories, roles, redirect, fail_silently) return commands.check(predicate) -def with_role(*role_ids: int) -> Callable: - """Returns True if the user has any one of the roles in role_ids.""" - async def predicate(ctx: Context) -> bool: - """With role checker predicate.""" - return with_role_check(ctx, *role_ids) - return commands.check(predicate) - +def has_no_roles(*roles: Union[str, int]) -> Callable: + """ + Returns True if the user does not have any of the roles specified. -def without_role(*role_ids: int) -> Callable: - """Returns True if the user does not have any of the roles in role_ids.""" + `roles` are the names or IDs of the disallowed roles. + """ async def predicate(ctx: Context) -> bool: - return without_role_check(ctx, *role_ids) + try: + await commands.has_any_role(*roles).predicate(ctx) + except commands.MissingAnyRole: + return True + else: + # This error is never shown to users, so don't bother trying to make it too pretty. + roles_ = ", ".join(f"'{item}'" for item in roles) + raise commands.CheckFailure(f"You have at least one of the disallowed roles: {roles_}") + return commands.check(predicate) @@ -96,7 +81,7 @@ def locked() -> Callable: embed = Embed() embed.colour = Colour.red() - log.debug(f"User tried to invoke a locked command.") + log.debug("User tried to invoke a locked command.") embed.description = ( "You're already using this command. Please wait until it is done before you use it again." ) @@ -137,13 +122,12 @@ def redirect_output(destination_channel: int, bypass_roles: Container[int] = Non log.trace(f"Redirecting output of {ctx.author}'s command '{ctx.command.name}' to {redirect_channel.name}") ctx.channel = redirect_channel await ctx.channel.send(f"Here's the output of your command, {ctx.author.mention}") - await func(self, ctx, *args, **kwargs) + create_task(func(self, ctx, *args, **kwargs)) message = await old_channel.send( f"Hey, {ctx.author.mention}, you can find the output of your command here: " f"{redirect_channel.mention}" ) - if RedirectOutput.delete_invocation: await sleep(RedirectOutput.delete_delay) @@ -154,6 +138,7 @@ def redirect_output(destination_channel: int, bypass_roles: Container[int] = Non with suppress(NotFound): await ctx.message.delete() log.trace("Redirect output: Deleted invocation message") + return inner return wrap diff --git a/bot/cogs/__init__.py b/bot/exts/__init__.py index e69de29bb..e69de29bb 100644 --- a/bot/cogs/__init__.py +++ b/bot/exts/__init__.py diff --git a/tests/bot/cogs/__init__.py b/bot/exts/backend/__init__.py index e69de29bb..e69de29bb 100644 --- a/tests/bot/cogs/__init__.py +++ b/bot/exts/backend/__init__.py diff --git a/bot/cogs/config_verifier.py b/bot/exts/backend/config_verifier.py index d72c6c22e..d72c6c22e 100644 --- a/bot/cogs/config_verifier.py +++ b/bot/exts/backend/config_verifier.py diff --git a/bot/cogs/error_handler.py b/bot/exts/backend/error_handler.py index 6a622d2ce..f9d4de638 100644 --- a/bot/cogs/error_handler.py +++ b/bot/exts/backend/error_handler.py @@ -2,14 +2,15 @@ import contextlib import logging import typing as t -from discord.ext.commands import Cog, Command, Context, errors +from discord import Embed +from discord.ext.commands import Cog, Context, errors from sentry_sdk import push_scope from bot.api import ResponseCodeError from bot.bot import Bot -from bot.constants import Channels +from bot.constants import Channels, Colours from bot.converters import TagNameConverter -from bot.decorators import InChannelCheckFailure +from bot.utils.checks import InWhitelistCheckFailure log = logging.getLogger(__name__) @@ -20,6 +21,14 @@ class ErrorHandler(Cog): def __init__(self, bot: Bot): self.bot = bot + def _get_error_embed(self, title: str, body: str) -> Embed: + """Return an embed that contains the exception.""" + return Embed( + title=title, + colour=Colours.soft_red, + description=body + ) + @Cog.listener() async def on_command_error(self, ctx: Context, e: errors.CommandError) -> None: """ @@ -79,19 +88,13 @@ class ErrorHandler(Cog): f"{e.__class__.__name__}: {e}" ) - async def get_help_command(self, command: t.Optional[Command]) -> t.Tuple: - """Return the help command invocation args to display help for `command`.""" - parent = None - if command is not None: - parent = command.parent - - # Retrieve the help command for the invoked command. - if parent and command: - return self.bot.get_command("help"), parent.name, command.name - elif command: - return self.bot.get_command("help"), command.name - else: - return self.bot.get_command("help") + @staticmethod + def get_help_command(ctx: Context) -> t.Coroutine: + """Return a prepared `help` command invocation coroutine.""" + if ctx.command: + return ctx.send_help(ctx.command) + + return ctx.send_help() async def try_silence(self, ctx: Context) -> bool: """ @@ -165,25 +168,39 @@ class ErrorHandler(Cog): * ArgumentParsingError: send an error message * Other: send an error message and the help command """ - # TODO: use ctx.send_help() once PR #519 is merged. - help_command = await self.get_help_command(ctx.command) + prepared_help_command = self.get_help_command(ctx) if isinstance(e, errors.MissingRequiredArgument): - await ctx.send(f"Missing required argument `{e.param.name}`.") - await ctx.invoke(*help_command) + embed = self._get_error_embed("Missing required argument", e.param.name) + await ctx.send(embed=embed) + await prepared_help_command + self.bot.stats.incr("errors.missing_required_argument") elif isinstance(e, errors.TooManyArguments): - await ctx.send(f"Too many arguments provided.") - await ctx.invoke(*help_command) + embed = self._get_error_embed("Too many arguments", str(e)) + await ctx.send(embed=embed) + await prepared_help_command + self.bot.stats.incr("errors.too_many_arguments") elif isinstance(e, errors.BadArgument): - await ctx.send(f"Bad argument: {e}\n") - await ctx.invoke(*help_command) + embed = self._get_error_embed("Bad argument", str(e)) + await ctx.send(embed=embed) + await prepared_help_command + self.bot.stats.incr("errors.bad_argument") elif isinstance(e, errors.BadUnionArgument): - await ctx.send(f"Bad argument: {e}\n```{e.errors[-1]}```") + embed = self._get_error_embed("Bad argument", f"{e}\n{e.errors[-1]}") + await ctx.send(embed=embed) + self.bot.stats.incr("errors.bad_union_argument") elif isinstance(e, errors.ArgumentParsingError): - await ctx.send(f"Argument parsing error: {e}") + embed = self._get_error_embed("Argument parsing error", str(e)) + await ctx.send(embed=embed) + self.bot.stats.incr("errors.argument_parsing_error") else: - await ctx.send("Something about your input seems off. Check the arguments:") - await ctx.invoke(*help_command) + embed = self._get_error_embed( + "Input error", + "Something about your input seems off. Check the arguments and try again." + ) + await ctx.send(embed=embed) + await prepared_help_command + self.bot.stats.incr("errors.other_user_input_error") @staticmethod async def handle_check_failure(ctx: Context, e: errors.CheckFailure) -> None: @@ -196,7 +213,7 @@ class ErrorHandler(Cog): * BotMissingRole * BotMissingAnyRole * NoPrivateMessage - * InChannelCheckFailure + * InWhitelistCheckFailure """ bot_missing_errors = ( errors.BotMissingPermissions, @@ -205,10 +222,12 @@ class ErrorHandler(Cog): ) if isinstance(e, bot_missing_errors): + ctx.bot.stats.incr("errors.bot_permission_error") await ctx.send( - f"Sorry, it looks like I don't have the permissions or roles I need to do that." + "Sorry, it looks like I don't have the permissions or roles I need to do that." ) - elif isinstance(e, (InChannelCheckFailure, errors.NoPrivateMessage)): + elif isinstance(e, (InWhitelistCheckFailure, errors.NoPrivateMessage)): + ctx.bot.stats.incr("errors.wrong_channel_or_dm_error") await ctx.send(e) @staticmethod @@ -217,16 +236,20 @@ class ErrorHandler(Cog): if e.status == 404: await ctx.send("There does not seem to be anything matching your query.") log.debug(f"API responded with 404 for command {ctx.command}") + ctx.bot.stats.incr("errors.api_error_404") elif e.status == 400: content = await e.response.json() log.debug(f"API responded with 400 for command {ctx.command}: %r.", content) await ctx.send("According to the API, your request is malformed.") + ctx.bot.stats.incr("errors.api_error_400") elif 500 <= e.status < 600: await ctx.send("Sorry, there seems to be an internal issue with the API.") log.warning(f"API responded with {e.status} for command {ctx.command}") + ctx.bot.stats.incr("errors.api_internal_server_error") else: await ctx.send(f"Got an unexpected status code from the API (`{e.status}`).") log.warning(f"Unexpected API response for command {ctx.command}: {e.status}") + ctx.bot.stats.incr(f"errors.api_error_{e.status}") @staticmethod async def handle_unexpected_error(ctx: Context, e: errors.CommandError) -> None: @@ -236,6 +259,8 @@ class ErrorHandler(Cog): f"```{e.__class__.__name__}: {e}```" ) + ctx.bot.stats.incr("errors.unexpected") + with push_scope() as scope: scope.user = { "id": ctx.author.id, diff --git a/bot/cogs/logging.py b/bot/exts/backend/logging.py index 94fa2b139..94fa2b139 100644 --- a/bot/cogs/logging.py +++ b/bot/exts/backend/logging.py diff --git a/bot/exts/backend/sync/__init__.py b/bot/exts/backend/sync/__init__.py new file mode 100644 index 000000000..829098f79 --- /dev/null +++ b/bot/exts/backend/sync/__init__.py @@ -0,0 +1,8 @@ +from bot.bot import Bot + + +def setup(bot: Bot) -> None: + """Load the Sync cog.""" + # Defer import to reduce side effects from importing the sync package. + from bot.exts.backend.sync._cog import Sync + bot.add_cog(Sync(bot)) diff --git a/bot/cogs/sync/cog.py b/bot/exts/backend/sync/_cog.py index 5708be3f4..6e85e2b7d 100644 --- a/bot/cogs/sync/cog.py +++ b/bot/exts/backend/sync/_cog.py @@ -8,7 +8,7 @@ from discord.ext.commands import Cog, Context from bot import constants from bot.api import ResponseCodeError from bot.bot import Bot -from bot.cogs.sync import syncers +from bot.exts.backend.sync import _syncers log = logging.getLogger(__name__) @@ -18,8 +18,8 @@ class Sync(Cog): def __init__(self, bot: Bot) -> None: self.bot = bot - self.role_syncer = syncers.RoleSyncer(self.bot) - self.user_syncer = syncers.UserSyncer(self.bot) + self.role_syncer = _syncers.RoleSyncer(self.bot) + self.user_syncer = _syncers.UserSyncer(self.bot) self.bot.loop.create_task(self.sync_guild()) @@ -34,18 +34,22 @@ class Sync(Cog): for syncer in (self.role_syncer, self.user_syncer): await syncer.sync(guild) - async def patch_user(self, user_id: int, updated_information: Dict[str, Any]) -> None: + async def patch_user(self, user_id: int, json: Dict[str, Any], ignore_404: bool = False) -> None: """Send a PATCH request to partially update a user in the database.""" try: - await self.bot.api_client.patch(f"bot/users/{user_id}", json=updated_information) + await self.bot.api_client.patch(f"bot/users/{user_id}", json=json) except ResponseCodeError as e: if e.response.status != 404: raise - log.warning("Unable to update user, got 404. Assuming race condition from join event.") + if not ignore_404: + log.warning("Unable to update user, got 404. Assuming race condition from join event.") @Cog.listener() async def on_guild_role_create(self, role: Role) -> None: """Adds newly create role to the database table over the API.""" + if role.guild.id != constants.Guild.id: + return + await self.bot.api_client.post( 'bot/roles', json={ @@ -60,11 +64,17 @@ class Sync(Cog): @Cog.listener() async def on_guild_role_delete(self, role: Role) -> None: """Deletes role from the database when it's deleted from the guild.""" + if role.guild.id != constants.Guild.id: + return + await self.bot.api_client.delete(f'bot/roles/{role.id}') @Cog.listener() async def on_guild_role_update(self, before: Role, after: Role) -> None: """Syncs role with the database if any of the stored attributes were updated.""" + if after.guild.id != constants.Guild.id: + return + was_updated = ( before.name != after.name or before.colour != after.colour @@ -93,8 +103,10 @@ class Sync(Cog): previously left), it will update the user's information. If the user is not yet known by the database, the user is added. """ + if member.guild.id != constants.Guild.id: + return + packed = { - 'avatar_hash': member.avatar, 'discriminator': int(member.discriminator), 'id': member.id, 'in_guild': True, @@ -123,26 +135,32 @@ class Sync(Cog): @Cog.listener() async def on_member_remove(self, member: Member) -> None: """Set the in_guild field to False when a member leaves the guild.""" - await self.patch_user(member.id, updated_information={"in_guild": False}) + if member.guild.id != constants.Guild.id: + return + + await self.patch_user(member.id, json={"in_guild": False}) @Cog.listener() async def on_member_update(self, before: Member, after: Member) -> None: """Update the roles of the member in the database if a change is detected.""" + if after.guild.id != constants.Guild.id: + return + if before.roles != after.roles: updated_information = {"roles": sorted(role.id for role in after.roles)} - await self.patch_user(after.id, updated_information=updated_information) + await self.patch_user(after.id, json=updated_information) @Cog.listener() async def on_user_update(self, before: User, after: User) -> None: """Update the user information in the database if a relevant change is detected.""" - attrs = ("name", "discriminator", "avatar") + attrs = ("name", "discriminator") if any(getattr(before, attr) != getattr(after, attr) for attr in attrs): updated_information = { "name": after.name, "discriminator": int(after.discriminator), - "avatar_hash": after.avatar, } - await self.patch_user(after.id, updated_information=updated_information) + # A 404 likely means the user is in another guild. + await self.patch_user(after.id, json=updated_information, ignore_404=True) @commands.group(name='sync') @commands.has_permissions(administrator=True) diff --git a/bot/exts/backend/sync/_syncers.py b/bot/exts/backend/sync/_syncers.py new file mode 100644 index 000000000..3d4a09df3 --- /dev/null +++ b/bot/exts/backend/sync/_syncers.py @@ -0,0 +1,194 @@ +import abc +import logging +import typing as t +from collections import namedtuple + +from discord import Guild +from discord.ext.commands import Context + +from bot.api import ResponseCodeError +from bot.bot import Bot + +log = logging.getLogger(__name__) + +# These objects are declared as namedtuples because tuples are hashable, +# something that we make use of when diffing site roles against guild roles. +_Role = namedtuple('Role', ('id', 'name', 'colour', 'permissions', 'position')) +_User = namedtuple('User', ('id', 'name', 'discriminator', 'roles', 'in_guild')) +_Diff = namedtuple('Diff', ('created', 'updated', 'deleted')) + + +class Syncer(abc.ABC): + """Base class for synchronising the database with objects in the Discord cache.""" + + def __init__(self, bot: Bot) -> None: + self.bot = bot + + @property + @abc.abstractmethod + def name(self) -> str: + """The name of the syncer; used in output messages and logging.""" + raise NotImplementedError # pragma: no cover + + @abc.abstractmethod + async def _get_diff(self, guild: Guild) -> _Diff: + """Return the difference between the cache of `guild` and the database.""" + raise NotImplementedError # pragma: no cover + + @abc.abstractmethod + async def _sync(self, diff: _Diff) -> None: + """Perform the API calls for synchronisation.""" + raise NotImplementedError # pragma: no cover + + async def sync(self, guild: Guild, ctx: t.Optional[Context] = None) -> None: + """ + Synchronise the database with the cache of `guild`. + + If `ctx` is given, send a message with the results. + """ + log.info(f"Starting {self.name} syncer.") + + if ctx: + message = await ctx.send(f"📊 Synchronising {self.name}s.") + else: + message = None + diff = await self._get_diff(guild) + + try: + await self._sync(diff) + except ResponseCodeError as e: + log.exception(f"{self.name} syncer failed!") + + # Don't show response text because it's probably some really long HTML. + results = f"status {e.status}\n```{e.response_json or 'See log output for details'}```" + content = f":x: Synchronisation of {self.name}s failed: {results}" + else: + diff_dict = diff._asdict() + results = (f"{name} `{len(val)}`" for name, val in diff_dict.items() if val is not None) + results = ", ".join(results) + + log.info(f"{self.name} syncer finished: {results}.") + content = f":ok_hand: Synchronisation of {self.name}s complete: {results}" + + if message: + await message.edit(content=content) + + +class RoleSyncer(Syncer): + """Synchronise the database with roles in the cache.""" + + name = "role" + + async def _get_diff(self, guild: Guild) -> _Diff: + """Return the difference of roles between the cache of `guild` and the database.""" + log.trace("Getting the diff for roles.") + roles = await self.bot.api_client.get('bot/roles') + + # Pack DB roles and guild roles into one common, hashable format. + # They're hashable so that they're easily comparable with sets later. + db_roles = {_Role(**role_dict) for role_dict in roles} + guild_roles = { + _Role( + id=role.id, + name=role.name, + colour=role.colour.value, + permissions=role.permissions.value, + position=role.position, + ) + for role in guild.roles + } + + guild_role_ids = {role.id for role in guild_roles} + api_role_ids = {role.id for role in db_roles} + new_role_ids = guild_role_ids - api_role_ids + deleted_role_ids = api_role_ids - guild_role_ids + + # New roles are those which are on the cached guild but not on the + # DB guild, going by the role ID. We need to send them in for creation. + roles_to_create = {role for role in guild_roles if role.id in new_role_ids} + roles_to_update = guild_roles - db_roles - roles_to_create + roles_to_delete = {role for role in db_roles if role.id in deleted_role_ids} + + return _Diff(roles_to_create, roles_to_update, roles_to_delete) + + async def _sync(self, diff: _Diff) -> None: + """Synchronise the database with the role cache of `guild`.""" + log.trace("Syncing created roles...") + for role in diff.created: + await self.bot.api_client.post('bot/roles', json=role._asdict()) + + log.trace("Syncing updated roles...") + for role in diff.updated: + await self.bot.api_client.put(f'bot/roles/{role.id}', json=role._asdict()) + + log.trace("Syncing deleted roles...") + for role in diff.deleted: + await self.bot.api_client.delete(f'bot/roles/{role.id}') + + +class UserSyncer(Syncer): + """Synchronise the database with users in the cache.""" + + name = "user" + + async def _get_diff(self, guild: Guild) -> _Diff: + """Return the difference of users between the cache of `guild` and the database.""" + log.trace("Getting the diff for users.") + users = await self.bot.api_client.get('bot/users') + + # Pack DB roles and guild roles into one common, hashable format. + # They're hashable so that they're easily comparable with sets later. + db_users = { + user_dict['id']: _User( + roles=tuple(sorted(user_dict.pop('roles'))), + **user_dict + ) + for user_dict in users + } + guild_users = { + member.id: _User( + id=member.id, + name=member.name, + discriminator=int(member.discriminator), + roles=tuple(sorted(role.id for role in member.roles)), + in_guild=True + ) + for member in guild.members + } + + users_to_create = set() + users_to_update = set() + + for db_user in db_users.values(): + guild_user = guild_users.get(db_user.id) + if guild_user is not None: + if db_user != guild_user: + users_to_update.add(guild_user) + + elif db_user.in_guild: + # The user is known in the DB but not the guild, and the + # DB currently specifies that the user is a member of the guild. + # This means that the user has left since the last sync. + # Update the `in_guild` attribute of the user on the site + # to signify that the user left. + new_api_user = db_user._replace(in_guild=False) + users_to_update.add(new_api_user) + + new_user_ids = set(guild_users.keys()) - set(db_users.keys()) + for user_id in new_user_ids: + # The user is known on the guild but not on the API. This means + # that the user has joined since the last sync. Create it. + new_user = guild_users[user_id] + users_to_create.add(new_user) + + return _Diff(users_to_create, users_to_update, None) + + async def _sync(self, diff: _Diff) -> None: + """Synchronise the database with the user cache of `guild`.""" + log.trace("Syncing created users...") + for user in diff.created: + await self.bot.api_client.post('bot/users', json=user._asdict()) + + log.trace("Syncing updated users...") + for user in diff.updated: + await self.bot.api_client.put(f'bot/users/{user.id}', json=user._asdict()) diff --git a/tests/bot/cogs/moderation/__init__.py b/bot/exts/filters/__init__.py index e69de29bb..e69de29bb 100644 --- a/tests/bot/cogs/moderation/__init__.py +++ b/bot/exts/filters/__init__.py diff --git a/bot/exts/filters/antimalware.py b/bot/exts/filters/antimalware.py new file mode 100644 index 000000000..7894ec48f --- /dev/null +++ b/bot/exts/filters/antimalware.py @@ -0,0 +1,102 @@ +import logging +import typing as t +from os.path import splitext + +from discord import Embed, Message, NotFound +from discord.ext.commands import Cog + +from bot.bot import Bot +from bot.constants import Channels, STAFF_ROLES, URLs + +log = logging.getLogger(__name__) + +PY_EMBED_DESCRIPTION = ( + "It looks like you tried to attach a Python file - " + f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" +) + +TXT_EMBED_DESCRIPTION = ( + "**Uh-oh!** It looks like your message got zapped by our spam filter. " + "We currently don't allow `.txt` attachments, so here are some tips to help you travel safely: \n\n" + "• If you attempted to send a message longer than 2000 characters, try shortening your message " + "to fit within the character limit or use a pasting service (see below) \n\n" + "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " + "{cmd_channel_mention} for more information) or use a pasting service like: " + f"\n\n{URLs.site_schema}{URLs.site_paste}" +) + +DISALLOWED_EMBED_DESCRIPTION = ( + "It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " + "We currently allow the following file types: **{joined_whitelist}**.\n\n" + "Feel free to ask in {meta_channel_mention} if you think this is a mistake." +) + + +class AntiMalware(Cog): + """Delete messages which contain attachments with non-whitelisted file extensions.""" + + def __init__(self, bot: Bot): + self.bot = bot + + def _get_whitelisted_file_formats(self) -> list: + """Get the file formats currently on the whitelist.""" + return self.bot.filter_list_cache['FILE_FORMAT.True'].keys() + + def _get_disallowed_extensions(self, message: Message) -> t.Iterable[str]: + """Get an iterable containing all the disallowed extensions of attachments.""" + file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} + extensions_blocked = file_extensions - set(self._get_whitelisted_file_formats()) + return extensions_blocked + + @Cog.listener() + async def on_message(self, message: Message) -> None: + """Identify messages with prohibited attachments.""" + # Return when message don't have attachment and don't moderate DMs + if not message.attachments or not message.guild: + return + + # Ignore webhook and bot messages + if message.webhook_id or message.author.bot: + return + + # Check if user is staff, if is, return + # Since we only care that roles exist to iterate over, check for the attr rather than a User/Member instance + if hasattr(message.author, "roles") and any(role.id in STAFF_ROLES for role in message.author.roles): + return + + embed = Embed() + extensions_blocked = self._get_disallowed_extensions(message) + blocked_extensions_str = ', '.join(extensions_blocked) + if ".py" in extensions_blocked: + # Short-circuit on *.py files to provide a pastebin link + embed.description = PY_EMBED_DESCRIPTION + elif ".txt" in extensions_blocked: + # Work around Discord AutoConversion of messages longer than 2000 chars to .txt + cmd_channel = self.bot.get_channel(Channels.bot_commands) + embed.description = TXT_EMBED_DESCRIPTION.format(cmd_channel_mention=cmd_channel.mention) + elif extensions_blocked: + meta_channel = self.bot.get_channel(Channels.meta) + embed.description = DISALLOWED_EMBED_DESCRIPTION.format( + joined_whitelist=', '.join(self._get_whitelisted_file_formats()), + blocked_extensions_str=blocked_extensions_str, + meta_channel_mention=meta_channel.mention, + ) + + if embed.description: + log.info( + f"User '{message.author}' ({message.author.id}) uploaded blacklisted file(s): {blocked_extensions_str}", + extra={"attachment_list": [attachment.filename for attachment in message.attachments]} + ) + + await message.channel.send(f"Hey {message.author.mention}!", embed=embed) + + # Delete the offending message: + try: + await message.delete() + except NotFound: + log.info(f"Tried to delete message `{message.id}`, but message could not be found.") + + +def setup(bot: Bot) -> None: + """Load the AntiMalware cog.""" + bot.add_cog(AntiMalware(bot)) diff --git a/bot/cogs/antispam.py b/bot/exts/filters/antispam.py index baa6b9459..4964283f1 100644 --- a/bot/cogs/antispam.py +++ b/bot/exts/filters/antispam.py @@ -11,7 +11,6 @@ from discord.ext.commands import Cog from bot import rules from bot.bot import Bot -from bot.cogs.moderation import ModLog from bot.constants import ( AntiSpam as AntiSpamConfig, Channels, Colours, DEBUG_MODE, Event, Filter, @@ -19,7 +18,8 @@ from bot.constants import ( STAFF_ROLES, ) from bot.converters import Duration -from bot.utils.messages import send_attachments +from bot.exts.moderation.modlog import ModLog +from bot.utils.messages import format_user, send_attachments log = logging.getLogger(__name__) @@ -27,14 +27,15 @@ log = logging.getLogger(__name__) RULE_FUNCTION_MAPPING = { 'attachments': rules.apply_attachments, 'burst': rules.apply_burst, - 'burst_shared': rules.apply_burst_shared, + # burst shared is temporarily disabled due to a bug + # 'burst_shared': rules.apply_burst_shared, 'chars': rules.apply_chars, 'discord_emojis': rules.apply_discord_emojis, 'duplicates': rules.apply_duplicates, 'links': rules.apply_links, 'mentions': rules.apply_mentions, 'newlines': rules.apply_newlines, - 'role_mentions': rules.apply_role_mentions + 'role_mentions': rules.apply_role_mentions, } @@ -67,7 +68,7 @@ class DeletionContext: async def upload_messages(self, actor_id: int, modlog: ModLog) -> None: """Method that takes care of uploading the queue and posting modlog alert.""" - triggered_by_users = ", ".join(f"{m} (`{m.id}`)" for m in self.members.values()) + triggered_by_users = ", ".join(format_user(m) for m in self.members.values()) mod_alert_message = ( f"**Triggered by:** {triggered_by_users}\n" @@ -94,7 +95,7 @@ class DeletionContext: await modlog.send_log_message( icon_url=Icons.filtering, colour=Colour(Colours.soft_red), - title=f"Spam detected!", + title="Spam detected!", text=mod_alert_message, thumbnail=last_message.author.avatar_url_as(static_format="png"), channel_id=Channels.mod_alerts, @@ -130,7 +131,7 @@ class AntiSpam(Cog): body += "\n\n**The cog has been unloaded.**" await self.mod_log.send_log_message( - title=f"Error: AntiSpam configuration validation failed!", + title="Error: AntiSpam configuration validation failed!", text=body, ping_everyone=True, icon_url=Icons.token_removed, @@ -182,6 +183,7 @@ class AntiSpam(Cog): # which contains the reason for why the message violated the rule and # an iterable of all members that violated the rule. if result is not None: + self.bot.stats.incr(f"mod_alerts.{rule_name}") reason, members, relevant_messages = result full_reason = f"`{rule_name}` rule: {reason}" @@ -218,7 +220,6 @@ class AntiSpam(Cog): # Get context and make sure the bot becomes the actor of infraction by patching the `author` attributes context = await self.bot.get_context(msg) context.author = self.bot.user - context.message.author = self.bot.user # Since we're going to invoke the tempmute command directly, we need to manually call the converter. dt_remove_role_after = await self.expiration_date_converter.convert(context, f"{remove_role_after}S") diff --git a/bot/exts/filters/filter_lists.py b/bot/exts/filters/filter_lists.py new file mode 100644 index 000000000..232c1e48b --- /dev/null +++ b/bot/exts/filters/filter_lists.py @@ -0,0 +1,272 @@ +import logging +from typing import Optional + +from discord import Colour, Embed +from discord.ext.commands import BadArgument, Cog, Context, IDConverter, group, has_any_role + +from bot import constants +from bot.api import ResponseCodeError +from bot.bot import Bot +from bot.converters import ValidDiscordServerInvite, ValidFilterListType +from bot.pagination import LinePaginator + +log = logging.getLogger(__name__) + + +class FilterLists(Cog): + """Commands for blacklisting and whitelisting things.""" + + methods_with_filterlist_types = [ + "allow_add", + "allow_delete", + "allow_get", + "deny_add", + "deny_delete", + "deny_get", + ] + + def __init__(self, bot: Bot) -> None: + self.bot = bot + self.bot.loop.create_task(self._amend_docstrings()) + + async def _amend_docstrings(self) -> None: + """Add the valid FilterList types to the docstrings, so they'll appear in !help invocations.""" + await self.bot.wait_until_guild_available() + + # Add valid filterlist types to the docstrings + valid_types = await ValidFilterListType.get_valid_types(self.bot) + valid_types = [f"`{type_.lower()}`" for type_ in valid_types] + + for method_name in self.methods_with_filterlist_types: + command = getattr(self, method_name) + command.help = ( + f"{command.help}\n\nValid **list_type** values are {', '.join(valid_types)}." + ) + + async def _add_data( + self, + ctx: Context, + allowed: bool, + list_type: ValidFilterListType, + content: str, + comment: Optional[str] = None, + ) -> None: + """Add an item to a filterlist.""" + allow_type = "whitelist" if allowed else "blacklist" + + # If this is a server invite, we gotta validate it. + if list_type == "GUILD_INVITE": + guild_data = await self._validate_guild_invite(ctx, content) + content = guild_data.get("id") + + # Unless the user has specified another comment, let's + # use the server name as the comment so that the list + # of guild IDs will be more easily readable when we + # display it. + if not comment: + comment = guild_data.get("name") + + # If it's a file format, let's make sure it has a leading dot. + elif list_type == "FILE_FORMAT" and not content.startswith("."): + content = f".{content}" + + # Try to add the item to the database + log.trace(f"Trying to add the {content} item to the {list_type} {allow_type}") + payload = { + "allowed": allowed, + "type": list_type, + "content": content, + "comment": comment, + } + + try: + item = await self.bot.api_client.post( + "bot/filter-lists", + json=payload + ) + except ResponseCodeError as e: + if e.status == 400: + await ctx.message.add_reaction("❌") + log.debug( + f"{ctx.author} tried to add data to a {allow_type}, but the API returned 400, " + "probably because the request violated the UniqueConstraint." + ) + raise BadArgument( + f"Unable to add the item to the {allow_type}. " + "The item probably already exists. Keep in mind that a " + "blacklist and a whitelist for the same item cannot co-exist, " + "and we do not permit any duplicates." + ) + raise + + # Insert the item into the cache + self.bot.insert_item_into_filter_list_cache(item) + await ctx.message.add_reaction("✅") + + async def _delete_data(self, ctx: Context, allowed: bool, list_type: ValidFilterListType, content: str) -> None: + """Remove an item from a filterlist.""" + allow_type = "whitelist" if allowed else "blacklist" + + # If this is a server invite, we need to convert it. + if list_type == "GUILD_INVITE" and not IDConverter()._get_id_match(content): + guild_data = await self._validate_guild_invite(ctx, content) + content = guild_data.get("id") + + # If it's a file format, let's make sure it has a leading dot. + elif list_type == "FILE_FORMAT" and not content.startswith("."): + content = f".{content}" + + # Find the content and delete it. + log.trace(f"Trying to delete the {content} item from the {list_type} {allow_type}") + item = self.bot.filter_list_cache[f"{list_type}.{allowed}"].get(content) + + if item is not None: + try: + await self.bot.api_client.delete( + f"bot/filter-lists/{item['id']}" + ) + del self.bot.filter_list_cache[f"{list_type}.{allowed}"][content] + await ctx.message.add_reaction("✅") + except ResponseCodeError as e: + log.debug( + f"{ctx.author} tried to delete an item with the id {item['id']}, but " + f"the API raised an unexpected error: {e}" + ) + await ctx.message.add_reaction("❌") + else: + await ctx.message.add_reaction("❌") + + async def _list_all_data(self, ctx: Context, allowed: bool, list_type: ValidFilterListType) -> None: + """Paginate and display all items in a filterlist.""" + allow_type = "whitelist" if allowed else "blacklist" + result = self.bot.filter_list_cache[f"{list_type}.{allowed}"] + + # Build a list of lines we want to show in the paginator + lines = [] + for content, metadata in result.items(): + line = f"• `{content}`" + + if comment := metadata.get("comment"): + line += f" - {comment}" + + lines.append(line) + lines = sorted(lines) + + # Build the embed + list_type_plural = list_type.lower().replace("_", " ").title() + "s" + embed = Embed( + title=f"{allow_type.title()}ed {list_type_plural} ({len(result)} total)", + colour=Colour.blue() + ) + log.trace(f"Trying to list {len(result)} items from the {list_type.lower()} {allow_type}") + + if result: + await LinePaginator.paginate(lines, ctx, embed, max_lines=15, empty=False) + else: + embed.description = "Hmmm, seems like there's nothing here yet." + await ctx.send(embed=embed) + await ctx.message.add_reaction("❌") + + async def _sync_data(self, ctx: Context) -> None: + """Syncs the filterlists with the API.""" + try: + log.trace("Attempting to sync FilterList cache with data from the API.") + await self.bot.cache_filter_list_data() + await ctx.message.add_reaction("✅") + except ResponseCodeError as e: + log.debug( + f"{ctx.author} tried to sync FilterList cache data but " + f"the API raised an unexpected error: {e}" + ) + await ctx.message.add_reaction("❌") + + @staticmethod + async def _validate_guild_invite(ctx: Context, invite: str) -> dict: + """ + Validates a guild invite, and returns the guild info as a dict. + + Will raise a BadArgument if the guild invite is invalid. + """ + log.trace(f"Attempting to validate whether or not {invite} is a guild invite.") + validator = ValidDiscordServerInvite() + guild_data = await validator.convert(ctx, invite) + + # If we make it this far without raising a BadArgument, the invite is + # valid. Let's return a dict of guild information. + log.trace(f"{invite} validated as server invite. Converting to ID.") + return guild_data + + @group(aliases=("allowlist", "allow", "al", "wl")) + async def whitelist(self, ctx: Context) -> None: + """Group for whitelisting commands.""" + if not ctx.invoked_subcommand: + await ctx.send_help(ctx.command) + + @group(aliases=("denylist", "deny", "bl", "dl")) + async def blacklist(self, ctx: Context) -> None: + """Group for blacklisting commands.""" + if not ctx.invoked_subcommand: + await ctx.send_help(ctx.command) + + @whitelist.command(name="add", aliases=("a", "set")) + async def allow_add( + self, + ctx: Context, + list_type: ValidFilterListType, + content: str, + *, + comment: Optional[str] = None, + ) -> None: + """Add an item to the specified allowlist.""" + await self._add_data(ctx, True, list_type, content, comment) + + @blacklist.command(name="add", aliases=("a", "set")) + async def deny_add( + self, + ctx: Context, + list_type: ValidFilterListType, + content: str, + *, + comment: Optional[str] = None, + ) -> None: + """Add an item to the specified denylist.""" + await self._add_data(ctx, False, list_type, content, comment) + + @whitelist.command(name="remove", aliases=("delete", "rm",)) + async def allow_delete(self, ctx: Context, list_type: ValidFilterListType, content: str) -> None: + """Remove an item from the specified allowlist.""" + await self._delete_data(ctx, True, list_type, content) + + @blacklist.command(name="remove", aliases=("delete", "rm",)) + async def deny_delete(self, ctx: Context, list_type: ValidFilterListType, content: str) -> None: + """Remove an item from the specified denylist.""" + await self._delete_data(ctx, False, list_type, content) + + @whitelist.command(name="get", aliases=("list", "ls", "fetch", "show")) + async def allow_get(self, ctx: Context, list_type: ValidFilterListType) -> None: + """Get the contents of a specified allowlist.""" + await self._list_all_data(ctx, True, list_type) + + @blacklist.command(name="get", aliases=("list", "ls", "fetch", "show")) + async def deny_get(self, ctx: Context, list_type: ValidFilterListType) -> None: + """Get the contents of a specified denylist.""" + await self._list_all_data(ctx, False, list_type) + + @whitelist.command(name="sync", aliases=("s",)) + async def allow_sync(self, ctx: Context) -> None: + """Syncs both allowlists and denylists with the API.""" + await self._sync_data(ctx) + + @blacklist.command(name="sync", aliases=("s",)) + async def deny_sync(self, ctx: Context) -> None: + """Syncs both allowlists and denylists with the API.""" + await self._sync_data(ctx) + + async def cog_check(self, ctx: Context) -> bool: + """Only allow moderators to invoke the commands in this cog.""" + return await has_any_role(*constants.MODERATION_ROLES).predicate(ctx) + + +def setup(bot: Bot) -> None: + """Load the FilterLists cog.""" + bot.add_cog(FilterLists(bot)) diff --git a/bot/exts/filters/filtering.py b/bot/exts/filters/filtering.py new file mode 100644 index 000000000..92cdfb8f5 --- /dev/null +++ b/bot/exts/filters/filtering.py @@ -0,0 +1,605 @@ +import asyncio +import logging +import re +from datetime import datetime, timedelta +from typing import Any, Dict, List, Mapping, NamedTuple, Optional, Union + +import dateutil +import discord.errors +from async_rediscache import RedisCache +from dateutil.relativedelta import relativedelta +from discord import Colour, HTTPException, Member, Message, NotFound, TextChannel +from discord.ext.commands import Cog +from discord.utils import escape_markdown + +from bot.api import ResponseCodeError +from bot.bot import Bot +from bot.constants import ( + Channels, Colours, Filter, + Guild, Icons, URLs +) +from bot.exts.moderation.modlog import ModLog +from bot.utils.messages import format_user +from bot.utils.regex import INVITE_RE +from bot.utils.scheduling import Scheduler + +log = logging.getLogger(__name__) + +# Regular expressions +CODE_BLOCK_RE = re.compile( + r"(?P<delim>``?)[^`]+?(?P=delim)(?!`+)" # Inline codeblock + r"|```(.+?)```", # Multiline codeblock + re.DOTALL | re.MULTILINE +) +EVERYONE_PING_RE = re.compile(rf"@everyone|<@&{Guild.id}>|@here") +SPOILER_RE = re.compile(r"(\|\|.+?\|\|)", re.DOTALL) +URL_RE = re.compile(r"(https?://[^\s]+)", flags=re.IGNORECASE) +ZALGO_RE = re.compile(r"[\u0300-\u036F\u0489]") + +# Other constants. +DAYS_BETWEEN_ALERTS = 3 +OFFENSIVE_MSG_DELETE_TIME = timedelta(days=Filter.offensive_msg_delete_days) + +FilterMatch = Union[re.Match, dict, bool, List[discord.Embed]] + + +class Stats(NamedTuple): + """Additional stats on a triggered filter to append to a mod log.""" + + message_content: str + additional_embeds: Optional[List[discord.Embed]] + additional_embeds_msg: Optional[str] + + +class Filtering(Cog): + """Filtering out invites, blacklisting domains, and warning us of certain regular expressions.""" + + # Redis cache mapping a user ID to the last timestamp a bad nickname alert was sent + name_alerts = RedisCache() + + def __init__(self, bot: Bot): + self.bot = bot + self.scheduler = Scheduler(self.__class__.__name__) + self.name_lock = asyncio.Lock() + + staff_mistake_str = "If you believe this was a mistake, please let staff know!" + self.filters = { + "filter_zalgo": { + "enabled": Filter.filter_zalgo, + "function": self._has_zalgo, + "type": "filter", + "content_only": True, + "user_notification": Filter.notify_user_zalgo, + "notification_msg": ( + "Your post has been removed for abusing Unicode character rendering (aka Zalgo text). " + f"{staff_mistake_str}" + ), + "schedule_deletion": False + }, + "filter_invites": { + "enabled": Filter.filter_invites, + "function": self._has_invites, + "type": "filter", + "content_only": True, + "user_notification": Filter.notify_user_invites, + "notification_msg": ( + f"Per Rule 6, your invite link has been removed. {staff_mistake_str}\n\n" + r"Our server rules can be found here: <https://pythondiscord.com/pages/rules>" + ), + "schedule_deletion": False + }, + "filter_domains": { + "enabled": Filter.filter_domains, + "function": self._has_urls, + "type": "filter", + "content_only": True, + "user_notification": Filter.notify_user_domains, + "notification_msg": ( + f"Your URL has been removed because it matched a blacklisted domain. {staff_mistake_str}" + ), + "schedule_deletion": False + }, + "filter_everyone_ping": { + "enabled": Filter.filter_everyone_ping, + "function": self._has_everyone_ping, + "type": "filter", + "content_only": True, + "user_notification": Filter.notify_user_everyone_ping, + "notification_msg": ( + "Please don't try to ping `@everyone` or `@here`. " + f"Your message has been removed. {staff_mistake_str}" + ), + "schedule_deletion": False, + "ping_everyone": False + }, + "watch_regex": { + "enabled": Filter.watch_regex, + "function": self._has_watch_regex_match, + "type": "watchlist", + "content_only": True, + "schedule_deletion": True + }, + "watch_rich_embeds": { + "enabled": Filter.watch_rich_embeds, + "function": self._has_rich_embed, + "type": "watchlist", + "content_only": False, + "schedule_deletion": False + } + } + + self.bot.loop.create_task(self.reschedule_offensive_msg_deletion()) + + def cog_unload(self) -> None: + """Cancel scheduled tasks.""" + self.scheduler.cancel_all() + + def _get_filterlist_items(self, list_type: str, *, allowed: bool) -> list: + """Fetch items from the filter_list_cache.""" + return self.bot.filter_list_cache[f"{list_type.upper()}.{allowed}"].keys() + + @staticmethod + def _expand_spoilers(text: str) -> str: + """Return a string containing all interpretations of a spoilered message.""" + split_text = SPOILER_RE.split(text) + return ''.join( + split_text[0::2] + split_text[1::2] + split_text + ) + + @property + def mod_log(self) -> ModLog: + """Get currently loaded ModLog cog instance.""" + return self.bot.get_cog("ModLog") + + @Cog.listener() + async def on_message(self, msg: Message) -> None: + """Invoke message filter for new messages.""" + await self._filter_message(msg) + + # Ignore webhook messages. + if msg.webhook_id is None: + await self.check_bad_words_in_name(msg.author) + + @Cog.listener() + async def on_message_edit(self, before: Message, after: Message) -> None: + """ + Invoke message filter for message edits. + + If there have been multiple edits, calculate the time delta from the previous edit. + """ + if not before.edited_at: + delta = relativedelta(after.edited_at, before.created_at).microseconds + else: + delta = relativedelta(after.edited_at, before.edited_at).microseconds + await self._filter_message(after, delta) + + def get_name_matches(self, name: str) -> List[re.Match]: + """Check bad words from passed string (name). Return list of matches.""" + matches = [] + watchlist_patterns = self._get_filterlist_items('filter_token', allowed=False) + for pattern in watchlist_patterns: + if match := re.search(pattern, name, flags=re.IGNORECASE): + matches.append(match) + return matches + + async def check_send_alert(self, member: Member) -> bool: + """When there is less than 3 days after last alert, return `False`, otherwise `True`.""" + if last_alert := await self.name_alerts.get(member.id): + last_alert = datetime.utcfromtimestamp(last_alert) + if datetime.utcnow() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert: + log.trace(f"Last alert was too recent for {member}'s nickname.") + return False + + return True + + async def check_bad_words_in_name(self, member: Member) -> None: + """Send a mod alert every 3 days if a username still matches a watchlist pattern.""" + # Use lock to avoid race conditions + async with self.name_lock: + # Check whether the users display name contains any words in our blacklist + matches = self.get_name_matches(member.display_name) + + if not matches or not await self.check_send_alert(member): + return + + log.info(f"Sending bad nickname alert for '{member.display_name}' ({member.id}).") + + log_string = ( + f"**User:** {format_user(member)}\n" + f"**Display Name:** {escape_markdown(member.display_name)}\n" + f"**Bad Matches:** {', '.join(match.group() for match in matches)}" + ) + + await self.mod_log.send_log_message( + icon_url=Icons.token_removed, + colour=Colours.soft_red, + title="Username filtering alert", + text=log_string, + channel_id=Channels.mod_alerts, + thumbnail=member.avatar_url + ) + + # Update time when alert sent + await self.name_alerts.set(member.id, datetime.utcnow().timestamp()) + + async def filter_eval(self, result: str, msg: Message) -> bool: + """ + Filter the result of an !eval to see if it violates any of our rules, and then respond accordingly. + + Also requires the original message, to check whether to filter and for mod logs. + Returns whether a filter was triggered or not. + """ + filter_triggered = False + # Should we filter this message? + if self._check_filter(msg): + for filter_name, _filter in self.filters.items(): + # Is this specific filter enabled in the config? + # We also do not need to worry about filters that take the full message, + # since all we have is an arbitrary string. + if _filter["enabled"] and _filter["content_only"]: + match = await _filter["function"](result) + + if match: + # If this is a filter (not a watchlist), we set the variable so we know + # that it has been triggered + if _filter["type"] == "filter": + filter_triggered = True + + stats = self._add_stats(filter_name, match, result) + await self._send_log(filter_name, _filter["type"], msg, stats, is_eval=True) + + break # We don't want multiple filters to trigger + + return filter_triggered + + async def _filter_message(self, msg: Message, delta: Optional[int] = None) -> None: + """Filter the input message to see if it violates any of our rules, and then respond accordingly.""" + # Should we filter this message? + if self._check_filter(msg): + for filter_name, _filter in self.filters.items(): + # Is this specific filter enabled in the config? + if _filter["enabled"]: + # Double trigger check for the embeds filter + if filter_name == "watch_rich_embeds": + # If the edit delta is less than 0.001 seconds, then we're probably dealing + # with a double filter trigger. + if delta is not None and delta < 100: + continue + + # Does the filter only need the message content or the full message? + if _filter["content_only"]: + match = await _filter["function"](msg.content) + else: + match = await _filter["function"](msg) + + if match: + is_private = msg.channel.type is discord.ChannelType.private + + # If this is a filter (not a watchlist) and not in a DM, delete the message. + if _filter["type"] == "filter" and not is_private: + try: + # Embeds (can?) trigger both the `on_message` and `on_message_edit` + # event handlers, triggering filtering twice for the same message. + # + # If `on_message`-triggered filtering already deleted the message + # then `on_message_edit`-triggered filtering will raise exception + # since the message no longer exists. + # + # In addition, to avoid sending two notifications to the user, the + # logs, and mod_alert, we return if the message no longer exists. + await msg.delete() + except discord.errors.NotFound: + return + + # Notify the user if the filter specifies + if _filter["user_notification"]: + await self.notify_member(msg.author, _filter["notification_msg"], msg.channel) + + # If the message is classed as offensive, we store it in the site db and + # it will be deleted it after one week. + if _filter["schedule_deletion"] and not is_private: + delete_date = (msg.created_at + OFFENSIVE_MSG_DELETE_TIME).isoformat() + data = { + 'id': msg.id, + 'channel_id': msg.channel.id, + 'delete_date': delete_date + } + + try: + await self.bot.api_client.post('bot/offensive-messages', json=data) + except ResponseCodeError as e: + if e.status == 400 and "already exists" in e.response_json.get("id", [""])[0]: + log.debug(f"Offensive message {msg.id} already exists.") + else: + log.error(f"Offensive message {msg.id} failed to post: {e}") + else: + self.schedule_msg_delete(data) + log.trace(f"Offensive message {msg.id} will be deleted on {delete_date}") + + stats = self._add_stats(filter_name, match, msg.content) + await self._send_log(filter_name, _filter, msg, stats) + + break # We don't want multiple filters to trigger + + async def _send_log( + self, + filter_name: str, + _filter: Dict[str, Any], + msg: discord.Message, + stats: Stats, + *, + is_eval: bool = False, + ) -> None: + """Send a mod log for a triggered filter.""" + if msg.channel.type is discord.ChannelType.private: + channel_str = "via DM" + ping_everyone = False + else: + channel_str = f"in {msg.channel.mention}" + # Allow specific filters to override ping_everyone + ping_everyone = Filter.ping_everyone and _filter.get("ping_everyone", True) + + eval_msg = "using !eval " if is_eval else "" + message = ( + f"The {filter_name} {_filter['type']} was triggered by {format_user(msg.author)} " + f"{channel_str} {eval_msg}with [the following message]({msg.jump_url}):\n\n" + f"{stats.message_content}" + ) + + log.debug(message) + + # Send pretty mod log embed to mod-alerts + await self.mod_log.send_log_message( + icon_url=Icons.filtering, + colour=Colour(Colours.soft_red), + title=f"{_filter['type'].title()} triggered!", + text=message, + thumbnail=msg.author.avatar_url_as(static_format="png"), + channel_id=Channels.mod_alerts, + ping_everyone=ping_everyone, + additional_embeds=stats.additional_embeds, + additional_embeds_msg=stats.additional_embeds_msg + ) + + def _add_stats(self, name: str, match: FilterMatch, content: str) -> Stats: + """Adds relevant statistical information to the relevant filter and increments the bot's stats.""" + # Word and match stats for watch_regex + if name == "watch_regex": + surroundings = match.string[max(match.start() - 10, 0): match.end() + 10] + message_content = ( + f"**Match:** '{match[0]}'\n" + f"**Location:** '...{escape_markdown(surroundings)}...'\n" + f"\n**Original Message:**\n{escape_markdown(content)}" + ) + else: # Use original content + message_content = content + + additional_embeds = None + additional_embeds_msg = None + + self.bot.stats.incr(f"filters.{name}") + + # The function returns True for invalid invites. + # They have no data so additional embeds can't be created for them. + if name == "filter_invites" and match is not True: + additional_embeds = [] + for _, data in match.items(): + embed = discord.Embed(description=( + f"**Members:**\n{data['members']}\n" + f"**Active:**\n{data['active']}" + )) + embed.set_author(name=data["name"]) + embed.set_thumbnail(url=data["icon"]) + embed.set_footer(text=f"Guild ID: {data['id']}") + additional_embeds.append(embed) + additional_embeds_msg = "For the following guild(s):" + + elif name == "watch_rich_embeds": + additional_embeds = match + additional_embeds_msg = "With the following embed(s):" + + return Stats(message_content, additional_embeds, additional_embeds_msg) + + @staticmethod + def _check_filter(msg: Message) -> bool: + """Check whitelists to see if we should filter this message.""" + role_whitelisted = False + + if type(msg.author) is Member: # Only Member has roles, not User. + for role in msg.author.roles: + if role.id in Filter.role_whitelist: + role_whitelisted = True + + return ( + msg.channel.id not in Filter.channel_whitelist # Channel not in whitelist + and not role_whitelisted # Role not in whitelist + and not msg.author.bot # Author not a bot + ) + + async def _has_watch_regex_match(self, text: str) -> Union[bool, re.Match]: + """ + Return True if `text` matches any regex from `word_watchlist` or `token_watchlist` configs. + + `word_watchlist`'s patterns are placed between word boundaries while `token_watchlist` is + matched as-is. Spoilers are expanded, if any, and URLs are ignored. + """ + if SPOILER_RE.search(text): + text = self._expand_spoilers(text) + + # Make sure it's not a URL + if URL_RE.search(text): + return False + + watchlist_patterns = self._get_filterlist_items('filter_token', allowed=False) + for pattern in watchlist_patterns: + match = re.search(pattern, text, flags=re.IGNORECASE) + if match: + return match + + async def _has_urls(self, text: str) -> bool: + """Returns True if the text contains one of the blacklisted URLs from the config file.""" + if not URL_RE.search(text): + return False + + text = text.lower() + domain_blacklist = self._get_filterlist_items("domain_name", allowed=False) + + for url in domain_blacklist: + if url.lower() in text: + return True + + return False + + @staticmethod + async def _has_zalgo(text: str) -> bool: + """ + Returns True if the text contains zalgo characters. + + Zalgo range is \u0300 – \u036F and \u0489. + """ + return bool(ZALGO_RE.search(text)) + + async def _has_invites(self, text: str) -> Union[dict, bool]: + """ + Checks if there's any invites in the text content that aren't in the guild whitelist. + + If any are detected, a dictionary of invite data is returned, with a key per invite. + If none are detected, False is returned. + + Attempts to catch some of common ways to try to cheat the system. + """ + # Remove backslashes to prevent escape character aroundfuckery like + # discord\.gg/gdudes-pony-farm + text = text.replace("\\", "") + + invites = INVITE_RE.findall(text) + invite_data = dict() + for invite in invites: + if invite in invite_data: + continue + + response = await self.bot.http_session.get( + f"{URLs.discord_invite_api}/{invite}", params={"with_counts": "true"} + ) + response = await response.json() + guild = response.get("guild") + if guild is None: + # Lack of a "guild" key in the JSON response indicates either an group DM invite, an + # expired invite, or an invalid invite. The API does not currently differentiate + # between invalid and expired invites + return True + + guild_id = guild.get("id") + guild_invite_whitelist = self._get_filterlist_items("guild_invite", allowed=True) + guild_invite_blacklist = self._get_filterlist_items("guild_invite", allowed=False) + + # Is this invite allowed? + guild_partnered_or_verified = ( + 'PARTNERED' in guild.get("features", []) + or 'VERIFIED' in guild.get("features", []) + ) + invite_not_allowed = ( + guild_id in guild_invite_blacklist # Blacklisted guilds are never permitted. + or guild_id not in guild_invite_whitelist # Whitelisted guilds are always permitted. + and not guild_partnered_or_verified # Otherwise guilds have to be Verified or Partnered. + ) + + if invite_not_allowed: + guild_icon_hash = guild["icon"] + guild_icon = ( + "https://cdn.discordapp.com/icons/" + f"{guild_id}/{guild_icon_hash}.png?size=512" + ) + + invite_data[invite] = { + "name": guild["name"], + "id": guild['id'], + "icon": guild_icon, + "members": response["approximate_member_count"], + "active": response["approximate_presence_count"] + } + + return invite_data if invite_data else False + + @staticmethod + async def _has_rich_embed(msg: Message) -> Union[bool, List[discord.Embed]]: + """Determines if `msg` contains any rich embeds not auto-generated from a URL.""" + if msg.embeds: + for embed in msg.embeds: + if embed.type == "rich": + urls = URL_RE.findall(msg.content) + if not embed.url or embed.url not in urls: + # If `embed.url` does not exist or if `embed.url` is not part of the content + # of the message, it's unlikely to be an auto-generated embed by Discord. + return msg.embeds + else: + log.trace( + "Found a rich embed sent by a regular user account, " + "but it was likely just an automatic URL embed." + ) + return False + return False + + @staticmethod + async def _has_everyone_ping(text: str) -> bool: + """Determines if `msg` contains an @everyone or @here ping outside of a codeblock.""" + # First pass to avoid running re.sub on every message + if not EVERYONE_PING_RE.search(text): + return False + + content_without_codeblocks = CODE_BLOCK_RE.sub("", text) + return bool(EVERYONE_PING_RE.search(content_without_codeblocks)) + + async def notify_member(self, filtered_member: Member, reason: str, channel: TextChannel) -> None: + """ + Notify filtered_member about a moderation action with the reason str. + + First attempts to DM the user, fall back to in-channel notification if user has DMs disabled + """ + try: + await filtered_member.send(reason) + except discord.errors.Forbidden: + await channel.send(f"{filtered_member.mention} {reason}") + + def schedule_msg_delete(self, msg: dict) -> None: + """Delete an offensive message once its deletion date is reached.""" + delete_at = dateutil.parser.isoparse(msg['delete_date']).replace(tzinfo=None) + self.scheduler.schedule_at(delete_at, msg['id'], self.delete_offensive_msg(msg)) + + async def reschedule_offensive_msg_deletion(self) -> None: + """Get all the pending message deletion from the API and reschedule them.""" + await self.bot.wait_until_ready() + response = await self.bot.api_client.get('bot/offensive-messages',) + + now = datetime.utcnow() + + for msg in response: + delete_at = dateutil.parser.isoparse(msg['delete_date']).replace(tzinfo=None) + + if delete_at < now: + await self.delete_offensive_msg(msg) + else: + self.schedule_msg_delete(msg) + + async def delete_offensive_msg(self, msg: Mapping[str, str]) -> None: + """Delete an offensive message, and then delete it from the db.""" + try: + channel = self.bot.get_channel(msg['channel_id']) + if channel: + msg_obj = await channel.fetch_message(msg['id']) + await msg_obj.delete() + except NotFound: + log.info( + f"Tried to delete message {msg['id']}, but the message can't be found " + f"(it has been probably already deleted)." + ) + except HTTPException as e: + log.warning(f"Failed to delete message {msg['id']}: status {e.status}") + + await self.bot.api_client.delete(f'bot/offensive-messages/{msg["id"]}') + log.info(f"Deleted the offensive message with id {msg['id']}.") + + +def setup(bot: Bot) -> None: + """Load the Filtering cog.""" + bot.add_cog(Filtering(bot)) diff --git a/bot/cogs/security.py b/bot/exts/filters/security.py index c680c5e27..c680c5e27 100644 --- a/bot/cogs/security.py +++ b/bot/exts/filters/security.py diff --git a/bot/exts/filters/token_remover.py b/bot/exts/filters/token_remover.py new file mode 100644 index 000000000..bd6a1f97a --- /dev/null +++ b/bot/exts/filters/token_remover.py @@ -0,0 +1,233 @@ +import base64 +import binascii +import logging +import re +import typing as t + +from discord import Colour, Message, NotFound +from discord.ext.commands import Cog + +from bot import utils +from bot.bot import Bot +from bot.constants import Channels, Colours, Event, Icons +from bot.exts.moderation.modlog import ModLog +from bot.utils.messages import format_user + +log = logging.getLogger(__name__) + +LOG_MESSAGE = ( + "Censored a seemingly valid token sent by {author} in {channel}, " + "token was `{user_id}.{timestamp}.{hmac}`" +) +UNKNOWN_USER_LOG_MESSAGE = "Decoded user ID: `{user_id}` (Not present in server)." +KNOWN_USER_LOG_MESSAGE = ( + "Decoded user ID: `{user_id}` **(Present in server)**.\n" + "This matches `{user_name}` and means this is likely a valid **{kind}** token." +) +DELETION_MESSAGE_TEMPLATE = ( + "Hey {mention}! I noticed you posted a seemingly valid Discord API " + "token in your message and have removed your message. " + "This means that your token has been **compromised**. " + "Please change your token **immediately** at: " + "<https://discordapp.com/developers/applications/me>\n\n" + "Feel free to re-post it with the token removed. " + "If you believe this was a mistake, please let us know!" +) +DISCORD_EPOCH = 1_420_070_400 +TOKEN_EPOCH = 1_293_840_000 + +# Three parts delimited by dots: user ID, creation timestamp, HMAC. +# The HMAC isn't parsed further, but it's in the regex to ensure it at least exists in the string. +# Each part only matches base64 URL-safe characters. +# Padding has never been observed, but the padding character '=' is matched just in case. +TOKEN_RE = re.compile(r"([\w\-=]+)\.([\w\-=]+)\.([\w\-=]+)", re.ASCII) + + +class Token(t.NamedTuple): + """A Discord Bot token.""" + + user_id: str + timestamp: str + hmac: str + + +class TokenRemover(Cog): + """Scans messages for potential discord.py bot tokens and removes them.""" + + def __init__(self, bot: Bot): + self.bot = bot + + @property + def mod_log(self) -> ModLog: + """Get currently loaded ModLog cog instance.""" + return self.bot.get_cog("ModLog") + + @Cog.listener() + async def on_message(self, msg: Message) -> None: + """ + Check each message for a string that matches Discord's token pattern. + + See: https://discordapp.com/developers/docs/reference#snowflakes + """ + # Ignore DMs; can't delete messages in there anyway. + if not msg.guild or msg.author.bot: + return + + found_token = self.find_token_in_message(msg) + if found_token: + await self.take_action(msg, found_token) + + @Cog.listener() + async def on_message_edit(self, before: Message, after: Message) -> None: + """ + Check each edit for a string that matches Discord's token pattern. + + See: https://discordapp.com/developers/docs/reference#snowflakes + """ + await self.on_message(after) + + async def take_action(self, msg: Message, found_token: Token) -> None: + """Remove the `msg` containing the `found_token` and send a mod log message.""" + self.mod_log.ignore(Event.message_delete, msg.id) + + try: + await msg.delete() + except NotFound: + log.debug(f"Failed to remove token in message {msg.id}: message already deleted.") + return + + await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention)) + + log_message = self.format_log_message(msg, found_token) + userid_message, mention_everyone = self.format_userid_log_message(msg, found_token) + log.debug(log_message) + + # Send pretty mod log embed to mod-alerts + await self.mod_log.send_log_message( + icon_url=Icons.token_removed, + colour=Colour(Colours.soft_red), + title="Token removed!", + text=log_message + "\n" + userid_message, + thumbnail=msg.author.avatar_url_as(static_format="png"), + channel_id=Channels.mod_alerts, + ping_everyone=mention_everyone, + ) + + self.bot.stats.incr("tokens.removed_tokens") + + @classmethod + def format_userid_log_message(cls, msg: Message, token: Token) -> t.Tuple[str, bool]: + """ + Format the portion of the log message that includes details about the detected user ID. + + If the user is resolved to a member, the format includes the user ID, name, and the + kind of user detected. + + If we resolve to a member and it is not a bot, we also return True to ping everyone. + + Returns a tuple of (log_message, mention_everyone) + """ + user_id = cls.extract_user_id(token.user_id) + user = msg.guild.get_member(user_id) + + if user: + return KNOWN_USER_LOG_MESSAGE.format( + user_id=user_id, + user_name=str(user), + kind="BOT" if user.bot else "USER", + ), not user.bot + else: + return UNKNOWN_USER_LOG_MESSAGE.format(user_id=user_id), False + + @staticmethod + def format_log_message(msg: Message, token: Token) -> str: + """Return the generic portion of the log message to send for `token` being censored in `msg`.""" + return LOG_MESSAGE.format( + author=format_user(msg.author), + channel=msg.channel.mention, + user_id=token.user_id, + timestamp=token.timestamp, + hmac='x' * len(token.hmac), + ) + + @classmethod + def find_token_in_message(cls, msg: Message) -> t.Optional[Token]: + """Return a seemingly valid token found in `msg` or `None` if no token is found.""" + # Use finditer rather than search to guard against method calls prematurely returning the + # token check (e.g. `message.channel.send` also matches our token pattern) + for match in TOKEN_RE.finditer(msg.content): + token = Token(*match.groups()) + if ( + (cls.extract_user_id(token.user_id) is not None) + and cls.is_valid_timestamp(token.timestamp) + and cls.is_maybe_valid_hmac(token.hmac) + ): + # Short-circuit on first match + return token + + # No matching substring + return + + @staticmethod + def extract_user_id(b64_content: str) -> t.Optional[int]: + """Return a user ID integer from part of a potential token, or None if it couldn't be decoded.""" + b64_content = utils.pad_base64(b64_content) + + try: + decoded_bytes = base64.urlsafe_b64decode(b64_content) + string = decoded_bytes.decode('utf-8') + if not (string.isascii() and string.isdigit()): + # This case triggers if there are fancy unicode digits in the base64 encoding, + # that means it's not a valid user id. + return None + return int(string) + except (binascii.Error, ValueError): + return None + + @staticmethod + def is_valid_timestamp(b64_content: str) -> bool: + """ + Return True if `b64_content` decodes to a valid timestamp. + + If the timestamp is greater than the Discord epoch, it's probably valid. + See: https://i.imgur.com/7WdehGn.png + """ + b64_content = utils.pad_base64(b64_content) + + try: + decoded_bytes = base64.urlsafe_b64decode(b64_content) + timestamp = int.from_bytes(decoded_bytes, byteorder="big") + except (binascii.Error, ValueError) as e: + log.debug(f"Failed to decode token timestamp '{b64_content}': {e}") + return False + + # Seems like newer tokens don't need the epoch added, but add anyway since an upper bound + # is not checked. + if timestamp + TOKEN_EPOCH >= DISCORD_EPOCH: + return True + else: + log.debug(f"Invalid token timestamp '{b64_content}': smaller than Discord epoch") + return False + + @staticmethod + def is_maybe_valid_hmac(b64_content: str) -> bool: + """ + Determine if a given HMAC portion of a token is potentially valid. + + If the HMAC has 3 or less characters, it's probably a dummy value like "xxxxxxxxxx", + and thus the token can probably be skipped. + """ + unique = len(set(b64_content.lower())) + if unique <= 3: + log.debug( + f"Considering the HMAC {b64_content} a dummy because it has {unique}" + " case-insensitively unique characters" + ) + return False + else: + return True + + +def setup(bot: Bot) -> None: + """Load the TokenRemover cog.""" + bot.add_cog(TokenRemover(bot)) diff --git a/bot/cogs/webhook_remover.py b/bot/exts/filters/webhook_remover.py index 49692113d..08fe94055 100644 --- a/bot/cogs/webhook_remover.py +++ b/bot/exts/filters/webhook_remover.py @@ -1,14 +1,15 @@ import logging import re -from discord import Colour, Message +from discord import Colour, Message, NotFound from discord.ext.commands import Cog from bot.bot import Bot -from bot.cogs.moderation.modlog import ModLog from bot.constants import Channels, Colours, Event, Icons +from bot.exts.moderation.modlog import ModLog +from bot.utils.messages import format_user -WEBHOOK_URL_RE = re.compile(r"((?:https?://)?discordapp\.com/api/webhooks/\d+/)\S+/?", re.I) +WEBHOOK_URL_RE = re.compile(r"((?:https?://)?discord(?:app)?\.com/api/webhooks/\d+/)\S+/?", re.IGNORECASE) ALERT_MESSAGE_TEMPLATE = ( "{user}, looks like you posted a Discord webhook URL. Therefore, your " @@ -35,12 +36,18 @@ class WebhookRemover(Cog): """Delete `msg` and send a warning that it contained the Discord webhook `redacted_url`.""" # Don't log this, due internal delete, not by user. Will make different entry. self.mod_log.ignore(Event.message_delete, msg.id) - await msg.delete() + + try: + await msg.delete() + except NotFound: + log.debug(f"Failed to remove webhook in message {msg.id}: message already deleted.") + return + await msg.channel.send(ALERT_MESSAGE_TEMPLATE.format(user=msg.author.mention)) message = ( - f"{msg.author} (`{msg.author.id}`) posted a Discord webhook URL " - f"to #{msg.channel}. Webhook URL was `{redacted_url}`" + f"{format_user(msg.author)} posted a Discord webhook URL to {msg.channel.mention}. " + f"Webhook URL was `{redacted_url}`" ) log.debug(message) @@ -54,9 +61,15 @@ class WebhookRemover(Cog): channel_id=Channels.mod_alerts ) + self.bot.stats.incr("tokens.removed_webhooks") + @Cog.listener() async def on_message(self, msg: Message) -> None: """Check if a Discord webhook URL is in `message`.""" + # Ignore DMs; can't delete messages in there anyway. + if not msg.guild or msg.author.bot: + return + matches = WEBHOOK_URL_RE.search(msg.content) if matches: await self.delete_and_respond(msg, matches[1] + "xxx") diff --git a/tests/bot/cogs/sync/__init__.py b/bot/exts/fun/__init__.py index e69de29bb..e69de29bb 100644 --- a/tests/bot/cogs/sync/__init__.py +++ b/bot/exts/fun/__init__.py diff --git a/bot/cogs/duck_pond.py b/bot/exts/fun/duck_pond.py index 1f84a0609..82084ea88 100644 --- a/bot/cogs/duck_pond.py +++ b/bot/exts/fun/duck_pond.py @@ -1,13 +1,16 @@ +import asyncio import logging -from typing import Optional, Union +from typing import Union import discord from discord import Color, Embed, Member, Message, RawReactionActionEvent, User, errors -from discord.ext.commands import Cog +from discord.ext.commands import Cog, Context, command from bot import constants from bot.bot import Bot +from bot.utils.checks import has_any_role from bot.utils.messages import send_attachments +from bot.utils.webhooks import send_webhook log = logging.getLogger(__name__) @@ -18,7 +21,9 @@ class DuckPond(Cog): def __init__(self, bot: Bot): self.bot = bot self.webhook_id = constants.Webhooks.duck_pond + self.webhook = None self.bot.loop.create_task(self.fetch_webhook()) + self.relay_lock = None async def fetch_webhook(self) -> None: """Fetches the webhook object, so we can post to it.""" @@ -47,23 +52,13 @@ class DuckPond(Cog): return True return False - async def send_webhook( - self, - content: Optional[str] = None, - username: Optional[str] = None, - avatar_url: Optional[str] = None, - embed: Optional[Embed] = None, - ) -> None: - """Send a webhook to the duck_pond channel.""" - try: - await self.webhook.send( - content=content, - username=username, - avatar_url=avatar_url, - embed=embed - ) - except discord.HTTPException: - log.exception("Failed to send a message to the Duck Pool webhook") + @staticmethod + def _is_duck_emoji(emoji: Union[str, discord.PartialEmoji, discord.Emoji]) -> bool: + """Check if the emoji is a valid duck emoji.""" + if isinstance(emoji, str): + return emoji == "🦆" + else: + return hasattr(emoji, "name") and emoji.name.startswith("ducky_") async def count_ducks(self, message: Message) -> int: """ @@ -71,33 +66,24 @@ class DuckPond(Cog): Only counts ducks added by staff members. """ - duck_count = 0 - duck_reactors = [] + duck_reactors = set() + # iterate over all reactions for reaction in message.reactions: - async for user in reaction.users(): - - # Is the user a staff member and not already counted as reactor? - if not self.is_staff(user) or user.id in duck_reactors: - continue - - # Is the emoji a duck? - if hasattr(reaction.emoji, "id"): - if reaction.emoji.id in constants.DuckPond.custom_emojis: - duck_count += 1 - duck_reactors.append(user.id) - elif isinstance(reaction.emoji, str): - if reaction.emoji == "🦆": - duck_count += 1 - duck_reactors.append(user.id) - return duck_count + # check if the current reaction is a duck + if not self._is_duck_emoji(reaction.emoji): + continue + + # update the set of reactors with all staff reactors + duck_reactors |= {user.id async for user in reaction.users() if self.is_staff(user)} + + return len(duck_reactors) async def relay_message(self, message: Message) -> None: """Relays the message's content and attachments to the duck pond channel.""" - clean_content = message.clean_content - - if clean_content: - await self.send_webhook( + if message.clean_content: + await send_webhook( + webhook=self.webhook, content=message.clean_content, username=message.author.display_name, avatar_url=message.author.avatar_url @@ -111,26 +97,44 @@ class DuckPond(Cog): description=":x: **This message contained an attachment, but it could not be retrieved**", color=Color.red() ) - await self.send_webhook( + await send_webhook( + webhook=self.webhook, embed=e, username=message.author.display_name, avatar_url=message.author.avatar_url ) except discord.HTTPException: - log.exception(f"Failed to send an attachment to the webhook") + log.exception("Failed to send an attachment to the webhook") - await message.add_reaction("✅") + async def locked_relay(self, message: discord.Message) -> bool: + """Relay a message after obtaining the relay lock.""" + if self.relay_lock is None: + # Lazily load the lock to ensure it's created within the + # appropriate event loop. + self.relay_lock = asyncio.Lock() - @staticmethod - def _payload_has_duckpond_emoji(payload: RawReactionActionEvent) -> bool: + async with self.relay_lock: + # check if the message has a checkmark after acquiring the lock + if await self.has_green_checkmark(message): + return False + + # relay the message + await self.relay_message(message) + + # add a green checkmark to indicate that the message was relayed + await message.add_reaction("✅") + return True + + def _payload_has_duckpond_emoji(self, emoji: discord.PartialEmoji) -> bool: """Test if the RawReactionActionEvent payload contains a duckpond emoji.""" - if payload.emoji.is_custom_emoji(): - if payload.emoji.id in constants.DuckPond.custom_emojis: - return True - elif payload.emoji.name == "🦆": - return True + if emoji.is_unicode_emoji(): + # For unicode PartialEmojis, the `name` attribute is just the string + # representation of the emoji. This is what the helper method + # expects, as unicode emojis show up as just a `str` instance when + # inspecting the reactions attached to a message. + emoji = emoji.name - return False + return self._is_duck_emoji(emoji) @Cog.listener() async def on_raw_reaction_add(self, payload: RawReactionActionEvent) -> None: @@ -141,20 +145,31 @@ class DuckPond(Cog): amount of ducks specified in the config under duck_pond/threshold, it will send the message off to the duck pond. """ + # Ignore other guilds and DMs. + if payload.guild_id != constants.Guild.id: + return + + # Was this reaction issued in a blacklisted channel? + if payload.channel_id in constants.DuckPond.channel_blacklist: + return + # Is the emoji in the reaction a duck? - if not self._payload_has_duckpond_emoji(payload): + if not self._payload_has_duckpond_emoji(payload.emoji): return channel = discord.utils.get(self.bot.get_all_channels(), id=payload.channel_id) + if channel is None: + return + message = await channel.fetch_message(payload.message_id) member = discord.utils.get(message.guild.members, id=payload.user_id) - # Is the member a human and a staff member? - if not self.is_staff(member) or member.bot: + # Was the message sent by a human staff member? + if not self.is_staff(message.author) or message.author.bot: return - # Does the message already have a green checkmark? - if await self.has_green_checkmark(message): + # Is the reactor a human staff member? + if not self.is_staff(member) or member.bot: return # Time to count our ducks! @@ -162,12 +177,18 @@ class DuckPond(Cog): # If we've got more than the required amount of ducks, send the message to the duck_pond. if duck_count >= constants.DuckPond.threshold: - await self.relay_message(message) + await self.locked_relay(message) @Cog.listener() async def on_raw_reaction_remove(self, payload: RawReactionActionEvent) -> None: """Ensure that people don't remove the green checkmark from duck ponded messages.""" + # Ignore other guilds and DMs. + if payload.guild_id != constants.Guild.id: + return + channel = discord.utils.get(self.bot.get_all_channels(), id=payload.channel_id) + if channel is None: + return # Prevent the green checkmark from being removed if payload.emoji.name == "✅": @@ -176,6 +197,15 @@ class DuckPond(Cog): if duck_count >= constants.DuckPond.threshold: await message.add_reaction("✅") + @command(name="duckify", aliases=("duckpond", "pondify")) + @has_any_role(constants.Roles.admins) + async def duckify(self, ctx: Context, message: discord.Message) -> None: + """Relay a message to the duckpond, no ducks required!""" + if await self.locked_relay(message): + await ctx.message.add_reaction("🦆") + else: + await ctx.message.add_reaction("❌") + def setup(bot: Bot) -> None: """Load the DuckPond cog.""" diff --git a/bot/cogs/off_topic_names.py b/bot/exts/fun/off_topic_names.py index fd386858e..7fc93b88c 100644 --- a/bot/cogs/off_topic_names.py +++ b/bot/exts/fun/off_topic_names.py @@ -3,47 +3,19 @@ import logging from datetime import datetime, timedelta from discord import Colour, Embed -from discord.ext.commands import BadArgument, Cog, Context, Converter, group +from discord.ext.commands import Cog, Context, group, has_any_role from discord.utils import sleep_until from bot.api import ResponseCodeError from bot.bot import Bot from bot.constants import Channels, MODERATION_ROLES -from bot.decorators import with_role +from bot.converters import OffTopicName from bot.pagination import LinePaginator - CHANNELS = (Channels.off_topic_0, Channels.off_topic_1, Channels.off_topic_2) log = logging.getLogger(__name__) -class OffTopicName(Converter): - """A converter that ensures an added off-topic name is valid.""" - - @staticmethod - async def convert(ctx: Context, argument: str) -> str: - """Attempt to replace any invalid characters with their approximate Unicode equivalent.""" - allowed_characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ!?'`-" - - # Chain multiple words to a single one - argument = "-".join(argument.split()) - - if not (2 <= len(argument) <= 96): - raise BadArgument("Channel name must be between 2 and 96 chars long") - - elif not all(c.isalnum() or c in allowed_characters for c in argument): - raise BadArgument( - "Channel name must only consist of " - "alphanumeric characters, minus signs or apostrophes." - ) - - # Replace invalid characters with unicode alternatives. - table = str.maketrans( - allowed_characters, '𝖠𝖡𝖢𝖣𝖤𝖥𝖦𝖧𝖨𝖩𝖪𝖫𝖬𝖭𝖮𝖯𝖰𝖱𝖲𝖳𝖴𝖵𝖶𝖷𝖸𝖹ǃ？’’-' - ) - return argument.translate(table) - - async def update_names(bot: Bot) -> None: """Background updater task that performs the daily channel name update.""" while True: @@ -93,13 +65,13 @@ class OffTopicNames(Cog): self.updater_task = self.bot.loop.create_task(coro) @group(name='otname', aliases=('otnames', 'otn'), invoke_without_command=True) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def otname_group(self, ctx: Context) -> None: """Add or list items from the off-topic channel name rotation.""" - await ctx.send_help("otname") + await ctx.send_help(ctx.command) @otname_group.command(name='add', aliases=('a',)) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def add_command(self, ctx: Context, *, name: OffTopicName) -> None: """ Adds a new off-topic name to the rotation. @@ -122,7 +94,7 @@ class OffTopicNames(Cog): await self._add_name(ctx, name) @otname_group.command(name='forceadd', aliases=('fa',)) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def force_add_command(self, ctx: Context, *, name: OffTopicName) -> None: """Forcefully adds a new off-topic name to the rotation.""" await self._add_name(ctx, name) @@ -135,7 +107,7 @@ class OffTopicNames(Cog): await ctx.send(f":ok_hand: Added `{name}` to the names list.") @otname_group.command(name='delete', aliases=('remove', 'rm', 'del', 'd')) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def delete_command(self, ctx: Context, *, name: OffTopicName) -> None: """Removes a off-topic name from the rotation.""" await self.bot.api_client.delete(f'bot/off-topic-channel-names/{name}') @@ -144,7 +116,7 @@ class OffTopicNames(Cog): await ctx.send(f":ok_hand: Removed `{name}` from the names list.") @otname_group.command(name='list', aliases=('l',)) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def list_command(self, ctx: Context) -> None: """ Lists all currently known off-topic channel names in a paginator. @@ -164,7 +136,7 @@ class OffTopicNames(Cog): await ctx.send(embed=embed) @otname_group.command(name='search', aliases=('s',)) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def search_command(self, ctx: Context, *, query: OffTopicName) -> None: """Search for an off-topic name.""" result = await self.bot.api_client.get('bot/off-topic-channel-names') diff --git a/bot/exts/help_channels.py b/bot/exts/help_channels.py new file mode 100644 index 000000000..9e33a6aba --- /dev/null +++ b/bot/exts/help_channels.py @@ -0,0 +1,943 @@ +import asyncio +import json +import logging +import random +import typing as t +from collections import deque +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import discord +import discord.abc +from async_rediscache import RedisCache +from discord.ext import commands + +from bot import constants +from bot.bot import Bot +from bot.utils.scheduling import Scheduler + +log = logging.getLogger(__name__) + +ASKING_GUIDE_URL = "https://pythondiscord.com/pages/asking-good-questions/" +MAX_CHANNELS_PER_CATEGORY = 50 +EXCLUDED_CHANNELS = (constants.Channels.how_to_get_help, constants.Channels.cooldown) + +HELP_CHANNEL_TOPIC = """ +This is a Python help channel. You can claim your own help channel in the Python Help: Available category. +""" + +AVAILABLE_MSG = f""" +This help channel is now **available**, which means that you can claim it by simply typing your \ +question into it. Once claimed, the channel will move into the **Python Help: Occupied** category, \ +and will be yours until it has been inactive for {constants.HelpChannels.idle_minutes} minutes or \ +is closed manually with `!close`. When that happens, it will be set to **dormant** and moved into \ +the **Help: Dormant** category. + +Try to write the best question you can by providing a detailed description and telling us what \ +you've tried already. For more information on asking a good question, \ +check out our guide on **[asking good questions]({ASKING_GUIDE_URL})**. +""" + +DORMANT_MSG = f""" +This help channel has been marked as **dormant**, and has been moved into the **Help: Dormant** \ +category at the bottom of the channel list. It is no longer possible to send messages in this \ +channel until it becomes available again. + +If your question wasn't answered yet, you can claim a new help channel from the \ +**Help: Available** category by simply asking your question again. Consider rephrasing the \ +question to maximize your chance of getting a good answer. If you're not sure how, have a look \ +through our guide for **[asking a good question]({ASKING_GUIDE_URL})**. +""" + +CoroutineFunc = t.Callable[..., t.Coroutine] + + +class HelpChannels(commands.Cog): + """ + Manage the help channel system of the guild. + + The system is based on a 3-category system: + + Available Category + + * Contains channels which are ready to be occupied by someone who needs help + * Will always contain `constants.HelpChannels.max_available` channels; refilled automatically + from the pool of dormant channels + * Prioritise using the channels which have been dormant for the longest amount of time + * If there are no more dormant channels, the bot will automatically create a new one + * If there are no dormant channels to move, helpers will be notified (see `notify()`) + * When a channel becomes available, the dormant embed will be edited to show `AVAILABLE_MSG` + * User can only claim a channel at an interval `constants.HelpChannels.claim_minutes` + * To keep track of cooldowns, user which claimed a channel will have a temporary role + + In Use Category + + * Contains all channels which are occupied by someone needing help + * Channel moves to dormant category after `constants.HelpChannels.idle_minutes` of being idle + * Command can prematurely mark a channel as dormant + * Channel claimant is allowed to use the command + * Allowed roles for the command are configurable with `constants.HelpChannels.cmd_whitelist` + * When a channel becomes dormant, an embed with `DORMANT_MSG` will be sent + + Dormant Category + + * Contains channels which aren't in use + * Channels are used to refill the Available category + + Help channels are named after the chemical elements in `bot/resources/elements.json`. + """ + + # This cache tracks which channels are claimed by which members. + # RedisCache[discord.TextChannel.id, t.Union[discord.User.id, discord.Member.id]] + help_channel_claimants = RedisCache() + + # This cache maps a help channel to whether it has had any + # activity other than the original claimant. True being no other + # activity and False being other activity. + # RedisCache[discord.TextChannel.id, bool] + unanswered = RedisCache() + + # This dictionary maps a help channel to the time it was claimed + # RedisCache[discord.TextChannel.id, UtcPosixTimestamp] + claim_times = RedisCache() + + # This cache maps a help channel to original question message in same channel. + # RedisCache[discord.TextChannel.id, discord.Message.id] + question_messages = RedisCache() + + def __init__(self, bot: Bot): + self.bot = bot + self.scheduler = Scheduler(self.__class__.__name__) + + # Categories + self.available_category: discord.CategoryChannel = None + self.in_use_category: discord.CategoryChannel = None + self.dormant_category: discord.CategoryChannel = None + + # Queues + self.channel_queue: asyncio.Queue[discord.TextChannel] = None + self.name_queue: t.Deque[str] = None + + self.name_positions = self.get_names() + self.last_notification: t.Optional[datetime] = None + + # Asyncio stuff + self.queue_tasks: t.List[asyncio.Task] = [] + self.ready = asyncio.Event() + self.on_message_lock = asyncio.Lock() + self.init_task = self.bot.loop.create_task(self.init_cog()) + + def cog_unload(self) -> None: + """Cancel the init task and scheduled tasks when the cog unloads.""" + log.trace("Cog unload: cancelling the init_cog task") + self.init_task.cancel() + + log.trace("Cog unload: cancelling the channel queue tasks") + for task in self.queue_tasks: + task.cancel() + + self.scheduler.cancel_all() + + def create_channel_queue(self) -> asyncio.Queue: + """ + Return a queue of dormant channels to use for getting the next available channel. + + The channels are added to the queue in a random order. + """ + log.trace("Creating the channel queue.") + + channels = list(self.get_category_channels(self.dormant_category)) + random.shuffle(channels) + + log.trace("Populating the channel queue with channels.") + queue = asyncio.Queue() + for channel in channels: + queue.put_nowait(channel) + + return queue + + async def create_dormant(self) -> t.Optional[discord.TextChannel]: + """ + Create and return a new channel in the Dormant category. + + The new channel will sync its permission overwrites with the category. + + Return None if no more channel names are available. + """ + log.trace("Getting a name for a new dormant channel.") + + try: + name = self.name_queue.popleft() + except IndexError: + log.debug("No more names available for new dormant channels.") + return None + + log.debug(f"Creating a new dormant channel named {name}.") + return await self.dormant_category.create_text_channel(name, topic=HELP_CHANNEL_TOPIC) + + def create_name_queue(self) -> deque: + """Return a queue of element names to use for creating new channels.""" + log.trace("Creating the chemical element name queue.") + + used_names = self.get_used_names() + + log.trace("Determining the available names.") + available_names = (name for name in self.name_positions if name not in used_names) + + log.trace("Populating the name queue with names.") + return deque(available_names) + + async def dormant_check(self, ctx: commands.Context) -> bool: + """Return True if the user is the help channel claimant or passes the role check.""" + if await self.help_channel_claimants.get(ctx.channel.id) == ctx.author.id: + log.trace(f"{ctx.author} is the help channel claimant, passing the check for dormant.") + self.bot.stats.incr("help.dormant_invoke.claimant") + return True + + log.trace(f"{ctx.author} is not the help channel claimant, checking roles.") + has_role = await commands.has_any_role(*constants.HelpChannels.cmd_whitelist).predicate(ctx) + + if has_role: + self.bot.stats.incr("help.dormant_invoke.staff") + + return has_role + + @commands.command(name="close", aliases=["dormant", "solved"], enabled=False) + async def close_command(self, ctx: commands.Context) -> None: + """ + Make the current in-use help channel dormant. + + Make the channel dormant if the user passes the `dormant_check`, + delete the message that invoked this, + and reset the send permissions cooldown for the user who started the session. + """ + log.trace("close command invoked; checking if the channel is in-use.") + if ctx.channel.category == self.in_use_category: + if await self.dormant_check(ctx): + await self.remove_cooldown_role(ctx.author) + + # Ignore missing task when cooldown has passed but the channel still isn't dormant. + if ctx.author.id in self.scheduler: + self.scheduler.cancel(ctx.author.id) + + await self.move_to_dormant(ctx.channel, "command") + self.scheduler.cancel(ctx.channel.id) + else: + log.debug(f"{ctx.author} invoked command 'dormant' outside an in-use help channel") + + async def get_available_candidate(self) -> discord.TextChannel: + """ + Return a dormant channel to turn into an available channel. + + If no channel is available, wait indefinitely until one becomes available. + """ + log.trace("Getting an available channel candidate.") + + try: + channel = self.channel_queue.get_nowait() + except asyncio.QueueEmpty: + log.info("No candidate channels in the queue; creating a new channel.") + channel = await self.create_dormant() + + if not channel: + log.info("Couldn't create a candidate channel; waiting to get one from the queue.") + await self.notify() + channel = await self.wait_for_dormant_channel() + + return channel + + @staticmethod + def get_clean_channel_name(channel: discord.TextChannel) -> str: + """Return a clean channel name without status emojis prefix.""" + prefix = constants.HelpChannels.name_prefix + try: + # Try to remove the status prefix using the index of the channel prefix + name = channel.name[channel.name.index(prefix):] + log.trace(f"The clean name for `{channel}` is `{name}`") + except ValueError: + # If, for some reason, the channel name does not contain "help-" fall back gracefully + log.info(f"Can't get clean name because `{channel}` isn't prefixed by `{prefix}`.") + name = channel.name + + return name + + @staticmethod + def is_excluded_channel(channel: discord.abc.GuildChannel) -> bool: + """Check if a channel should be excluded from the help channel system.""" + return not isinstance(channel, discord.TextChannel) or channel.id in EXCLUDED_CHANNELS + + def get_category_channels(self, category: discord.CategoryChannel) -> t.Iterable[discord.TextChannel]: + """Yield the text channels of the `category` in an unsorted manner.""" + log.trace(f"Getting text channels in the category '{category}' ({category.id}).") + + # This is faster than using category.channels because the latter sorts them. + for channel in self.bot.get_guild(constants.Guild.id).channels: + if channel.category_id == category.id and not self.is_excluded_channel(channel): + yield channel + + async def get_in_use_time(self, channel_id: int) -> t.Optional[timedelta]: + """Return the duration `channel_id` has been in use. Return None if it's not in use.""" + log.trace(f"Calculating in use time for channel {channel_id}.") + + claimed_timestamp = await self.claim_times.get(channel_id) + if claimed_timestamp: + claimed = datetime.utcfromtimestamp(claimed_timestamp) + return datetime.utcnow() - claimed + + @staticmethod + def get_names() -> t.List[str]: + """ + Return a truncated list of prefixed element names. + + The amount of names is configured with `HelpChannels.max_total_channels`. + The prefix is configured with `HelpChannels.name_prefix`. + """ + count = constants.HelpChannels.max_total_channels + prefix = constants.HelpChannels.name_prefix + + log.trace(f"Getting the first {count} element names from JSON.") + + with Path("bot/resources/elements.json").open(encoding="utf-8") as elements_file: + all_names = json.load(elements_file) + + if prefix: + return [prefix + name for name in all_names[:count]] + else: + return all_names[:count] + + def get_used_names(self) -> t.Set[str]: + """Return channel names which are already being used.""" + log.trace("Getting channel names which are already being used.") + + names = set() + for cat in (self.available_category, self.in_use_category, self.dormant_category): + for channel in self.get_category_channels(cat): + names.add(self.get_clean_channel_name(channel)) + + if len(names) > MAX_CHANNELS_PER_CATEGORY: + log.warning( + f"Too many help channels ({len(names)}) already exist! " + f"Discord only supports {MAX_CHANNELS_PER_CATEGORY} in a category." + ) + + log.trace(f"Got {len(names)} used names: {names}") + return names + + @classmethod + async def get_idle_time(cls, channel: discord.TextChannel) -> t.Optional[int]: + """ + Return the time elapsed, in seconds, since the last message sent in the `channel`. + + Return None if the channel has no messages. + """ + log.trace(f"Getting the idle time for #{channel} ({channel.id}).") + + msg = await cls.get_last_message(channel) + if not msg: + log.debug(f"No idle time available; #{channel} ({channel.id}) has no messages.") + return None + + idle_time = (datetime.utcnow() - msg.created_at).seconds + + log.trace(f"#{channel} ({channel.id}) has been idle for {idle_time} seconds.") + return idle_time + + @staticmethod + async def get_last_message(channel: discord.TextChannel) -> t.Optional[discord.Message]: + """Return the last message sent in the channel or None if no messages exist.""" + log.trace(f"Getting the last message in #{channel} ({channel.id}).") + + try: + return await channel.history(limit=1).next() # noqa: B305 + except discord.NoMoreItems: + log.debug(f"No last message available; #{channel} ({channel.id}) has no messages.") + return None + + async def init_available(self) -> None: + """Initialise the Available category with channels.""" + log.trace("Initialising the Available category with channels.") + + channels = list(self.get_category_channels(self.available_category)) + missing = constants.HelpChannels.max_available - len(channels) + + # If we've got less than `max_available` channel available, we should add some. + if missing > 0: + log.trace(f"Moving {missing} missing channels to the Available category.") + for _ in range(missing): + await self.move_to_available() + + # If for some reason we have more than `max_available` channels available, + # we should move the superfluous ones over to dormant. + elif missing < 0: + log.trace(f"Moving {abs(missing)} superfluous available channels over to the Dormant category.") + for channel in channels[:abs(missing)]: + await self.move_to_dormant(channel, "auto") + + async def init_categories(self) -> None: + """Get the help category objects. Remove the cog if retrieval fails.""" + log.trace("Getting the CategoryChannel objects for the help categories.") + + try: + self.available_category = await self.try_get_channel( + constants.Categories.help_available + ) + self.in_use_category = await self.try_get_channel(constants.Categories.help_in_use) + self.dormant_category = await self.try_get_channel(constants.Categories.help_dormant) + except discord.HTTPException: + log.exception("Failed to get a category; cog will be removed") + self.bot.remove_cog(self.qualified_name) + + async def init_cog(self) -> None: + """Initialise the help channel system.""" + log.trace("Waiting for the guild to be available before initialisation.") + await self.bot.wait_until_guild_available() + + log.trace("Initialising the cog.") + await self.init_categories() + await self.check_cooldowns() + + self.channel_queue = self.create_channel_queue() + self.name_queue = self.create_name_queue() + + log.trace("Moving or rescheduling in-use channels.") + for channel in self.get_category_channels(self.in_use_category): + await self.move_idle_channel(channel, has_task=False) + + # Prevent the command from being used until ready. + # The ready event wasn't used because channels could change categories between the time + # the command is invoked and the cog is ready (e.g. if move_idle_channel wasn't called yet). + # This may confuse users. So would potentially long delays for the cog to become ready. + self.close_command.enabled = True + + await self.init_available() + + log.info("Cog is ready!") + self.ready.set() + + self.report_stats() + + def report_stats(self) -> None: + """Report the channel count stats.""" + total_in_use = sum(1 for _ in self.get_category_channels(self.in_use_category)) + total_available = sum(1 for _ in self.get_category_channels(self.available_category)) + total_dormant = sum(1 for _ in self.get_category_channels(self.dormant_category)) + + self.bot.stats.gauge("help.total.in_use", total_in_use) + self.bot.stats.gauge("help.total.available", total_available) + self.bot.stats.gauge("help.total.dormant", total_dormant) + + @staticmethod + def is_claimant(member: discord.Member) -> bool: + """Return True if `member` has the 'Help Cooldown' role.""" + return any(constants.Roles.help_cooldown == role.id for role in member.roles) + + def match_bot_embed(self, message: t.Optional[discord.Message], description: str) -> bool: + """Return `True` if the bot's `message`'s embed description matches `description`.""" + if not message or not message.embeds: + return False + + bot_msg_desc = message.embeds[0].description + if bot_msg_desc is discord.Embed.Empty: + log.trace("Last message was a bot embed but it was empty.") + return False + return message.author == self.bot.user and bot_msg_desc.strip() == description.strip() + + @staticmethod + def is_in_category(channel: discord.TextChannel, category_id: int) -> bool: + """Return True if `channel` is within a category with `category_id`.""" + actual_category = getattr(channel, "category", None) + return actual_category is not None and actual_category.id == category_id + + async def move_idle_channel(self, channel: discord.TextChannel, has_task: bool = True) -> None: + """ + Make the `channel` dormant if idle or schedule the move if still active. + + If `has_task` is True and rescheduling is required, the extant task to make the channel + dormant will first be cancelled. + """ + log.trace(f"Handling in-use channel #{channel} ({channel.id}).") + + if not await self.is_empty(channel): + idle_seconds = constants.HelpChannels.idle_minutes * 60 + else: + idle_seconds = constants.HelpChannels.deleted_idle_minutes * 60 + + time_elapsed = await self.get_idle_time(channel) + + if time_elapsed is None or time_elapsed >= idle_seconds: + log.info( + f"#{channel} ({channel.id}) is idle longer than {idle_seconds} seconds " + f"and will be made dormant." + ) + + await self.move_to_dormant(channel, "auto") + else: + # Cancel the existing task, if any. + if has_task: + self.scheduler.cancel(channel.id) + + delay = idle_seconds - time_elapsed + log.info( + f"#{channel} ({channel.id}) is still active; " + f"scheduling it to be moved after {delay} seconds." + ) + + self.scheduler.schedule_later(delay, channel.id, self.move_idle_channel(channel)) + + async def move_to_bottom_position(self, channel: discord.TextChannel, category_id: int, **options) -> None: + """ + Move the `channel` to the bottom position of `category` and edit channel attributes. + + To ensure "stable sorting", we use the `bulk_channel_update` endpoint and provide the current + positions of the other channels in the category as-is. This should make sure that the channel + really ends up at the bottom of the category. + + If `options` are provided, the channel will be edited after the move is completed. This is the + same order of operations that `discord.TextChannel.edit` uses. For information on available + options, see the documention on `discord.TextChannel.edit`. While possible, position-related + options should be avoided, as it may interfere with the category move we perform. + """ + # Get a fresh copy of the category from the bot to avoid the cache mismatch issue we had. + category = await self.try_get_channel(category_id) + + payload = [{"id": c.id, "position": c.position} for c in category.channels] + + # Calculate the bottom position based on the current highest position in the category. If the + # category is currently empty, we simply use the current position of the channel to avoid making + # unnecessary changes to positions in the guild. + bottom_position = payload[-1]["position"] + 1 if payload else channel.position + + payload.append( + { + "id": channel.id, + "position": bottom_position, + "parent_id": category.id, + "lock_permissions": True, + } + ) + + # We use d.py's method to ensure our request is processed by d.py's rate limit manager + await self.bot.http.bulk_channel_update(category.guild.id, payload) + + # Now that the channel is moved, we can edit the other attributes + if options: + await channel.edit(**options) + + async def move_to_available(self) -> None: + """Make a channel available.""" + log.trace("Making a channel available.") + + channel = await self.get_available_candidate() + log.info(f"Making #{channel} ({channel.id}) available.") + + await self.send_available_message(channel) + + log.trace(f"Moving #{channel} ({channel.id}) to the Available category.") + + await self.move_to_bottom_position( + channel=channel, + category_id=constants.Categories.help_available, + ) + + self.report_stats() + + async def move_to_dormant(self, channel: discord.TextChannel, caller: str) -> None: + """ + Make the `channel` dormant. + + A caller argument is provided for metrics. + """ + log.info(f"Moving #{channel} ({channel.id}) to the Dormant category.") + + await self.help_channel_claimants.delete(channel.id) + await self.move_to_bottom_position( + channel=channel, + category_id=constants.Categories.help_dormant, + ) + + self.bot.stats.incr(f"help.dormant_calls.{caller}") + + in_use_time = await self.get_in_use_time(channel.id) + if in_use_time: + self.bot.stats.timing("help.in_use_time", in_use_time) + + unanswered = await self.unanswered.get(channel.id) + if unanswered: + self.bot.stats.incr("help.sessions.unanswered") + elif unanswered is not None: + self.bot.stats.incr("help.sessions.answered") + + log.trace(f"Position of #{channel} ({channel.id}) is actually {channel.position}.") + log.trace(f"Sending dormant message for #{channel} ({channel.id}).") + embed = discord.Embed(description=DORMANT_MSG) + await channel.send(embed=embed) + + await self.unpin(channel) + + log.trace(f"Pushing #{channel} ({channel.id}) into the channel queue.") + self.channel_queue.put_nowait(channel) + self.report_stats() + + async def move_to_in_use(self, channel: discord.TextChannel) -> None: + """Make a channel in-use and schedule it to be made dormant.""" + log.info(f"Moving #{channel} ({channel.id}) to the In Use category.") + + await self.move_to_bottom_position( + channel=channel, + category_id=constants.Categories.help_in_use, + ) + + timeout = constants.HelpChannels.idle_minutes * 60 + + log.trace(f"Scheduling #{channel} ({channel.id}) to become dormant in {timeout} sec.") + self.scheduler.schedule_later(timeout, channel.id, self.move_idle_channel(channel)) + self.report_stats() + + async def notify(self) -> None: + """ + Send a message notifying about a lack of available help channels. + + Configuration: + + * `HelpChannels.notify` - toggle notifications + * `HelpChannels.notify_channel` - destination channel for notifications + * `HelpChannels.notify_minutes` - minimum interval between notifications + * `HelpChannels.notify_roles` - roles mentioned in notifications + """ + if not constants.HelpChannels.notify: + return + + log.trace("Notifying about lack of channels.") + + if self.last_notification: + elapsed = (datetime.utcnow() - self.last_notification).seconds + minimum_interval = constants.HelpChannels.notify_minutes * 60 + should_send = elapsed >= minimum_interval + else: + should_send = True + + if not should_send: + log.trace("Notification not sent because it's too recent since the previous one.") + return + + try: + log.trace("Sending notification message.") + + channel = self.bot.get_channel(constants.HelpChannels.notify_channel) + mentions = " ".join(f"<@&{role}>" for role in constants.HelpChannels.notify_roles) + allowed_roles = [discord.Object(id_) for id_ in constants.HelpChannels.notify_roles] + + message = await channel.send( + f"{mentions} A new available help channel is needed but there " + f"are no more dormant ones. Consider freeing up some in-use channels manually by " + f"using the `{constants.Bot.prefix}dormant` command within the channels.", + allowed_mentions=discord.AllowedMentions(everyone=False, roles=allowed_roles) + ) + + self.bot.stats.incr("help.out_of_channel_alerts") + + self.last_notification = message.created_at + except Exception: + # Handle it here cause this feature isn't critical for the functionality of the system. + log.exception("Failed to send notification about lack of dormant channels!") + + async def check_for_answer(self, message: discord.Message) -> None: + """Checks for whether new content in a help channel comes from non-claimants.""" + channel = message.channel + + # Confirm the channel is an in use help channel + if self.is_in_category(channel, constants.Categories.help_in_use): + log.trace(f"Checking if #{channel} ({channel.id}) has been answered.") + + # Check if there is an entry in unanswered + if await self.unanswered.contains(channel.id): + claimant_id = await self.help_channel_claimants.get(channel.id) + if not claimant_id: + # The mapping for this channel doesn't exist, we can't do anything. + return + + # Check the message did not come from the claimant + if claimant_id != message.author.id: + # Mark the channel as answered + await self.unanswered.set(channel.id, False) + + @commands.Cog.listener() + async def on_message(self, message: discord.Message) -> None: + """Move an available channel to the In Use category and replace it with a dormant one.""" + if message.author.bot: + return # Ignore messages sent by bots. + + channel = message.channel + + await self.check_for_answer(message) + + if not self.is_in_category(channel, constants.Categories.help_available) or self.is_excluded_channel(channel): + return # Ignore messages outside the Available category or in excluded channels. + + log.trace("Waiting for the cog to be ready before processing messages.") + await self.ready.wait() + + log.trace("Acquiring lock to prevent a channel from being processed twice...") + async with self.on_message_lock: + log.trace(f"on_message lock acquired for {message.id}.") + + if not self.is_in_category(channel, constants.Categories.help_available): + log.debug( + f"Message {message.id} will not make #{channel} ({channel.id}) in-use " + f"because another message in the channel already triggered that." + ) + return + + log.info(f"Channel #{channel} was claimed by `{message.author.id}`.") + await self.move_to_in_use(channel) + await self.revoke_send_permissions(message.author) + + await self.pin(message) + + # Add user with channel for dormant check. + await self.help_channel_claimants.set(channel.id, message.author.id) + + self.bot.stats.incr("help.claimed") + + # Must use a timezone-aware datetime to ensure a correct POSIX timestamp. + timestamp = datetime.now(timezone.utc).timestamp() + await self.claim_times.set(channel.id, timestamp) + + await self.unanswered.set(channel.id, True) + + log.trace(f"Releasing on_message lock for {message.id}.") + + # Move a dormant channel to the Available category to fill in the gap. + # This is done last and outside the lock because it may wait indefinitely for a channel to + # be put in the queue. + await self.move_to_available() + + @commands.Cog.listener() + async def on_message_delete(self, msg: discord.Message) -> None: + """ + Reschedule an in-use channel to become dormant sooner if the channel is empty. + + The new time for the dormant task is configured with `HelpChannels.deleted_idle_minutes`. + """ + if not self.is_in_category(msg.channel, constants.Categories.help_in_use): + return + + if not await self.is_empty(msg.channel): + return + + log.info(f"Claimant of #{msg.channel} ({msg.author}) deleted message, channel is empty now. Rescheduling task.") + + # Cancel existing dormant task before scheduling new. + self.scheduler.cancel(msg.channel.id) + + delay = constants.HelpChannels.deleted_idle_minutes * 60 + self.scheduler.schedule_later(delay, msg.channel.id, self.move_idle_channel(msg.channel)) + + async def is_empty(self, channel: discord.TextChannel) -> bool: + """Return True if there's an AVAILABLE_MSG and the messages leading up are bot messages.""" + log.trace(f"Checking if #{channel} ({channel.id}) is empty.") + + # A limit of 100 results in a single API call. + # If AVAILABLE_MSG isn't found within 100 messages, then assume the channel is not empty. + # Not gonna do an extensive search for it cause it's too expensive. + async for msg in channel.history(limit=100): + if not msg.author.bot: + log.trace(f"#{channel} ({channel.id}) has a non-bot message.") + return False + + if self.match_bot_embed(msg, AVAILABLE_MSG): + log.trace(f"#{channel} ({channel.id}) has the available message embed.") + return True + + return False + + async def check_cooldowns(self) -> None: + """Remove expired cooldowns and re-schedule active ones.""" + log.trace("Checking all cooldowns to remove or re-schedule them.") + guild = self.bot.get_guild(constants.Guild.id) + cooldown = constants.HelpChannels.claim_minutes * 60 + + for channel_id, member_id in await self.help_channel_claimants.items(): + member = guild.get_member(member_id) + if not member: + continue # Member probably left the guild. + + in_use_time = await self.get_in_use_time(channel_id) + + if not in_use_time or in_use_time.seconds > cooldown: + # Remove the role if no claim time could be retrieved or if the cooldown expired. + # Since the channel is in the claimants cache, it is definitely strange for a time + # to not exist. However, it isn't a reason to keep the user stuck with a cooldown. + await self.remove_cooldown_role(member) + else: + # The member is still on a cooldown; re-schedule it for the remaining time. + delay = cooldown - in_use_time.seconds + self.scheduler.schedule_later(delay, member.id, self.remove_cooldown_role(member)) + + async def add_cooldown_role(self, member: discord.Member) -> None: + """Add the help cooldown role to `member`.""" + log.trace(f"Adding cooldown role for {member} ({member.id}).") + await self._change_cooldown_role(member, member.add_roles) + + async def remove_cooldown_role(self, member: discord.Member) -> None: + """Remove the help cooldown role from `member`.""" + log.trace(f"Removing cooldown role for {member} ({member.id}).") + await self._change_cooldown_role(member, member.remove_roles) + + async def _change_cooldown_role(self, member: discord.Member, coro_func: CoroutineFunc) -> None: + """ + Change `member`'s cooldown role via awaiting `coro_func` and handle errors. + + `coro_func` is intended to be `discord.Member.add_roles` or `discord.Member.remove_roles`. + """ + guild = self.bot.get_guild(constants.Guild.id) + role = guild.get_role(constants.Roles.help_cooldown) + if role is None: + log.warning(f"Help cooldown role ({constants.Roles.help_cooldown}) could not be found!") + return + + try: + await coro_func(role) + except discord.NotFound: + log.debug(f"Failed to change role for {member} ({member.id}): member not found") + except discord.Forbidden: + log.debug( + f"Forbidden to change role for {member} ({member.id}); " + f"possibly due to role hierarchy" + ) + except discord.HTTPException as e: + log.error(f"Failed to change role for {member} ({member.id}): {e.status} {e.code}") + + async def revoke_send_permissions(self, member: discord.Member) -> None: + """ + Disallow `member` to send messages in the Available category for a certain time. + + The time until permissions are reinstated can be configured with + `HelpChannels.claim_minutes`. + """ + log.trace( + f"Revoking {member}'s ({member.id}) send message permissions in the Available category." + ) + + await self.add_cooldown_role(member) + + # Cancel the existing task, if any. + # Would mean the user somehow bypassed the lack of permissions (e.g. user is guild owner). + if member.id in self.scheduler: + self.scheduler.cancel(member.id) + + delay = constants.HelpChannels.claim_minutes * 60 + self.scheduler.schedule_later(delay, member.id, self.remove_cooldown_role(member)) + + async def send_available_message(self, channel: discord.TextChannel) -> None: + """Send the available message by editing a dormant message or sending a new message.""" + channel_info = f"#{channel} ({channel.id})" + log.trace(f"Sending available message in {channel_info}.") + + embed = discord.Embed(description=AVAILABLE_MSG) + + msg = await self.get_last_message(channel) + if self.match_bot_embed(msg, DORMANT_MSG): + log.trace(f"Found dormant message {msg.id} in {channel_info}; editing it.") + await msg.edit(embed=embed) + else: + log.trace(f"Dormant message not found in {channel_info}; sending a new message.") + await channel.send(embed=embed) + + async def try_get_channel(self, channel_id: int) -> discord.abc.GuildChannel: + """Attempt to get or fetch a channel and return it.""" + log.trace(f"Getting the channel {channel_id}.") + + channel = self.bot.get_channel(channel_id) + if not channel: + log.debug(f"Channel {channel_id} is not in cache; fetching from API.") + channel = await self.bot.fetch_channel(channel_id) + + log.trace(f"Channel #{channel} ({channel_id}) retrieved.") + return channel + + async def pin_wrapper(self, msg_id: int, channel: discord.TextChannel, *, pin: bool) -> bool: + """ + Pin message `msg_id` in `channel` if `pin` is True or unpin if it's False. + + Return True if successful and False otherwise. + """ + channel_str = f"#{channel} ({channel.id})" + if pin: + func = self.bot.http.pin_message + verb = "pin" + else: + func = self.bot.http.unpin_message + verb = "unpin" + + try: + await func(channel.id, msg_id) + except discord.HTTPException as e: + if e.code == 10008: + log.debug(f"Message {msg_id} in {channel_str} doesn't exist; can't {verb}.") + else: + log.exception( + f"Error {verb}ning message {msg_id} in {channel_str}: {e.status} ({e.code})" + ) + return False + else: + log.trace(f"{verb.capitalize()}ned message {msg_id} in {channel_str}.") + return True + + async def pin(self, message: discord.Message) -> None: + """Pin an initial question `message` and store it in a cache.""" + if await self.pin_wrapper(message.id, message.channel, pin=True): + await self.question_messages.set(message.channel.id, message.id) + + async def unpin(self, channel: discord.TextChannel) -> None: + """Unpin the initial question message sent in `channel`.""" + msg_id = await self.question_messages.pop(channel.id) + if msg_id is None: + log.debug(f"#{channel} ({channel.id}) doesn't have a message pinned.") + else: + await self.pin_wrapper(msg_id, channel, pin=False) + + async def wait_for_dormant_channel(self) -> discord.TextChannel: + """Wait for a dormant channel to become available in the queue and return it.""" + log.trace("Waiting for a dormant channel.") + + task = asyncio.create_task(self.channel_queue.get()) + self.queue_tasks.append(task) + channel = await task + + log.trace(f"Channel #{channel} ({channel.id}) finally retrieved from the queue.") + self.queue_tasks.remove(task) + + return channel + + +def validate_config() -> None: + """Raise a ValueError if the cog's config is invalid.""" + log.trace("Validating config.") + total = constants.HelpChannels.max_total_channels + available = constants.HelpChannels.max_available + + if total == 0 or available == 0: + raise ValueError("max_total_channels and max_available and must be greater than 0.") + + if total < available: + raise ValueError( + f"max_total_channels ({total}) must be greater than or equal to max_available " + f"({available})." + ) + + if total > MAX_CHANNELS_PER_CATEGORY: + raise ValueError( + f"max_total_channels ({total}) must be less than or equal to " + f"{MAX_CHANNELS_PER_CATEGORY} due to Discord's limit on channels per category." + ) + + +def setup(bot: Bot) -> None: + """Load the HelpChannels cog.""" + try: + validate_config() + except ValueError as e: + log.error(f"HelpChannels cog will not be loaded due to misconfiguration: {e}") + else: + bot.add_cog(HelpChannels(bot)) diff --git a/bot/exts/info/__init__.py b/bot/exts/info/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/bot/exts/info/__init__.py diff --git a/bot/cogs/doc.py b/bot/exts/info/doc.py index ddff9d14c..c16a99225 100644 --- a/bot/cogs/doc.py +++ b/bot/exts/info/doc.py @@ -21,8 +21,8 @@ from urllib3.exceptions import ProtocolError from bot.bot import Bot from bot.constants import MODERATION_ROLES, RedirectOutput from bot.converters import ValidPythonIdentifier, ValidURL -from bot.decorators import with_role from bot.pagination import LinePaginator +from bot.utils.messages import wait_for_deletion log = logging.getLogger(__name__) @@ -391,10 +391,11 @@ class Doc(commands.Cog): await error_message.delete(delay=NOT_FOUND_DELETE_DELAY) await ctx.message.delete(delay=NOT_FOUND_DELETE_DELAY) else: - await ctx.send(embed=doc_embed) + msg = await ctx.send(embed=doc_embed) + await wait_for_deletion(msg, (ctx.author.id,), client=self.bot) @docs_group.command(name='set', aliases=('s',)) - @with_role(*MODERATION_ROLES) + @commands.has_any_role(*MODERATION_ROLES) async def set_command( self, ctx: commands.Context, package_name: ValidPythonIdentifier, base_url: ValidURL, inventory_url: InventoryURL @@ -431,7 +432,7 @@ class Doc(commands.Cog): await ctx.send(f"Added package `{package_name}` to database and refreshed inventory.") @docs_group.command(name='delete', aliases=('remove', 'rm', 'd')) - @with_role(*MODERATION_ROLES) + @commands.has_any_role(*MODERATION_ROLES) async def delete_command(self, ctx: commands.Context, package_name: ValidPythonIdentifier) -> None: """ Removes the specified package from the database. @@ -448,7 +449,7 @@ class Doc(commands.Cog): await ctx.send(f"Successfully deleted `{package_name}` and refreshed inventory.") @docs_group.command(name="refresh", aliases=("rfsh", "r")) - @with_role(*MODERATION_ROLES) + @commands.has_any_role(*MODERATION_ROLES) async def refresh_command(self, ctx: commands.Context) -> None: """Refresh inventories and send differences to channel.""" old_inventories = set(self.base_urls) diff --git a/bot/exts/info/help.py b/bot/exts/info/help.py new file mode 100644 index 000000000..99d503f5c --- /dev/null +++ b/bot/exts/info/help.py @@ -0,0 +1,355 @@ +import itertools +import logging +from collections import namedtuple +from contextlib import suppress +from typing import List, Union + +from discord import Colour, Embed +from discord.ext.commands import Bot, Cog, Command, Context, Group, HelpCommand +from fuzzywuzzy import fuzz, process +from fuzzywuzzy.utils import full_process + +from bot import constants +from bot.constants import Channels, STAFF_ROLES +from bot.decorators import redirect_output +from bot.pagination import LinePaginator +from bot.utils.messages import wait_for_deletion + +log = logging.getLogger(__name__) + +COMMANDS_PER_PAGE = 8 +PREFIX = constants.Bot.prefix + +Category = namedtuple("Category", ["name", "description", "cogs"]) + + +class HelpQueryNotFound(ValueError): + """ + Raised when a HelpSession Query doesn't match a command or cog. + + Contains the custom attribute of ``possible_matches``. + + Instances of this object contain a dictionary of any command(s) that were close to matching the + query, where keys are the possible matched command names and values are the likeness match scores. + """ + + def __init__(self, arg: str, possible_matches: dict = None): + super().__init__(arg) + self.possible_matches = possible_matches + + +class CustomHelpCommand(HelpCommand): + """ + An interactive instance for the bot help command. + + Cogs can be grouped into custom categories. All cogs with the same category will be displayed + under a single category name in the help output. Custom categories are defined inside the cogs + as a class attribute named `category`. A description can also be specified with the attribute + `category_description`. If a description is not found in at least one cog, the default will be + the regular description (class docstring) of the first cog found in the category. + """ + + def __init__(self): + super().__init__(command_attrs={"help": "Shows help for bot commands"}) + + @redirect_output(destination_channel=Channels.bot_commands, bypass_roles=STAFF_ROLES) + async def command_callback(self, ctx: Context, *, command: str = None) -> None: + """Attempts to match the provided query with a valid command or cog.""" + # the only reason we need to tamper with this is because d.py does not support "categories", + # so we need to deal with them ourselves. + + bot = ctx.bot + + if command is None: + # quick and easy, send bot help if command is none + mapping = self.get_bot_mapping() + await self.send_bot_help(mapping) + return + + cog_matches = [] + description = None + for cog in bot.cogs.values(): + if hasattr(cog, "category") and cog.category == command: + cog_matches.append(cog) + if hasattr(cog, "category_description"): + description = cog.category_description + + if cog_matches: + category = Category(name=command, description=description, cogs=cog_matches) + await self.send_category_help(category) + return + + # it's either a cog, group, command or subcommand; let the parent class deal with it + await super().command_callback(ctx, command=command) + + async def get_all_help_choices(self) -> set: + """ + Get all the possible options for getting help in the bot. + + This will only display commands the author has permission to run. + + These include: + - Category names + - Cog names + - Group command names (and aliases) + - Command names (and aliases) + - Subcommand names (with parent group and aliases for subcommand, but not including aliases for group) + + Options and choices are case sensitive. + """ + # first get all commands including subcommands and full command name aliases + choices = set() + for command in await self.filter_commands(self.context.bot.walk_commands()): + # the the command or group name + choices.add(str(command)) + + if isinstance(command, Command): + # all aliases if it's just a command + choices.update(command.aliases) + else: + # otherwise we need to add the parent name in + choices.update(f"{command.full_parent_name} {alias}" for alias in command.aliases) + + # all cog names + choices.update(self.context.bot.cogs) + + # all category names + choices.update(cog.category for cog in self.context.bot.cogs.values() if hasattr(cog, "category")) + return choices + + async def command_not_found(self, string: str) -> "HelpQueryNotFound": + """ + Handles when a query does not match a valid command, group, cog or category. + + Will return an instance of the `HelpQueryNotFound` exception with the error message and possible matches. + """ + choices = await self.get_all_help_choices() + + # Run fuzzywuzzy's processor beforehand, and avoid matching if processed string is empty + # This avoids fuzzywuzzy from raising a warning on inputs with only non-alphanumeric characters + if (processed := full_process(string)): + result = process.extractBests(processed, choices, scorer=fuzz.ratio, score_cutoff=60, processor=None) + else: + result = [] + + return HelpQueryNotFound(f'Query "{string}" not found.', dict(result)) + + async def subcommand_not_found(self, command: Command, string: str) -> "HelpQueryNotFound": + """ + Redirects the error to `command_not_found`. + + `command_not_found` deals with searching and getting best choices for both commands and subcommands. + """ + return await self.command_not_found(f"{command.qualified_name} {string}") + + async def send_error_message(self, error: HelpQueryNotFound) -> None: + """Send the error message to the channel.""" + embed = Embed(colour=Colour.red(), title=str(error)) + + if getattr(error, "possible_matches", None): + matches = "\n".join(f"`{match}`" for match in error.possible_matches) + embed.description = f"**Did you mean:**\n{matches}" + + await self.context.send(embed=embed) + + async def command_formatting(self, command: Command) -> Embed: + """ + Takes a command and turns it into an embed. + + It will add an author, command signature + help, aliases and a note if the user can't run the command. + """ + embed = Embed() + embed.set_author(name="Command Help", icon_url=constants.Icons.questionmark) + + parent = command.full_parent_name + + name = str(command) if not parent else f"{parent} {command.name}" + command_details = f"**```{PREFIX}{name} {command.signature}```**\n" + + # show command aliases + aliases = [f"`{alias}`" if not parent else f"`{parent} {alias}`" for alias in command.aliases] + aliases += [f"`{alias}`" for alias in getattr(command, "root_aliases", ())] + aliases = ", ".join(sorted(aliases)) + if aliases: + command_details += f"**Can also use:** {aliases}\n\n" + + # check if the user is allowed to run this command + if not await command.can_run(self.context): + command_details += "***You cannot run this command.***\n\n" + + command_details += f"*{command.help or 'No details provided.'}*\n" + embed.description = command_details + + return embed + + async def send_command_help(self, command: Command) -> None: + """Send help for a single command.""" + embed = await self.command_formatting(command) + message = await self.context.send(embed=embed) + await wait_for_deletion(message, (self.context.author.id,), self.context.bot) + + @staticmethod + def get_commands_brief_details(commands_: List[Command], return_as_list: bool = False) -> Union[List[str], str]: + """ + Formats the prefix, command name and signature, and short doc for an iterable of commands. + + return_as_list is helpful for passing these command details into the paginator as a list of command details. + """ + details = [] + for command in commands_: + signature = f" {command.signature}" if command.signature else "" + details.append( + f"\n**`{PREFIX}{command.qualified_name}{signature}`**\n*{command.short_doc or 'No details provided'}*" + ) + if return_as_list: + return details + else: + return "".join(details) + + async def send_group_help(self, group: Group) -> None: + """Sends help for a group command.""" + subcommands = group.commands + + if len(subcommands) == 0: + # no subcommands, just treat it like a regular command + await self.send_command_help(group) + return + + # remove commands that the user can't run and are hidden, and sort by name + commands_ = await self.filter_commands(subcommands, sort=True) + + embed = await self.command_formatting(group) + + command_details = self.get_commands_brief_details(commands_) + if command_details: + embed.description += f"\n**Subcommands:**\n{command_details}" + + message = await self.context.send(embed=embed) + await wait_for_deletion(message, (self.context.author.id,), self.context.bot) + + async def send_cog_help(self, cog: Cog) -> None: + """Send help for a cog.""" + # sort commands by name, and remove any the user cant run or are hidden. + commands_ = await self.filter_commands(cog.get_commands(), sort=True) + + embed = Embed() + embed.set_author(name="Command Help", icon_url=constants.Icons.questionmark) + embed.description = f"**{cog.qualified_name}**\n*{cog.description}*" + + command_details = self.get_commands_brief_details(commands_) + if command_details: + embed.description += f"\n\n**Commands:**\n{command_details}" + + message = await self.context.send(embed=embed) + await wait_for_deletion(message, (self.context.author.id,), self.context.bot) + + @staticmethod + def _category_key(command: Command) -> str: + """ + Returns a cog name of a given command for use as a key for `sorted` and `groupby`. + + A zero width space is used as a prefix for results with no cogs to force them last in ordering. + """ + if command.cog: + with suppress(AttributeError): + if command.cog.category: + return f"**{command.cog.category}**" + return f"**{command.cog_name}**" + else: + return "**\u200bNo Category:**" + + async def send_category_help(self, category: Category) -> None: + """ + Sends help for a bot category. + + This sends a brief help for all commands in all cogs registered to the category. + """ + embed = Embed() + embed.set_author(name="Command Help", icon_url=constants.Icons.questionmark) + + all_commands = [] + for cog in category.cogs: + all_commands.extend(cog.get_commands()) + + filtered_commands = await self.filter_commands(all_commands, sort=True) + + command_detail_lines = self.get_commands_brief_details(filtered_commands, return_as_list=True) + description = f"**{category.name}**\n*{category.description}*" + + if command_detail_lines: + description += "\n\n**Commands:**" + + await LinePaginator.paginate( + command_detail_lines, + self.context, + embed, + prefix=description, + max_lines=COMMANDS_PER_PAGE, + max_size=2000, + ) + + async def send_bot_help(self, mapping: dict) -> None: + """Sends help for all bot commands and cogs.""" + bot = self.context.bot + + embed = Embed() + embed.set_author(name="Command Help", icon_url=constants.Icons.questionmark) + + filter_commands = await self.filter_commands(bot.commands, sort=True, key=self._category_key) + + cog_or_category_pages = [] + + for cog_or_category, _commands in itertools.groupby(filter_commands, key=self._category_key): + sorted_commands = sorted(_commands, key=lambda c: c.name) + + if len(sorted_commands) == 0: + continue + + command_detail_lines = self.get_commands_brief_details(sorted_commands, return_as_list=True) + + # Split cogs or categories which have too many commands to fit in one page. + # The length of commands is included for later use when aggregating into pages for the paginator. + for index in range(0, len(sorted_commands), COMMANDS_PER_PAGE): + truncated_lines = command_detail_lines[index:index + COMMANDS_PER_PAGE] + joined_lines = "".join(truncated_lines) + cog_or_category_pages.append((f"**{cog_or_category}**{joined_lines}", len(truncated_lines))) + + pages = [] + counter = 0 + page = "" + for page_details, length in cog_or_category_pages: + counter += length + if counter > COMMANDS_PER_PAGE: + # force a new page on paginator even if it falls short of the max pages + # since we still want to group categories/cogs. + counter = length + pages.append(page) + page = f"{page_details}\n\n" + else: + page += f"{page_details}\n\n" + + if page: + # add any remaining command help that didn't get added in the last iteration above. + pages.append(page) + + await LinePaginator.paginate(pages, self.context, embed=embed, max_lines=1, max_size=2000) + + +class Help(Cog): + """Custom Embed Pagination Help feature.""" + + def __init__(self, bot: Bot) -> None: + self.bot = bot + self.old_help_command = bot.help_command + bot.help_command = CustomHelpCommand() + bot.help_command.cog = self + + def cog_unload(self) -> None: + """Reset the help command when the cog is unloaded.""" + self.bot.help_command = self.old_help_command + + +def setup(bot: Bot) -> None: + """Load the Help cog.""" + bot.add_cog(Help(bot)) + log.info("Cog loaded: Help") diff --git a/bot/cogs/information.py b/bot/exts/info/information.py index 7921a4932..f6ed176f1 100644 --- a/bot/cogs/information.py +++ b/bot/exts/info/information.py @@ -4,29 +4,80 @@ import pprint import textwrap from collections import Counter, defaultdict from string import Template -from typing import Any, Mapping, Optional, Union +from typing import Any, Mapping, Optional, Tuple, Union -from discord import Colour, Embed, Member, Message, Role, Status, utils -from discord.ext.commands import BucketType, Cog, Context, Paginator, command, group +from discord import ChannelType, Colour, CustomActivity, Embed, Guild, Member, Message, Role, Status, utils +from discord.abc import GuildChannel +from discord.ext.commands import BucketType, Cog, Context, Paginator, command, group, has_any_role from discord.utils import escape_markdown from bot import constants from bot.bot import Bot -from bot.decorators import InChannelCheckFailure, in_channel, with_role +from bot.decorators import in_whitelist from bot.pagination import LinePaginator -from bot.utils.checks import cooldown_with_role_bypass, with_role_check +from bot.utils.checks import cooldown_with_role_bypass, has_no_roles_check, in_whitelist_check from bot.utils.time import time_since log = logging.getLogger(__name__) +STATUS_EMOTES = { + Status.offline: constants.Emojis.status_offline, + Status.dnd: constants.Emojis.status_dnd, + Status.idle: constants.Emojis.status_idle +} + + class Information(Cog): """A cog with commands for generating embeds with server info, such as server stats and user info.""" def __init__(self, bot: Bot): self.bot = bot - @with_role(*constants.MODERATION_ROLES) + @staticmethod + def role_can_read(channel: GuildChannel, role: Role) -> bool: + """Return True if `role` can read messages in `channel`.""" + overwrites = channel.overwrites_for(role) + return overwrites.read_messages is True + + def get_staff_channel_count(self, guild: Guild) -> int: + """ + Get the number of channels that are staff-only. + + We need to know two things about a channel: + - Does the @everyone role have explicit read deny permissions? + - Do staff roles have explicit read allow permissions? + + If the answer to both of these questions is yes, it's a staff channel. + """ + channel_ids = set() + for channel in guild.channels: + if channel.type is ChannelType.category: + continue + + everyone_can_read = self.role_can_read(channel, guild.default_role) + + for role in constants.STAFF_ROLES: + role_can_read = self.role_can_read(channel, guild.get_role(role)) + if role_can_read and not everyone_can_read: + channel_ids.add(channel.id) + break + + return len(channel_ids) + + @staticmethod + def get_channel_type_counts(guild: Guild) -> str: + """Return the total amounts of the various types of channels in `guild`.""" + channel_counter = Counter(c.type for c in guild.channels) + channel_type_list = [] + for channel, count in channel_counter.items(): + channel_type = str(channel).title() + channel_type_list.append(f"{channel_type} channels: {count}") + + channel_type_list = sorted(channel_type_list) + return "\n".join(channel_type_list) + + @has_any_role(*constants.STAFF_ROLES) @command(name="roles") async def roles_info(self, ctx: Context) -> None: """Returns a list of all roles and their corresponding IDs.""" @@ -46,7 +97,7 @@ class Information(Cog): await LinePaginator.paginate(role_list, ctx, embed, empty=False) - @with_role(*constants.MODERATION_ROLES) + @has_any_role(*constants.STAFF_ROLES) @command(name="role") async def role_info(self, ctx: Context, *roles: Union[Role, str]) -> None: """ @@ -72,10 +123,7 @@ class Information(Cog): parsed_roles.append(role) if failed_roles: - await ctx.send( - ":x: I could not convert the following role names to a role: \n- " - "\n- ".join(failed_roles) - ) + await ctx.send(f":x: Could not retrieve the following roles: {', '.join(failed_roles)}") for role in parsed_roles: h, s, v = colorsys.rgb_to_hsv(*role.colour.to_rgb()) @@ -102,15 +150,16 @@ class Information(Cog): roles = len(ctx.guild.roles) member_count = ctx.guild.member_count - - # How many of each type of channel? - channels = Counter(c.type for c in ctx.guild.channels) - channel_counts = "".join(sorted(f"{str(ch).title()} channels: {channels[ch]}\n" for ch in channels)).strip() + channel_counts = self.get_channel_type_counts(ctx.guild) # How many of each user status? statuses = Counter(member.status for member in ctx.guild.members) embed = Embed(colour=Colour.blurple()) + # How many staff members and staff channels do we have? + staff_member_count = len(ctx.guild.get_role(constants.Roles.helpers).members) + staff_channel_count = self.get_staff_channel_count(ctx.guild) + # Because channel_counts lacks leading whitespace, it breaks the dedent if it's inserted directly by the # f-string. While this is correctly formated by Discord, it makes unit testing difficult. To keep the formatting # without joining a tuple of strings we can use a Template string to insert the already-formatted channel_counts @@ -122,12 +171,16 @@ class Information(Cog): Voice region: {region} Features: {features} - **Counts** + **Channel counts** + $channel_counts + Staff channels: {staff_channel_count} + + **Member counts** Members: {member_count:,} + Staff members: {staff_member_count} Roles: {roles} - $channel_counts - **Members** + **Member statuses** {constants.Emojis.status_online} {statuses[Status.online]:,} {constants.Emojis.status_idle} {statuses[Status.idle]:,} {constants.Emojis.status_dnd} {statuses[Status.dnd]:,} @@ -145,18 +198,14 @@ class Information(Cog): user = ctx.author # Do a role check if this is being executed on someone other than the caller - elif user != ctx.author and not with_role_check(ctx, *constants.MODERATION_ROLES): + elif user != ctx.author and await has_no_roles_check(ctx, *constants.MODERATION_ROLES): await ctx.send("You may not use this command on users other than yourself.") return - # Non-staff may only do this in #bot-commands - if not with_role_check(ctx, *constants.STAFF_ROLES): - if not ctx.channel.id == constants.Channels.bot_commands: - raise InChannelCheckFailure(constants.Channels.bot_commands) - - embed = await self.create_user_embed(ctx, user) - - await ctx.send(embed=embed) + # Will redirect to #bot-commands if it fails. + if in_whitelist_check(ctx, roles=constants.STAFF_ROLES): + embed = await self.create_user_embed(ctx, user) + await ctx.send(embed=embed) async def create_user_embed(self, ctx: Context, user: Member) -> Embed: """Creates an embed containing information on the `user`.""" @@ -165,53 +214,94 @@ class Information(Cog): # Custom status custom_status = '' for activity in user.activities: - # Check activity.state for None value if user has a custom status set - # This guards against a custom status with an emoji but no text, which will cause - # escape_markdown to raise an exception - # This can be reworked after a move to d.py 1.3.0+, which adds a CustomActivity class - if activity.name == 'Custom Status' and activity.state: - state = escape_markdown(activity.state) - custom_status = f'Status: {state}\n' + if isinstance(activity, CustomActivity): + state = "" + + if activity.name: + state = escape_markdown(activity.name) + + emoji = "" + if activity.emoji: + # If an emoji is unicode use the emoji, else write the emote like :abc: + if not activity.emoji.id: + emoji += activity.emoji.name + " " + else: + emoji += f"`:{activity.emoji.name}:` " + + custom_status = f'Status: {emoji}{state}\n' name = str(user) if user.nick: name = f"{user.nick} ({name})" - joined = time_since(user.joined_at, precision="days") + badges = [] + + for badge, is_set in user.public_flags: + if is_set and (emoji := getattr(constants.Emojis, f"badge_{badge}", None)): + badges.append(emoji) + + joined = time_since(user.joined_at, max_units=3) roles = ", ".join(role.mention for role in user.roles[1:]) - description = [ - textwrap.dedent(f""" - **User Information** - Created: {created} - Profile: {user.mention} - ID: {user.id} - {custom_status} - **Member Information** - Joined: {joined} - Roles: {roles or None} - """).strip() + desktop_status = STATUS_EMOTES.get(user.desktop_status, constants.Emojis.status_online) + web_status = STATUS_EMOTES.get(user.web_status, constants.Emojis.status_online) + mobile_status = STATUS_EMOTES.get(user.mobile_status, constants.Emojis.status_online) + + fields = [ + ( + "User information", + textwrap.dedent(f""" + Created: {created} + Profile: {user.mention} + ID: {user.id} + {custom_status} + """).strip() + ), + ( + "Member information", + textwrap.dedent(f""" + Joined: {joined} + Roles: {roles or None} + """).strip() + ), + ( + "Status", + textwrap.dedent(f""" + {desktop_status} Desktop + {web_status} Web + {mobile_status} Mobile + """).strip() + ) ] + # Use getattr to future-proof for commands invoked via DMs. + show_verbose = ( + ctx.channel.id in constants.MODERATION_CHANNELS + or getattr(ctx.channel, "category_id", None) == constants.Categories.modmail + ) + # Show more verbose output in moderation channels for infractions and nominations - if ctx.channel.id in constants.MODERATION_CHANNELS: - description.append(await self.expanded_user_infraction_counts(user)) - description.append(await self.user_nomination_counts(user)) + if show_verbose: + fields.append(await self.expanded_user_infraction_counts(user)) + fields.append(await self.user_nomination_counts(user)) else: - description.append(await self.basic_user_infraction_counts(user)) + fields.append(await self.basic_user_infraction_counts(user)) # Let's build the embed now embed = Embed( title=name, - description="\n\n".join(description) + description=" ".join(badges) ) - embed.set_thumbnail(url=user.avatar_url_as(format="png")) + for field_name, field_content in fields: + embed.add_field(name=field_name, value=field_content, inline=False) + + embed.set_thumbnail(url=user.avatar_url_as(static_format="png")) embed.colour = user.top_role.colour if roles else Colour.blurple() return embed - async def basic_user_infraction_counts(self, member: Member) -> str: + async def basic_user_infraction_counts(self, member: Member) -> Tuple[str, str]: """Gets the total and active infraction counts for the given `member`.""" infractions = await self.bot.api_client.get( 'bot/infractions', @@ -224,11 +314,11 @@ class Information(Cog): total_infractions = len(infractions) active_infractions = sum(infraction['active'] for infraction in infractions) - infraction_output = f"**Infractions**\nTotal: {total_infractions}\nActive: {active_infractions}" + infraction_output = f"Total: {total_infractions}\nActive: {active_infractions}" - return infraction_output + return "Infractions", infraction_output - async def expanded_user_infraction_counts(self, member: Member) -> str: + async def expanded_user_infraction_counts(self, member: Member) -> Tuple[str, str]: """ Gets expanded infraction counts for the given `member`. @@ -242,9 +332,9 @@ class Information(Cog): } ) - infraction_output = ["**Infractions**"] + infraction_output = [] if not infractions: - infraction_output.append("This user has never received an infraction.") + infraction_output.append("No infractions") else: # Count infractions split by `type` and `active` status for this user infraction_types = set() @@ -267,9 +357,9 @@ class Information(Cog): infraction_output.append(line) - return "\n".join(infraction_output) + return "Infractions", "\n".join(infraction_output) - async def user_nomination_counts(self, member: Member) -> str: + async def user_nomination_counts(self, member: Member) -> Tuple[str, str]: """Gets the active and historical nomination counts for the given `member`.""" nominations = await self.bot.api_client.get( 'bot/nominations', @@ -278,21 +368,21 @@ class Information(Cog): } ) - output = ["**Nominations**"] + output = [] if not nominations: - output.append("This user has never been nominated.") + output.append("No nominations") else: count = len(nominations) is_currently_nominated = any(nomination["active"] for nomination in nominations) nomination_noun = "nomination" if count == 1 else "nominations" if is_currently_nominated: - output.append(f"This user is **currently** nominated ({count} {nomination_noun} in total).") + output.append(f"This user is **currently** nominated\n({count} {nomination_noun} in total)") else: output.append(f"This user has {count} historical {nomination_noun}, but is currently not nominated.") - return "\n".join(output) + return "Nominations", "\n".join(output) def format_fields(self, mapping: Mapping[str, Any], field_width: Optional[int] = None) -> str: """Format a mapping to be readable to a human.""" @@ -330,8 +420,8 @@ class Information(Cog): return out.rstrip() @cooldown_with_role_bypass(2, 60 * 3, BucketType.member, bypass_roles=constants.STAFF_ROLES) - @group(invoke_without_command=True) - @in_channel(constants.Channels.bot_commands, bypass_roles=constants.STAFF_ROLES) + @group(invoke_without_command=True, enabled=False) + @in_whitelist(channels=(constants.Channels.bot_commands,), roles=constants.STAFF_ROLES) async def raw(self, ctx: Context, *, message: Message, json: bool = False) -> None: """Shows information about the raw API response.""" # I *guess* it could be deleted right as the command is invoked but I felt like it wasn't worth handling @@ -365,7 +455,7 @@ class Information(Cog): for page in paginator.pages: await ctx.send(page) - @raw.command() + @raw.command(enabled=False) async def json(self, ctx: Context, message: Message) -> None: """Shows information about the raw API response in a copy-pasteable Python format.""" await ctx.invoke(self.raw, message=message, json=True) diff --git a/bot/exts/info/python_news.py b/bot/exts/info/python_news.py new file mode 100644 index 000000000..0ab5738a4 --- /dev/null +++ b/bot/exts/info/python_news.py @@ -0,0 +1,232 @@ +import logging +import typing as t +from datetime import date, datetime + +import discord +import feedparser +from bs4 import BeautifulSoup +from discord.ext.commands import Cog +from discord.ext.tasks import loop + +from bot import constants +from bot.bot import Bot +from bot.utils.webhooks import send_webhook + +PEPS_RSS_URL = "https://www.python.org/dev/peps/peps.rss/" + +RECENT_THREADS_TEMPLATE = "https://mail.python.org/archives/list/{name}@python.org/recent-threads" +THREAD_TEMPLATE_URL = "https://mail.python.org/archives/api/list/{name}@python.org/thread/{id}/" +MAILMAN_PROFILE_URL = "https://mail.python.org/archives/users/{id}/" +THREAD_URL = "https://mail.python.org/archives/list/{list}@python.org/thread/{id}/" + +AVATAR_URL = "https://www.python.org/static/opengraph-icon-200x200.png" + +log = logging.getLogger(__name__) + + +class PythonNews(Cog): + """Post new PEPs and Python News to `#python-news`.""" + + def __init__(self, bot: Bot): + self.bot = bot + self.webhook_names = {} + self.webhook: t.Optional[discord.Webhook] = None + + self.bot.loop.create_task(self.get_webhook_names()) + self.bot.loop.create_task(self.get_webhook_and_channel()) + + async def start_tasks(self) -> None: + """Start the tasks for fetching new PEPs and mailing list messages.""" + self.fetch_new_media.start() + + @loop(minutes=20) + async def fetch_new_media(self) -> None: + """Fetch new mailing list messages and then new PEPs.""" + await self.post_maillist_news() + await self.post_pep_news() + + async def sync_maillists(self) -> None: + """Sync currently in-use maillists with API.""" + # Wait until guild is available to avoid running before everything is ready + await self.bot.wait_until_guild_available() + + response = await self.bot.api_client.get("bot/bot-settings/news") + for mail in constants.PythonNews.mail_lists: + if mail not in response["data"]: + response["data"][mail] = [] + + # Because we are handling PEPs differently, we don't include it to mail lists + if "pep" not in response["data"]: + response["data"]["pep"] = [] + + await self.bot.api_client.put("bot/bot-settings/news", json=response) + + async def get_webhook_names(self) -> None: + """Get webhook author names from maillist API.""" + await self.bot.wait_until_guild_available() + + async with self.bot.http_session.get("https://mail.python.org/archives/api/lists") as resp: + lists = await resp.json() + + for mail in lists: + if mail["name"].split("@")[0] in constants.PythonNews.mail_lists: + self.webhook_names[mail["name"].split("@")[0]] = mail["display_name"] + + async def post_pep_news(self) -> None: + """Fetch new PEPs and when they don't have announcement in #python-news, create it.""" + # Wait until everything is ready and http_session available + await self.bot.wait_until_guild_available() + await self.sync_maillists() + + async with self.bot.http_session.get(PEPS_RSS_URL) as resp: + data = feedparser.parse(await resp.text("utf-8")) + + news_listing = await self.bot.api_client.get("bot/bot-settings/news") + payload = news_listing.copy() + pep_numbers = news_listing["data"]["pep"] + + # Reverse entries to send oldest first + data["entries"].reverse() + for new in data["entries"]: + try: + new_datetime = datetime.strptime(new["published"], "%a, %d %b %Y %X %Z") + except ValueError: + log.warning(f"Wrong datetime format passed in PEP new: {new['published']}") + continue + pep_nr = new["title"].split(":")[0].split()[1] + if ( + pep_nr in pep_numbers + or new_datetime.date() < date.today() + ): + continue + + # Build an embed and send a webhook + embed = discord.Embed( + title=new["title"], + description=new["summary"], + timestamp=new_datetime, + url=new["link"], + colour=constants.Colours.soft_green + ) + embed.set_footer(text=data["feed"]["title"], icon_url=AVATAR_URL) + msg = await send_webhook( + webhook=self.webhook, + username=data["feed"]["title"], + embed=embed, + avatar_url=AVATAR_URL, + wait=True, + ) + payload["data"]["pep"].append(pep_nr) + + # Increase overall PEP new stat + self.bot.stats.incr("python_news.posted.pep") + + if msg.channel.is_news(): + log.trace("Publishing PEP annnouncement because it was in a news channel") + await msg.publish() + + # Apply new sent news to DB to avoid duplicate sending + await self.bot.api_client.put("bot/bot-settings/news", json=payload) + + async def post_maillist_news(self) -> None: + """Send new maillist threads to #python-news that is listed in configuration.""" + await self.bot.wait_until_guild_available() + await self.sync_maillists() + existing_news = await self.bot.api_client.get("bot/bot-settings/news") + payload = existing_news.copy() + + for maillist in constants.PythonNews.mail_lists: + async with self.bot.http_session.get(RECENT_THREADS_TEMPLATE.format(name=maillist)) as resp: + recents = BeautifulSoup(await resp.text(), features="lxml") + + # When a <p> element is present in the response then the mailing list + # has not had any activity during the current month, so therefore it + # can be ignored. + if recents.p: + continue + + for thread in recents.html.body.div.find_all("a", href=True): + # We want only these threads that have identifiers + if "latest" in thread["href"]: + continue + + thread_information, email_information = await self.get_thread_and_first_mail( + maillist, thread["href"].split("/")[-2] + ) + + try: + new_date = datetime.strptime(email_information["date"], "%Y-%m-%dT%X%z") + except ValueError: + log.warning(f"Invalid datetime from Thread email: {email_information['date']}") + continue + + if ( + thread_information["thread_id"] in existing_news["data"][maillist] + or 'Re: ' in thread_information["subject"] + or new_date.date() < date.today() + ): + continue + + content = email_information["content"] + link = THREAD_URL.format(id=thread["href"].split("/")[-2], list=maillist) + + # Build an embed and send a message to the webhook + embed = discord.Embed( + title=thread_information["subject"], + description=content[:500] + f"... [continue reading]({link})" if len(content) > 500 else content, + timestamp=new_date, + url=link, + colour=constants.Colours.soft_green + ) + embed.set_author( + name=f"{email_information['sender_name']} ({email_information['sender']['address']})", + url=MAILMAN_PROFILE_URL.format(id=email_information["sender"]["mailman_id"]), + ) + embed.set_footer( + text=f"Posted to {self.webhook_names[maillist]}", + icon_url=AVATAR_URL, + ) + msg = await send_webhook( + webhook=self.webhook, + username=self.webhook_names[maillist], + embed=embed, + avatar_url=AVATAR_URL, + wait=True, + ) + payload["data"][maillist].append(thread_information["thread_id"]) + + # Increase this specific maillist counter in stats + self.bot.stats.incr(f"python_news.posted.{maillist.replace('-', '_')}") + + if msg.channel.is_news(): + log.trace("Publishing mailing list message because it was in a news channel") + await msg.publish() + + await self.bot.api_client.put("bot/bot-settings/news", json=payload) + + async def get_thread_and_first_mail(self, maillist: str, thread_identifier: str) -> t.Tuple[t.Any, t.Any]: + """Get mail thread and first mail from mail.python.org based on `maillist` and `thread_identifier`.""" + async with self.bot.http_session.get( + THREAD_TEMPLATE_URL.format(name=maillist, id=thread_identifier) + ) as resp: + thread_information = await resp.json() + + async with self.bot.http_session.get(thread_information["starting_email"]) as resp: + email_information = await resp.json() + return thread_information, email_information + + async def get_webhook_and_channel(self) -> None: + """Storage #python-news channel Webhook and `TextChannel` to `News.webhook` and `channel`.""" + await self.bot.wait_until_guild_available() + self.webhook = await self.bot.fetch_webhook(constants.PythonNews.webhook) + + await self.start_tasks() + + def cog_unload(self) -> None: + """Stop news posting tasks on cog unload.""" + self.fetch_new_media.cancel() + + +def setup(bot: Bot) -> None: + """Add `News` cog.""" + bot.add_cog(PythonNews(bot)) diff --git a/bot/cogs/reddit.py b/bot/exts/info/reddit.py index 426c34bfa..debe40c82 100644 --- a/bot/cogs/reddit.py +++ b/bot/exts/info/reddit.py @@ -8,15 +8,15 @@ from typing import List from aiohttp import BasicAuth, ClientError from discord import Colour, Embed, TextChannel -from discord.ext.commands import Cog, Context, group +from discord.ext.commands import Cog, Context, group, has_any_role from discord.ext.tasks import loop -from discord.utils import sleep_until +from discord.utils import escape_markdown, sleep_until from bot.bot import Bot from bot.constants import Channels, ERROR_REPLIES, Emojis, Reddit as RedditConfig, STAFF_ROLES, Webhooks from bot.converters import Subreddit -from bot.decorators import with_role from bot.pagination import LinePaginator +from bot.utils.messages import sub_clyde log = logging.getLogger(__name__) @@ -187,6 +187,8 @@ class Reddit(Cog): author = data["author"] title = textwrap.shorten(data["title"], width=64, placeholder="...") + # Normal brackets interfere with Markdown. + title = escape_markdown(title).replace("[", "⦋").replace("]", "⦌") link = self.URL + data["permalink"] embed.description += ( @@ -219,15 +221,19 @@ class Reddit(Cog): for subreddit in RedditConfig.subreddits: top_posts = await self.get_top_posts(subreddit=subreddit, time="day") - await self.webhook.send(username=f"{subreddit} Top Daily Posts", embed=top_posts) + username = sub_clyde(f"{subreddit} Top Daily Posts") + message = await self.webhook.send(username=username, embed=top_posts, wait=True) + + if message.channel.is_news(): + await message.publish() async def top_weekly_posts(self) -> None: """Post a summary of the top posts.""" for subreddit in RedditConfig.subreddits: # Send and pin the new weekly posts. top_posts = await self.get_top_posts(subreddit=subreddit, time="week") - - message = await self.webhook.send(wait=True, username=f"{subreddit} Top Weekly Posts", embed=top_posts) + username = sub_clyde(f"{subreddit} Top Weekly Posts") + message = await self.webhook.send(wait=True, username=username, embed=top_posts) if subreddit.lower() == "r/python": if not self.channel: @@ -243,10 +249,13 @@ class Reddit(Cog): await message.pin() + if message.channel.is_news(): + await message.publish() + @group(name="reddit", invoke_without_command=True) async def reddit_group(self, ctx: Context) -> None: """View the top posts from various subreddits.""" - await ctx.send_help("reddit") + await ctx.send_help(ctx.command) @reddit_group.command(name="top") async def top_command(self, ctx: Context, subreddit: Subreddit = "r/Python") -> None: @@ -272,7 +281,7 @@ class Reddit(Cog): await ctx.send(content=f"Here are this week's top {subreddit} posts!", embed=embed) - @with_role(*STAFF_ROLES) + @has_any_role(*STAFF_ROLES) @reddit_group.command(name="subreddits", aliases=("subs",)) async def subreddits_command(self, ctx: Context) -> None: """Send a paginated embed of all the subreddits we're relaying.""" diff --git a/bot/cogs/site.py b/bot/exts/info/site.py index c17761a2b..2d3a3d9f3 100644 --- a/bot/cogs/site.py +++ b/bot/exts/info/site.py @@ -21,9 +21,9 @@ class Site(Cog): @group(name="site", aliases=("s",), invoke_without_command=True) async def site_group(self, ctx: Context) -> None: """Commands for getting info about our website.""" - await ctx.send_help("site") + await ctx.send_help(ctx.command) - @site_group.command(name="home", aliases=("about",)) + @site_group.command(name="home", aliases=("about",), root_aliases=("home",)) async def site_main(self, ctx: Context) -> None: """Info about the website itself.""" url = f"{URLs.site_schema}{URLs.site}/" @@ -33,14 +33,14 @@ class Site(Cog): embed.colour = Colour.blurple() embed.description = ( f"[Our official website]({url}) is an open-source community project " - "created with Python and Flask. It contains information about the server " + "created with Python and Django. It contains information about the server " "itself, lets you sign up for upcoming events, has its own wiki, contains " "a list of valuable learning resources, and much more." ) await ctx.send(embed=embed) - @site_group.command(name="resources") + @site_group.command(name="resources", root_aliases=("resources", "resource")) async def site_resources(self, ctx: Context) -> None: """Info about the site's Resources page.""" learning_url = f"{PAGES_URL}/resources" @@ -56,7 +56,7 @@ class Site(Cog): await ctx.send(embed=embed) - @site_group.command(name="tools") + @site_group.command(name="tools", root_aliases=("tools",)) async def site_tools(self, ctx: Context) -> None: """Info about the site's Tools page.""" tools_url = f"{PAGES_URL}/resources/tools" @@ -87,7 +87,7 @@ class Site(Cog): await ctx.send(embed=embed) - @site_group.command(name="faq") + @site_group.command(name="faq", root_aliases=("faq",)) async def site_faq(self, ctx: Context) -> None: """Info about the site's FAQ page.""" url = f"{PAGES_URL}/frequently-asked-questions" @@ -104,7 +104,7 @@ class Site(Cog): await ctx.send(embed=embed) - @site_group.command(aliases=['r', 'rule'], name='rules') + @site_group.command(name="rules", aliases=("r", "rule"), root_aliases=("rules", "rule")) async def site_rules(self, ctx: Context, *rules: int) -> None: """Provides a link to all rules or, if specified, displays specific rule(s).""" rules_embed = Embed(title='Rules', color=Colour.blurple()) @@ -133,6 +133,9 @@ class Site(Cog): await ctx.send(f":x: Invalid rule indices: {indices}") return + for rule in rules: + self.bot.stats.incr(f"rule_uses.{rule}") + final_rules = tuple(f"**{pick}.** {full_rules[pick - 1]}" for pick in rules) await LinePaginator.paginate(final_rules, ctx, rules_embed, max_lines=3) diff --git a/bot/exts/info/source.py b/bot/exts/info/source.py new file mode 100644 index 000000000..7b41352d4 --- /dev/null +++ b/bot/exts/info/source.py @@ -0,0 +1,131 @@ +import inspect +from pathlib import Path +from typing import Optional, Tuple, Union + +from discord import Embed, utils +from discord.ext import commands + +from bot.bot import Bot +from bot.constants import URLs + +SourceType = Union[commands.HelpCommand, commands.Command, commands.Cog, str, commands.ExtensionNotLoaded] + + +class SourceConverter(commands.Converter): + """Convert an argument into a help command, tag, command, or cog.""" + + async def convert(self, ctx: commands.Context, argument: str) -> SourceType: + """Convert argument into source object.""" + if argument.lower().startswith("help"): + return ctx.bot.help_command + + cog = ctx.bot.get_cog(argument) + if cog: + return cog + + cmd = ctx.bot.get_command(argument) + if cmd: + return cmd + + tags_cog = ctx.bot.get_cog("Tags") + show_tag = True + + if not tags_cog: + show_tag = False + elif argument.lower() in tags_cog._cache: + return argument.lower() + + escaped_arg = utils.escape_markdown(argument) + + raise commands.BadArgument( + f"Unable to convert '{escaped_arg}' to valid command{', tag,' if show_tag else ''} or Cog." + ) + + +class BotSource(commands.Cog): + """Displays information about the bot's source code.""" + + def __init__(self, bot: Bot): + self.bot = bot + + @commands.command(name="source", aliases=("src",)) + async def source_command(self, ctx: commands.Context, *, source_item: SourceConverter = None) -> None: + """Display information and a GitHub link to the source code of a command, tag, or cog.""" + if not source_item: + embed = Embed(title="Bot's GitHub Repository") + embed.add_field(name="Repository", value=f"[Go to GitHub]({URLs.github_bot_repo})") + embed.set_thumbnail(url="https://avatars1.githubusercontent.com/u/9919") + await ctx.send(embed=embed) + return + + embed = await self.build_embed(source_item) + await ctx.send(embed=embed) + + def get_source_link(self, source_item: SourceType) -> Tuple[str, str, Optional[int]]: + """ + Build GitHub link of source item, return this link, file location and first line number. + + Raise BadArgument if `source_item` is a dynamically-created object (e.g. via internal eval). + """ + if isinstance(source_item, commands.Command): + src = source_item.callback.__code__ + filename = src.co_filename + elif isinstance(source_item, str): + tags_cog = self.bot.get_cog("Tags") + filename = tags_cog._cache[source_item]["location"] + else: + src = type(source_item) + try: + filename = inspect.getsourcefile(src) + except TypeError: + raise commands.BadArgument("Cannot get source for a dynamically-created object.") + + if not isinstance(source_item, str): + try: + lines, first_line_no = inspect.getsourcelines(src) + except OSError: + raise commands.BadArgument("Cannot get source for a dynamically-created object.") + + lines_extension = f"#L{first_line_no}-L{first_line_no+len(lines)-1}" + else: + first_line_no = None + lines_extension = "" + + # Handle tag file location differently than others to avoid errors in some cases + if not first_line_no: + file_location = Path(filename).relative_to("/bot/") + else: + file_location = Path(filename).relative_to(Path.cwd()).as_posix() + + url = f"{URLs.github_bot_repo}/blob/master/{file_location}{lines_extension}" + + return url, file_location, first_line_no or None + + async def build_embed(self, source_object: SourceType) -> Optional[Embed]: + """Build embed based on source object.""" + url, location, first_line = self.get_source_link(source_object) + + if isinstance(source_object, commands.HelpCommand): + title = "Help Command" + description = source_object.__doc__.splitlines()[1] + elif isinstance(source_object, commands.Command): + description = source_object.short_doc + title = f"Command: {source_object.qualified_name}" + elif isinstance(source_object, str): + title = f"Tag: {source_object}" + description = "" + else: + title = f"Cog: {source_object.qualified_name}" + description = source_object.description.splitlines()[0] + + embed = Embed(title=title, description=description) + embed.add_field(name="Source Code", value=f"[Go to GitHub]({url})") + line_text = f":{first_line}" if first_line else "" + embed.set_footer(text=f"{location}{line_text}") + + return embed + + +def setup(bot: Bot) -> None: + """Load the BotSource cog.""" + bot.add_cog(BotSource(bot)) diff --git a/bot/exts/info/stats.py b/bot/exts/info/stats.py new file mode 100644 index 000000000..d42f55466 --- /dev/null +++ b/bot/exts/info/stats.py @@ -0,0 +1,129 @@ +import string +from datetime import datetime + +from discord import Member, Message, Status +from discord.ext.commands import Cog, Context +from discord.ext.tasks import loop + +from bot.bot import Bot +from bot.constants import Categories, Channels, Guild, Stats as StatConf + + +CHANNEL_NAME_OVERRIDES = { + Channels.off_topic_0: "off_topic_0", + Channels.off_topic_1: "off_topic_1", + Channels.off_topic_2: "off_topic_2", + Channels.staff_lounge: "staff_lounge" +} + +ALLOWED_CHARS = string.ascii_letters + string.digits + "_" + + +class Stats(Cog): + """A cog which provides a way to hook onto Discord events and forward to stats.""" + + def __init__(self, bot: Bot): + self.bot = bot + self.last_presence_update = None + self.update_guild_boost.start() + + @Cog.listener() + async def on_message(self, message: Message) -> None: + """Report message events in the server to statsd.""" + if message.guild is None: + return + + if message.guild.id != Guild.id: + return + + cat = getattr(message.channel, "category", None) + if cat is not None and cat.id == Categories.modmail: + if message.channel.id != Channels.incidents: + # Do not report modmail channels to stats, there are too many + # of them for interesting statistics to be drawn out of this. + return + + reformatted_name = message.channel.name.replace('-', '_') + + if CHANNEL_NAME_OVERRIDES.get(message.channel.id): + reformatted_name = CHANNEL_NAME_OVERRIDES.get(message.channel.id) + + reformatted_name = "".join(char for char in reformatted_name if char in ALLOWED_CHARS) + + stat_name = f"channels.{reformatted_name}" + self.bot.stats.incr(stat_name) + + # Increment the total message count + self.bot.stats.incr("messages") + + @Cog.listener() + async def on_command_completion(self, ctx: Context) -> None: + """Report completed commands to statsd.""" + command_name = ctx.command.qualified_name.replace(" ", "_") + + self.bot.stats.incr(f"commands.{command_name}") + + @Cog.listener() + async def on_member_join(self, member: Member) -> None: + """Update member count stat on member join.""" + if member.guild.id != Guild.id: + return + + self.bot.stats.gauge("guild.total_members", len(member.guild.members)) + + @Cog.listener() + async def on_member_leave(self, member: Member) -> None: + """Update member count stat on member leave.""" + if member.guild.id != Guild.id: + return + + self.bot.stats.gauge("guild.total_members", len(member.guild.members)) + + @Cog.listener() + async def on_member_update(self, _before: Member, after: Member) -> None: + """Update presence estimates on member update.""" + if after.guild.id != Guild.id: + return + + if self.last_presence_update: + if (datetime.now() - self.last_presence_update).seconds < StatConf.presence_update_timeout: + return + + self.last_presence_update = datetime.now() + + online = 0 + idle = 0 + dnd = 0 + offline = 0 + + for member in after.guild.members: + if member.status is Status.online: + online += 1 + elif member.status is Status.dnd: + dnd += 1 + elif member.status is Status.idle: + idle += 1 + elif member.status is Status.offline: + offline += 1 + + self.bot.stats.gauge("guild.status.online", online) + self.bot.stats.gauge("guild.status.idle", idle) + self.bot.stats.gauge("guild.status.do_not_disturb", dnd) + self.bot.stats.gauge("guild.status.offline", offline) + + @loop(hours=1) + async def update_guild_boost(self) -> None: + """Post the server boost level and tier every hour.""" + await self.bot.wait_until_guild_available() + g = self.bot.get_guild(Guild.id) + self.bot.stats.gauge("boost.amount", g.premium_subscription_count) + self.bot.stats.gauge("boost.tier", g.premium_tier) + + def cog_unload(self) -> None: + """Stop the boost statistic task on unload of the Cog.""" + self.update_guild_boost.stop() + + +def setup(bot: Bot) -> None: + """Load the stats cog.""" + bot.add_cog(Stats(bot)) diff --git a/bot/cogs/tags.py b/bot/exts/info/tags.py index 5aa060f5e..ae95ac1ef 100644 --- a/bot/cogs/tags.py +++ b/bot/exts/info/tags.py @@ -4,7 +4,7 @@ import time from pathlib import Path from typing import Callable, Dict, Iterable, List, Optional -from discord import Colour, Embed +from discord import Colour, Embed, Member from discord.ext.commands import Cog, Context, group from bot import constants @@ -35,21 +35,37 @@ class Tags(Cog): @staticmethod def get_tags() -> dict: """Get all tags.""" - # Save all tags in memory. cache = {} - tag_files = Path("bot", "resources", "tags").iterdir() - for file in tag_files: - tag_title = file.stem - tag = { - "title": tag_title, - "embed": { - "description": file.read_text() + + base_path = Path("bot", "resources", "tags") + for file in base_path.glob("**/*"): + if file.is_file(): + tag_title = file.stem + tag = { + "title": tag_title, + "embed": { + "description": file.read_text(encoding="utf8"), + }, + "restricted_to": "developers", + "location": f"/bot/{file}" } - } - cache[tag_title] = tag + + # Convert to a list to allow negative indexing. + parents = list(file.relative_to(base_path).parents) + if len(parents) > 1: + # -1 would be '.' hence -2 is used as the index. + tag["restricted_to"] = parents[-2].name + + cache[tag_title] = tag + return cache @staticmethod + def check_accessibility(user: Member, tag: dict) -> bool: + """Check if user can access a tag.""" + return tag["restricted_to"].lower() in [role.name.lower() for role in user.roles] + + @staticmethod def _fuzzy_search(search: str, target: str) -> float: """A simple scoring algorithm based on how many letters are found / total, with order in mind.""" current, index = 0, 0 @@ -93,7 +109,7 @@ class Tags(Cog): return self._get_suggestions(tag_name) return found - def _get_tags_via_content(self, check: Callable[[Iterable], bool], keywords: str) -> list: + def _get_tags_via_content(self, check: Callable[[Iterable], bool], keywords: str, user: Member) -> list: """ Search for tags via contents. @@ -114,7 +130,8 @@ class Tags(Cog): matching_tags = [] for tag in self._cache.values(): - if check(query in tag['embed']['description'].casefold() for query in keywords_processed): + matches = (query in tag['embed']['description'].casefold() for query in keywords_processed) + if self.check_accessibility(user, tag) and check(matches): matching_tags.append(tag) return matching_tags @@ -152,7 +169,7 @@ class Tags(Cog): Only search for tags that has ALL the keywords. """ - matching_tags = self._get_tags_via_content(all, keywords) + matching_tags = self._get_tags_via_content(all, keywords, ctx.author) await self._send_matching_tags(ctx, keywords, matching_tags) @search_tag_content.command(name='any') @@ -162,7 +179,7 @@ class Tags(Cog): Search for tags that has ANY of the keywords. """ - matching_tags = self._get_tags_via_content(any, keywords or 'any') + matching_tags = self._get_tags_via_content(any, keywords or 'any', ctx.author) await self._send_matching_tags(ctx, keywords, matching_tags) @tags_group.command(name='get', aliases=('show', 'g')) @@ -198,7 +215,13 @@ class Tags(Cog): return if tag_name is not None: - founds = self._get_tag(tag_name) + temp_founds = self._get_tag(tag_name) + + founds = [] + + for found_tag in temp_founds: + if self.check_accessibility(ctx.author, found_tag): + founds.append(found_tag) if len(founds) == 1: tag = founds[0] @@ -207,10 +230,13 @@ class Tags(Cog): "time": time.time(), "channel": ctx.channel.id } + + self.bot.stats.incr(f"tags.usages.{tag['title'].replace('-', '_')}") + await wait_for_deletion( await ctx.send(embed=Embed.from_dict(tag['embed'])), [ctx.author.id], - client=self.bot + self.bot ) elif founds and len(tag_name) >= 3: await wait_for_deletion( @@ -221,7 +247,7 @@ class Tags(Cog): ) ), [ctx.author.id], - client=self.bot + self.bot ) else: @@ -234,7 +260,10 @@ class Tags(Cog): else: embed: Embed = Embed(title="**Current tags**") await LinePaginator.paginate( - sorted(f"**»** {tag['title']}" for tag in tags), + sorted( + f"**»** {tag['title']}" for tag in tags + if self.check_accessibility(ctx.author, tag) + ), ctx, embed, footer_text=FOOTER_TEXT, diff --git a/bot/exts/moderation/__init__.py b/bot/exts/moderation/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/bot/exts/moderation/__init__.py diff --git a/bot/cogs/defcon.py b/bot/exts/moderation/defcon.py index cc0f79fe8..caa6fb917 100644 --- a/bot/cogs/defcon.py +++ b/bot/exts/moderation/defcon.py @@ -6,12 +6,12 @@ from datetime import datetime, timedelta from enum import Enum from discord import Colour, Embed, Member -from discord.ext.commands import Cog, Context, group +from discord.ext.commands import Cog, Context, group, has_any_role from bot.bot import Bot -from bot.cogs.moderation import ModLog -from bot.constants import Channels, Colours, Emojis, Event, Icons, Roles -from bot.decorators import with_role +from bot.constants import Channels, Colours, Emojis, Event, Icons, MODERATION_ROLES, Roles +from bot.exts.moderation.modlog import ModLog +from bot.utils.messages import format_user log = logging.getLogger(__name__) @@ -81,7 +81,7 @@ class Defcon(Cog): else: self.enabled = False self.days = timedelta(days=0) - log.info(f"DEFCON disabled") + log.info("DEFCON disabled") await self.update_channel_topic() @@ -104,9 +104,10 @@ class Defcon(Cog): log.exception(f"Unable to send rejection message to user: {member}") await member.kick(reason="DEFCON active, user is too new") + self.bot.stats.incr("defcon.leaves") message = ( - f"{member} (`{member.id}`) was denied entry because their account is too new." + f"{format_user(member)} was denied entry because their account is too new." ) if not message_sent: @@ -118,13 +119,26 @@ class Defcon(Cog): ) @group(name='defcon', aliases=('dc',), invoke_without_command=True) - @with_role(Roles.admins, Roles.owners) + @has_any_role(*MODERATION_ROLES) async def defcon_group(self, ctx: Context) -> None: """Check the DEFCON status or run a subcommand.""" - await ctx.invoke(self.bot.get_command("help"), "defcon") + await ctx.send_help(ctx.command) async def _defcon_action(self, ctx: Context, days: int, action: Action) -> None: """Providing a structured way to do an defcon action.""" + try: + response = await self.bot.api_client.get('bot/bot-settings/defcon') + data = response['data'] + + if "enable_date" in data and action is Action.DISABLED: + enabled = datetime.fromisoformat(data["enable_date"]) + + delta = datetime.now() - enabled + + self.bot.stats.timing("defcon.enabled", delta) + except Exception: + pass + error = None try: await self.bot.api_client.put( @@ -135,6 +149,7 @@ class Defcon(Cog): # TODO: retrieve old days count 'days': days, 'enabled': action is not Action.DISABLED, + 'enable_date': datetime.now().isoformat() } } ) @@ -145,8 +160,10 @@ class Defcon(Cog): await ctx.send(self.build_defcon_msg(action, error)) await self.send_defcon_log(action, ctx.author, error) - @defcon_group.command(name='enable', aliases=('on', 'e')) - @with_role(Roles.admins, Roles.owners) + self.bot.stats.gauge("defcon.threshold", days) + + @defcon_group.command(name='enable', aliases=('on', 'e'), root_aliases=("defon",)) + @has_any_role(*MODERATION_ROLES) async def enable_command(self, ctx: Context) -> None: """ Enable DEFCON mode. Useful in a pinch, but be sure you know what you're doing! @@ -158,8 +175,8 @@ class Defcon(Cog): await self._defcon_action(ctx, days=0, action=Action.ENABLED) await self.update_channel_topic() - @defcon_group.command(name='disable', aliases=('off', 'd')) - @with_role(Roles.admins, Roles.owners) + @defcon_group.command(name='disable', aliases=('off', 'd'), root_aliases=("defoff",)) + @has_any_role(*MODERATION_ROLES) async def disable_command(self, ctx: Context) -> None: """Disable DEFCON mode. Useful in a pinch, but be sure you know what you're doing!""" self.enabled = False @@ -167,7 +184,7 @@ class Defcon(Cog): await self.update_channel_topic() @defcon_group.command(name='status', aliases=('s',)) - @with_role(Roles.admins, Roles.owners) + @has_any_role(*MODERATION_ROLES) async def status_command(self, ctx: Context) -> None: """Check the current status of DEFCON mode.""" embed = Embed( @@ -179,7 +196,7 @@ class Defcon(Cog): await ctx.send(embed=embed) @defcon_group.command(name='days') - @with_role(Roles.admins, Roles.owners) + @has_any_role(*MODERATION_ROLES) async def days_command(self, ctx: Context, days: int) -> None: """Set how old an account must be to join the server, in days, with DEFCON mode enabled.""" self.days = timedelta(days=days) diff --git a/bot/exts/moderation/dm_relay.py b/bot/exts/moderation/dm_relay.py new file mode 100644 index 000000000..14263e004 --- /dev/null +++ b/bot/exts/moderation/dm_relay.py @@ -0,0 +1,124 @@ +import logging +from typing import Optional + +import discord +from async_rediscache import RedisCache +from discord import Color +from discord.ext import commands +from discord.ext.commands import Cog + +from bot import constants +from bot.bot import Bot +from bot.converters import UserMentionOrID +from bot.utils.checks import in_whitelist_check +from bot.utils.messages import send_attachments +from bot.utils.webhooks import send_webhook + +log = logging.getLogger(__name__) + + +class DMRelay(Cog): + """Relay direct messages to and from the bot.""" + + # RedisCache[str, t.Union[discord.User.id, discord.Member.id]] + dm_cache = RedisCache() + + def __init__(self, bot: Bot): + self.bot = bot + self.webhook_id = constants.Webhooks.dm_log + self.webhook = None + self.bot.loop.create_task(self.fetch_webhook()) + + @commands.command(aliases=("reply",)) + async def send_dm(self, ctx: commands.Context, member: Optional[UserMentionOrID], *, message: str) -> None: + """ + Allows you to send a DM to a user from the bot. + + If `member` is not provided, it will send to the last user who DM'd the bot. + + This feature should be used extremely sparingly. Use ModMail if you need to have a serious + conversation with a user. This is just for responding to extraordinary DMs, having a little + fun with users, and telling people they are DMing the wrong bot. + + NOTE: This feature will be removed if it is overused. + """ + if not member: + user_id = await self.dm_cache.get("last_user") + member = ctx.guild.get_member(user_id) if user_id else None + + # If we still don't have a Member at this point, give up + if not member: + log.debug("This bot has never gotten a DM, or the RedisCache has been cleared.") + await ctx.message.add_reaction("❌") + return + + try: + await member.send(message) + except discord.errors.Forbidden: + log.debug("User has disabled DMs.") + await ctx.message.add_reaction("❌") + else: + await ctx.message.add_reaction("✅") + self.bot.stats.incr("dm_relay.dm_sent") + + async def fetch_webhook(self) -> None: + """Fetches the webhook object, so we can post to it.""" + await self.bot.wait_until_guild_available() + + try: + self.webhook = await self.bot.fetch_webhook(self.webhook_id) + except discord.HTTPException: + log.exception(f"Failed to fetch webhook with id `{self.webhook_id}`") + + @Cog.listener() + async def on_message(self, message: discord.Message) -> None: + """Relays the message's content and attachments to the dm_log channel.""" + # Only relay DMs from humans + if message.author.bot or message.guild or self.webhook is None: + return + + if message.clean_content: + await send_webhook( + webhook=self.webhook, + content=message.clean_content, + username=f"{message.author.display_name} ({message.author.id})", + avatar_url=message.author.avatar_url + ) + await self.dm_cache.set("last_user", message.author.id) + self.bot.stats.incr("dm_relay.dm_received") + + # Handle any attachments + if message.attachments: + try: + await send_attachments(message, self.webhook) + except (discord.errors.Forbidden, discord.errors.NotFound): + e = discord.Embed( + description=":x: **This message contained an attachment, but it could not be retrieved**", + color=Color.red() + ) + await send_webhook( + webhook=self.webhook, + embed=e, + username=f"{message.author.display_name} ({message.author.id})", + avatar_url=message.author.avatar_url + ) + except discord.HTTPException: + log.exception("Failed to send an attachment to the webhook") + + async def cog_check(self, ctx: commands.Context) -> bool: + """Only allow moderators to invoke the commands in this cog.""" + checks = [ + await commands.has_any_role(*constants.MODERATION_ROLES).predicate(ctx), + in_whitelist_check( + ctx, + channels=[constants.Channels.dm_log], + redirect=None, + fail_silently=True, + ) + ] + return all(checks) + + +def setup(bot: Bot) -> None: + """Load the DMRelay cog.""" + bot.add_cog(DMRelay(bot)) diff --git a/bot/exts/moderation/incidents.py b/bot/exts/moderation/incidents.py new file mode 100644 index 000000000..0e479d33f --- /dev/null +++ b/bot/exts/moderation/incidents.py @@ -0,0 +1,412 @@ +import asyncio +import logging +import typing as t +from datetime import datetime +from enum import Enum + +import discord +from discord.ext.commands import Cog + +from bot.bot import Bot +from bot.constants import Channels, Colours, Emojis, Guild, Webhooks +from bot.utils.messages import sub_clyde + +log = logging.getLogger(__name__) + +# Amount of messages for `crawl_task` to process at most on start-up - limited to 50 +# as in practice, there should never be this many messages, and if there are, +# something has likely gone very wrong +CRAWL_LIMIT = 50 + +# Seconds for `crawl_task` to sleep after adding reactions to a message +CRAWL_SLEEP = 2 + + +class Signal(Enum): + """ + Recognized incident status signals. + + This binds emoji to actions. The bot will only react to emoji linked here. + All other signals are seen as invalid. + """ + + ACTIONED = Emojis.incident_actioned + NOT_ACTIONED = Emojis.incident_unactioned + INVESTIGATING = Emojis.incident_investigating + + +# Reactions from non-mod roles will be removed +ALLOWED_ROLES: t.Set[int] = set(Guild.moderation_roles) + +# Message must have all of these emoji to pass the `has_signals` check +ALL_SIGNALS: t.Set[str] = {signal.value for signal in Signal} + +# An embed coupled with an optional file to be dispatched +# If the file is not None, the embed attempts to show it in its body +FileEmbed = t.Tuple[discord.Embed, t.Optional[discord.File]] + + +async def download_file(attachment: discord.Attachment) -> t.Optional[discord.File]: + """ + Download & return `attachment` file. + + If the download fails, the reason is logged and None will be returned. + 404 and 403 errors are only logged at debug level. + """ + log.debug(f"Attempting to download attachment: {attachment.filename}") + try: + return await attachment.to_file() + except (discord.NotFound, discord.Forbidden) as exc: + log.debug(f"Failed to download attachment: {exc}") + except Exception: + log.exception("Failed to download attachment") + + +async def make_embed(incident: discord.Message, outcome: Signal, actioned_by: discord.Member) -> FileEmbed: + """ + Create an embed representation of `incident` for the #incidents-archive channel. + + The name & discriminator of `actioned_by` and `outcome` will be presented in the + embed footer. Additionally, the embed is coloured based on `outcome`. + + The author of `incident` is not shown in the embed. It is assumed that this piece + of information will be relayed in other ways, e.g. webhook username. + + As mentions in embeds do not ping, we do not need to use `incident.clean_content`. + + If `incident` contains attachments, the first attachment will be downloaded and + returned alongside the embed. The embed attempts to display the attachment. + Should the download fail, we fallback on linking the `proxy_url`, which should + remain functional for some time after the original message is deleted. + """ + log.trace(f"Creating embed for {incident.id=}") + + if outcome is Signal.ACTIONED: + colour = Colours.soft_green + footer = f"Actioned by {actioned_by}" + else: + colour = Colours.soft_red + footer = f"Rejected by {actioned_by}" + + embed = discord.Embed( + description=incident.content, + timestamp=datetime.utcnow(), + colour=colour, + ) + embed.set_footer(text=footer, icon_url=actioned_by.avatar_url) + + if incident.attachments: + attachment = incident.attachments[0] # User-sent messages can only contain one attachment + file = await download_file(attachment) + + if file is not None: + embed.set_image(url=f"attachment://{attachment.filename}") # Embed displays the attached file + else: + embed.set_author(name="[Failed to relay attachment]", url=attachment.proxy_url) # Embed links the file + else: + file = None + + return embed, file + + +def is_incident(message: discord.Message) -> bool: + """True if `message` qualifies as an incident, False otherwise.""" + conditions = ( + message.channel.id == Channels.incidents, # Message sent in #incidents + not message.author.bot, # Not by a bot + not message.content.startswith("#"), # Doesn't start with a hash + not message.pinned, # And isn't header + ) + return all(conditions) + + +def own_reactions(message: discord.Message) -> t.Set[str]: + """Get the set of reactions placed on `message` by the bot itself.""" + return {str(reaction.emoji) for reaction in message.reactions if reaction.me} + + +def has_signals(message: discord.Message) -> bool: + """True if `message` already has all `Signal` reactions, False otherwise.""" + return ALL_SIGNALS.issubset(own_reactions(message)) + + +async def add_signals(incident: discord.Message) -> None: + """ + Add `Signal` member emoji to `incident` as reactions. + + If the emoji has already been placed on `incident` by the bot, it will be skipped. + """ + existing_reacts = own_reactions(incident) + + for signal_emoji in Signal: + if signal_emoji.value in existing_reacts: # This would not raise, but it is a superfluous API call + log.trace(f"Skipping emoji as it's already been placed: {signal_emoji}") + else: + log.trace(f"Adding reaction: {signal_emoji}") + await incident.add_reaction(signal_emoji.value) + + +class Incidents(Cog): + """ + Automation for the #incidents channel. + + This cog does not provide a command API, it only reacts to the following events. + + On start-up: + * Crawl #incidents and add missing `Signal` emoji where appropriate + * This is to retro-actively add the available options for messages which + were sent while the bot wasn't listening + * Pinned messages and message starting with # do not qualify as incidents + * See: `crawl_incidents` + + On message: + * Add `Signal` member emoji if message qualifies as an incident + * Ignore messages starting with # + * Use this if verbal communication is necessary + * Each such message must be deleted manually once appropriate + * See: `on_message` + + On reaction: + * Remove reaction if not permitted + * User does not have any of the roles in `ALLOWED_ROLES` + * Used emoji is not a `Signal` member + * If `Signal.ACTIONED` or `Signal.NOT_ACTIONED` were chosen, attempt to + relay the incident message to #incidents-archive + * If relay successful, delete original message + * See: `on_raw_reaction_add` + + Please refer to function docstrings for implementation details. + """ + + def __init__(self, bot: Bot) -> None: + """Prepare `event_lock` and schedule `crawl_task` on start-up.""" + self.bot = bot + + self.event_lock = asyncio.Lock() + self.crawl_task = self.bot.loop.create_task(self.crawl_incidents()) + + async def crawl_incidents(self) -> None: + """ + Crawl #incidents and add missing emoji where necessary. + + This is to catch-up should an incident be reported while the bot wasn't listening. + After adding each reaction, we take a short break to avoid drowning in ratelimits. + + Once this task is scheduled, listeners that change messages should await it. + The crawl assumes that the channel history doesn't change as we go over it. + + Behaviour is configured by: `CRAWL_LIMIT`, `CRAWL_SLEEP`. + """ + await self.bot.wait_until_guild_available() + incidents: discord.TextChannel = self.bot.get_channel(Channels.incidents) + + log.debug(f"Crawling messages in #incidents: {CRAWL_LIMIT=}, {CRAWL_SLEEP=}") + async for message in incidents.history(limit=CRAWL_LIMIT): + + if not is_incident(message): + log.trace(f"Skipping message {message.id}: not an incident") + continue + + if has_signals(message): + log.trace(f"Skipping message {message.id}: already has all signals") + continue + + await add_signals(message) + await asyncio.sleep(CRAWL_SLEEP) + + log.debug("Crawl task finished!") + + async def archive(self, incident: discord.Message, outcome: Signal, actioned_by: discord.Member) -> bool: + """ + Relay an embed representation of `incident` to the #incidents-archive channel. + + The following pieces of information are relayed: + * Incident message content (as embed description) + * Incident attachment (if image, shown in archive embed) + * Incident author name (as webhook author) + * Incident author avatar (as webhook avatar) + * Resolution signal `outcome` (as embed colour & footer) + * Moderator `actioned_by` (name & discriminator shown in footer) + + If `incident` contains an attachment, we try to add it to the archive embed. There is + no handing of extensions / file types - we simply dispatch the attachment file with the + webhook, and try to display it in the embed. Testing indicates that if the attachment + cannot be displayed (e.g. a text file), it's invisible in the embed, with no error. + + Return True if the relay finishes successfully. If anything goes wrong, meaning + not all information was relayed, return False. This signals that the original + message is not safe to be deleted, as we will lose some information. + """ + log.info(f"Archiving incident: {incident.id} (outcome: {outcome}, actioned by: {actioned_by})") + embed, attachment_file = await make_embed(incident, outcome, actioned_by) + + try: + webhook = await self.bot.fetch_webhook(Webhooks.incidents_archive) + await webhook.send( + embed=embed, + username=sub_clyde(incident.author.name), + avatar_url=incident.author.avatar_url, + file=attachment_file, + ) + except Exception: + log.exception(f"Failed to archive incident {incident.id} to #incidents-archive") + return False + else: + log.trace("Message archived successfully!") + return True + + def make_confirmation_task(self, incident: discord.Message, timeout: int = 5) -> asyncio.Task: + """ + Create a task to wait `timeout` seconds for `incident` to be deleted. + + If `timeout` passes, this will raise `asyncio.TimeoutError`, signaling that we haven't + been able to confirm that the message was deleted. + """ + log.trace(f"Confirmation task will wait {timeout=} seconds for {incident.id=} to be deleted") + + def check(payload: discord.RawReactionActionEvent) -> bool: + return payload.message_id == incident.id + + coroutine = self.bot.wait_for(event="raw_message_delete", check=check, timeout=timeout) + return self.bot.loop.create_task(coroutine) + + async def process_event(self, reaction: str, incident: discord.Message, member: discord.Member) -> None: + """ + Process a `reaction_add` event in #incidents. + + First, we check that the reaction is a recognized `Signal` member, and that it was sent by + a permitted user (at least one role in `ALLOWED_ROLES`). If not, the reaction is removed. + + If the reaction was either `Signal.ACTIONED` or `Signal.NOT_ACTIONED`, we attempt to relay + the report to #incidents-archive. If successful, the original message is deleted. + + We do not release `event_lock` until we receive the corresponding `message_delete` event. + This ensures that if there is a racing event awaiting the lock, it will fail to find the + message, and will abort. There is a `timeout` to ensure that this doesn't hold the lock + forever should something go wrong. + """ + members_roles: t.Set[int] = {role.id for role in member.roles} + if not members_roles & ALLOWED_ROLES: # Intersection is truthy on at least 1 common element + log.debug(f"Removing invalid reaction: user {member} is not permitted to send signals") + await incident.remove_reaction(reaction, member) + return + + try: + signal = Signal(reaction) + except ValueError: + log.debug(f"Removing invalid reaction: emoji {reaction} is not a valid signal") + await incident.remove_reaction(reaction, member) + return + + log.trace(f"Received signal: {signal}") + + if signal not in (Signal.ACTIONED, Signal.NOT_ACTIONED): + log.debug("Reaction was valid, but no action is currently defined for it") + return + + relay_successful = await self.archive(incident, signal, actioned_by=member) + if not relay_successful: + log.trace("Original message will not be deleted as we failed to relay it to the archive") + return + + timeout = 5 # Seconds + confirmation_task = self.make_confirmation_task(incident, timeout) + + log.trace("Deleting original message") + await incident.delete() + + log.trace(f"Awaiting deletion confirmation: {timeout=} seconds") + try: + await confirmation_task + except asyncio.TimeoutError: + log.info(f"Did not receive incident deletion confirmation within {timeout} seconds!") + else: + log.trace("Deletion was confirmed") + + async def resolve_message(self, message_id: int) -> t.Optional[discord.Message]: + """ + Get `discord.Message` for `message_id` from cache, or API. + + We first look into the local cache to see if the message is present. + + If not, we try to fetch the message from the API. This is necessary for messages + which were sent before the bot's current session. + + In an edge-case, it is also possible that the message was already deleted, and + the API will respond with a 404. In such a case, None will be returned. + This signals that the event for `message_id` should be ignored. + """ + await self.bot.wait_until_guild_available() # First make sure that the cache is ready + log.trace(f"Resolving message for: {message_id=}") + message: t.Optional[discord.Message] = self.bot._connection._get_message(message_id) + + if message is not None: + log.trace("Message was found in cache") + return message + + log.trace("Message not found, attempting to fetch") + try: + message = await self.bot.get_channel(Channels.incidents).fetch_message(message_id) + except discord.NotFound: + log.trace("Message doesn't exist, it was likely already relayed") + except Exception: + log.exception(f"Failed to fetch message {message_id}!") + else: + log.trace("Message fetched successfully!") + return message + + @Cog.listener() + async def on_raw_reaction_add(self, payload: discord.RawReactionActionEvent) -> None: + """ + Pre-process `payload` and pass it to `process_event` if appropriate. + + We abort instantly if `payload` doesn't relate to a message sent in #incidents, + or if it was sent by a bot. + + If `payload` relates to a message in #incidents, we first ensure that `crawl_task` has + finished, to make sure we don't mutate channel state as we're crawling it. + + Next, we acquire `event_lock` - to prevent racing, events are processed one at a time. + + Once we have the lock, the `discord.Message` object for this event must be resolved. + If the lock was previously held by an event which successfully relayed the incident, + this will fail and we abort the current event. + + Finally, with both the lock and the `discord.Message` instance in our hands, we delegate + to `process_event` to handle the event. + + The justification for using a raw listener is the need to receive events for messages + which were not cached in the current session. As a result, a certain amount of + complexity is introduced, but at the moment this doesn't appear to be avoidable. + """ + if payload.channel_id != Channels.incidents or payload.member.bot: + return + + log.trace(f"Received reaction add event in #incidents, waiting for crawler: {self.crawl_task.done()=}") + await self.crawl_task + + log.trace(f"Acquiring event lock: {self.event_lock.locked()=}") + async with self.event_lock: + message = await self.resolve_message(payload.message_id) + + if message is None: + log.debug("Listener will abort as related message does not exist!") + return + + if not is_incident(message): + log.debug("Ignoring event for a non-incident message") + return + + await self.process_event(str(payload.emoji), message, payload.member) + log.trace("Releasing event lock") + + @Cog.listener() + async def on_message(self, message: discord.Message) -> None: + """Pass `message` to `add_signals` if and only if it satisfies `is_incident`.""" + if is_incident(message): + await add_signals(message) + + +def setup(bot: Bot) -> None: + """Load the Incidents cog.""" + bot.add_cog(Incidents(bot)) diff --git a/bot/exts/moderation/infraction/__init__.py b/bot/exts/moderation/infraction/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/bot/exts/moderation/infraction/__init__.py diff --git a/bot/cogs/moderation/scheduler.py b/bot/exts/moderation/infraction/_scheduler.py index 917697be9..814b17830 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/exts/moderation/infraction/_scheduler.py @@ -1,4 +1,3 @@ -import asyncio import logging import textwrap import typing as t @@ -13,25 +12,28 @@ from discord.ext.commands import Context from bot import constants from bot.api import ResponseCodeError from bot.bot import Bot -from bot.constants import Colours, STAFF_CHANNELS -from bot.utils import time -from bot.utils.scheduling import Scheduler -from . import utils -from .modlog import ModLog -from .utils import UserSnowflake +from bot.constants import Colours, MODERATION_CHANNELS +from bot.exts.moderation.infraction import _utils +from bot.exts.moderation.infraction._utils import UserSnowflake +from bot.exts.moderation.modlog import ModLog +from bot.utils import messages, scheduling, time log = logging.getLogger(__name__) -class InfractionScheduler(Scheduler): +class InfractionScheduler: """Handles the application, pardoning, and expiration of infractions.""" def __init__(self, bot: Bot, supported_infractions: t.Container[str]): - super().__init__() - self.bot = bot + self.scheduler = scheduling.Scheduler(self.__class__.__name__) + self.bot.loop.create_task(self.reschedule_infractions(supported_infractions)) + def cog_unload(self) -> None: + """Cancel scheduled tasks.""" + self.scheduler.cancel_all() + @property def mod_log(self) -> ModLog: """Get the currently loaded ModLog cog instance.""" @@ -49,11 +51,11 @@ class InfractionScheduler(Scheduler): ) for infraction in infractions: if infraction["expires_at"] is not None and infraction["type"] in supported_infractions: - self.schedule_task(infraction["id"], infraction) + self.schedule_expiration(infraction) async def reapply_infraction( self, - infraction: utils.Infraction, + infraction: _utils.Infraction, apply_coro: t.Optional[t.Awaitable] ) -> None: """Reapply an infraction if it's still active or deactivate it if less than 60 sec left.""" @@ -77,13 +79,13 @@ class InfractionScheduler(Scheduler): async def apply_infraction( self, ctx: Context, - infraction: utils.Infraction, + infraction: _utils.Infraction, user: UserSnowflake, action_coro: t.Optional[t.Awaitable] = None ) -> None: """Apply an infraction to the user, log the infraction, and optionally notify the user.""" infr_type = infraction["type"] - icon = utils.INFRACTION_ICONS[infr_type][0] + icon = _utils.INFRACTION_ICONS[infr_type][0] reason = infraction["reason"] expiry = time.format_infraction_with_duration(infraction["expires_at"]) id_ = infraction['id'] @@ -91,7 +93,7 @@ class InfractionScheduler(Scheduler): log.trace(f"Applying {infr_type} infraction #{id_} to {user}.") # Default values for the confirmation message and mod log. - confirm_msg = f":ok_hand: applied" + confirm_msg = ":ok_hand: applied" # Specifying an expiry for a note or warning makes no sense. if infr_type in ("note", "warning"): @@ -101,11 +103,16 @@ class InfractionScheduler(Scheduler): dm_result = "" dm_log_text = "" - expiry_log_text = f"Expires: {expiry}" if expiry else "" + expiry_log_text = f"\nExpires: {expiry}" if expiry else "" log_title = "applied" log_content = None + failed = False # DM the user about the infraction if it's not a shadow/hidden infraction. + # This needs to happen before we apply the infraction, as the bot cannot + # send DMs to user that it doesn't share a guild with. If we were to + # apply kick/ban infractions first, this would mean that we'd make it + # impossible for us to deliver a DM. See python-discord/bot#982. if not infraction["hidden"]: dm_result = f"{constants.Emojis.failmail} " dm_log_text = "\nDM: **Failed**" @@ -118,22 +125,21 @@ class InfractionScheduler(Scheduler): log.error(f"Failed to DM {user.id}: could not fetch user (status {e.status})") else: # Accordingly display whether the user was successfully notified via DM. - if await utils.notify_infraction(user, infr_type, expiry, reason, icon): + if await _utils.notify_infraction(user, infr_type, expiry, reason, icon): dm_result = ":incoming_envelope: " dm_log_text = "\nDM: Sent" + end_msg = "" if infraction["actor"] == self.bot.user.id: log.trace( f"Infraction #{id_} actor is bot; including the reason in the confirmation message." ) - - end_msg = f" (reason: {infraction['reason']})" - elif ctx.channel.id not in STAFF_CHANNELS: + if reason: + end_msg = f" (reason: {textwrap.shorten(reason, width=1500, placeholder='...')})" + elif ctx.channel.id not in MODERATION_CHANNELS: log.trace( - f"Infraction #{id_} context is not in a staff channel; omitting infraction count." + f"Infraction #{id_} context is not in a mod channel; omitting infraction count." ) - - end_msg = "" else: log.trace(f"Fetching total infraction count for {user}.") @@ -151,10 +157,11 @@ class InfractionScheduler(Scheduler): await action_coro if expiry: # Schedule the expiration of the infraction. - self.schedule_task(infraction["id"], infraction) + self.schedule_expiration(infraction) except discord.HTTPException as e: # Accordingly display that applying the infraction failed. - confirm_msg = f":x: failed to apply" + # Don't use ctx.message.author; antispam only patches ctx.author. + confirm_msg = ":x: failed to apply" expiry_msg = "" log_content = ctx.author.mention log_title = "failed to apply" @@ -164,14 +171,26 @@ class InfractionScheduler(Scheduler): log.warning(f"{log_msg}: bot lacks permissions.") else: log.exception(log_msg) + failed = True + + if failed: + log.trace(f"Deleted infraction {infraction['id']} from database because applying infraction failed.") + try: + await self.bot.api_client.delete(f"bot/infractions/{id_}") + except ResponseCodeError as e: + confirm_msg += " and failed to delete" + log_title += " and failed to delete" + log.error(f"Deletion of {infr_type} infraction #{id_} failed with error code {e.status}.") + infr_message = "" + else: + infr_message = f" **{infr_type}** to {user.mention}{expiry_msg}{end_msg}" # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} confirmation message.") - await ctx.send( - f"{dm_result}{confirm_msg} **{infr_type}** to {user.mention}{expiry_msg}{end_msg}." - ) + await ctx.send(f"{dm_result}{confirm_msg}{infr_message}.") # Send a log message to the mod log. + # Don't use ctx.message.author for the actor; antispam only patches ctx.author. log.trace(f"Sending apply mod log for infraction #{id_}.") await self.mod_log.send_log_message( icon_url=icon, @@ -179,10 +198,9 @@ class InfractionScheduler(Scheduler): title=f"Infraction {log_title}: {infr_type}", thumbnail=user.avatar_url_as(static_format="png"), text=textwrap.dedent(f""" - Member: {user.mention} (`{user.id}`) - Actor: {ctx.message.author}{dm_log_text} + Member: {messages.format_user(user)} + Actor: {ctx.author.mention}{dm_log_text}{expiry_log_text} Reason: {reason} - {expiry_log_text} """), content=log_content, footer=f"ID {infraction['id']}" @@ -190,8 +208,19 @@ class InfractionScheduler(Scheduler): log.info(f"Applied {infr_type} infraction #{id_} to {user}.") - async def pardon_infraction(self, ctx: Context, infr_type: str, user: UserSnowflake) -> None: - """Prematurely end an infraction for a user and log the action in the mod log.""" + async def pardon_infraction( + self, + ctx: Context, + infr_type: str, + user: UserSnowflake, + send_msg: bool = True + ) -> None: + """ + Prematurely end an infraction for a user and log the action in the mod log. + + If `send_msg` is True, then a pardoning confirmation message will be sent to + the context channel. Otherwise, no such message will be sent. + """ log.trace(f"Pardoning {infr_type} infraction for {user}.") # Check the current active infraction @@ -213,48 +242,12 @@ class InfractionScheduler(Scheduler): # Deactivate the infraction and cancel its scheduled expiration task. log_text = await self.deactivate_infraction(response[0], send_log=False) - log_text["Member"] = f"{user.mention}(`{user.id}`)" - log_text["Actor"] = str(ctx.message.author) + log_text["Member"] = messages.format_user(user) + log_text["Actor"] = ctx.author.mention log_content = None id_ = response[0]['id'] footer = f"ID: {id_}" - # If multiple active infractions were found, mark them as inactive in the database - # and cancel their expiration tasks. - if len(response) > 1: - log.info( - f"Found more than one active {infr_type} infraction for user {user.id}; " - "deactivating the extra active infractions too." - ) - - footer = f"Infraction IDs: {', '.join(str(infr['id']) for infr in response)}" - - log_note = f"Found multiple **active** {infr_type} infractions in the database." - if "Note" in log_text: - log_text["Note"] = f" {log_note}" - else: - log_text["Note"] = log_note - - # deactivate_infraction() is not called again because: - # 1. Discord cannot store multiple active bans or assign multiples of the same role - # 2. It would send a pardon DM for each active infraction, which is redundant - for infraction in response[1:]: - id_ = infraction['id'] - try: - # Mark infraction as inactive in the database. - await self.bot.api_client.patch( - f"bot/infractions/{id_}", - json={"active": False} - ) - except ResponseCodeError: - log.exception(f"Failed to deactivate infraction #{id_} ({infr_type})") - # This is simpler and cleaner than trying to concatenate all the errors. - log_text["Failure"] = "See bot's logs for details." - - # Cancel pending expiration task. - if infraction["expires_at"] is not None: - self.cancel_task(infraction["id"]) - # Accordingly display whether the user was successfully notified via DM. dm_emoji = "" if log_text.get("DM") == "Sent": @@ -270,21 +263,25 @@ class InfractionScheduler(Scheduler): log.warning(f"Failed to pardon {infr_type} infraction #{id_} for {user}.") else: - confirm_msg = f":ok_hand: pardoned" + confirm_msg = ":ok_hand: pardoned" log_title = "pardoned" log.info(f"Pardoned {infr_type} infraction #{id_} for {user}.") # Send a confirmation message to the invoking context. - log.trace(f"Sending infraction #{id_} pardon confirmation message.") - await ctx.send( - f"{dm_emoji}{confirm_msg} infraction **{infr_type}** for {user.mention}. " - f"{log_text.get('Failure', '')}" - ) + if send_msg: + log.trace(f"Sending infraction #{id_} pardon confirmation message.") + await ctx.send( + f"{dm_emoji}{confirm_msg} infraction **{infr_type}** for {user.mention}. " + f"{log_text.get('Failure', '')}" + ) + + # Move reason to end of entry to avoid cutting out some keys + log_text["Reason"] = log_text.pop("Reason") # Send a log message to the mod log. await self.mod_log.send_log_message( - icon_url=utils.INFRACTION_ICONS[infr_type][1], + icon_url=_utils.INFRACTION_ICONS[infr_type][1], colour=Colours.soft_green, title=f"Infraction {log_title}: {infr_type}", thumbnail=user.avatar_url_as(static_format="png"), @@ -295,7 +292,7 @@ class InfractionScheduler(Scheduler): async def deactivate_infraction( self, - infraction: utils.Infraction, + infraction: _utils.Infraction, send_log: bool = True ) -> t.Dict[str, str]: """ @@ -324,7 +321,7 @@ class InfractionScheduler(Scheduler): log_content = None log_text = { "Member": f"<@{user_id}>", - "Actor": str(self.bot.get_user(actor) or actor), + "Actor": f"<@{actor}>", "Reason": infraction["reason"], "Created": created, } @@ -341,7 +338,7 @@ class InfractionScheduler(Scheduler): ) except discord.Forbidden: log.warning(f"Failed to deactivate infraction #{id_} ({type_}): bot lacks permissions.") - log_text["Failure"] = f"The bot lacks permissions to do this (role hierarchy?)" + log_text["Failure"] = "The bot lacks permissions to do this (role hierarchy?)" log_content = mod_role.mention except discord.HTTPException as e: log.exception(f"Failed to deactivate infraction #{id_} ({type_})") @@ -386,31 +383,33 @@ class InfractionScheduler(Scheduler): # Cancel the expiration task. if infraction["expires_at"] is not None: - self.cancel_task(infraction["id"]) + self.scheduler.cancel(infraction["id"]) # Send a log message to the mod log. if send_log: - log_title = f"expiration failed" if "Failure" in log_text else "expired" + log_title = "expiration failed" if "Failure" in log_text else "expired" user = self.bot.get_user(user_id) avatar = user.avatar_url_as(static_format="png") if user else None + # Move reason to end so when reason is too long, this is not gonna cut out required items. + log_text["Reason"] = log_text.pop("Reason") + log.trace(f"Sending deactivation mod log for infraction #{id_}.") await self.mod_log.send_log_message( - icon_url=utils.INFRACTION_ICONS[type_][1], + icon_url=_utils.INFRACTION_ICONS[type_][1], colour=Colours.soft_green, title=f"Infraction {log_title}: {type_}", thumbnail=avatar, text="\n".join(f"{k}: {v}" for k, v in log_text.items()), footer=f"ID: {id_}", content=log_content, - ) return log_text @abstractmethod - async def _pardon_action(self, infraction: utils.Infraction) -> t.Optional[t.Dict[str, str]]: + async def _pardon_action(self, infraction: _utils.Infraction) -> t.Optional[t.Dict[str, str]]: """ Execute deactivation steps specific to the infraction's type and return a log dict. @@ -418,7 +417,7 @@ class InfractionScheduler(Scheduler): """ raise NotImplementedError - async def _scheduled_task(self, infraction: utils.Infraction) -> None: + def schedule_expiration(self, infraction: _utils.Infraction) -> None: """ Marks an infraction expired after the delay from time of scheduling to time of expiration. @@ -426,8 +425,4 @@ class InfractionScheduler(Scheduler): expiration task is cancelled. """ expiry = dateutil.parser.isoparse(infraction["expires_at"]).replace(tzinfo=None) - await time.wait_until(expiry) - - # Because deactivate_infraction() explicitly cancels this scheduled task, it is shielded - # to avoid prematurely cancelling itself. - await asyncio.shield(self.deactivate_infraction(infraction)) + self.scheduler.schedule_at(expiry, infraction["id"], self.deactivate_infraction(infraction)) diff --git a/bot/cogs/moderation/utils.py b/bot/exts/moderation/infraction/_utils.py index 3598f3b1f..1d91964f1 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/exts/moderation/infraction/_utils.py @@ -1,5 +1,4 @@ import logging -import textwrap import typing as t from datetime import datetime @@ -28,6 +27,18 @@ UserObject = t.Union[discord.Member, discord.User] UserSnowflake = t.Union[UserObject, discord.Object] Infraction = t.Dict[str, t.Union[str, int, bool]] +APPEAL_EMAIL = "[email protected]" + +INFRACTION_TITLE = f"Please review our rules over at {RULES_URL}" +INFRACTION_APPEAL_FOOTER = f"To appeal this infraction, send an e-mail to {APPEAL_EMAIL}" +INFRACTION_AUTHOR_NAME = "Infraction information" + +INFRACTION_DESCRIPTION_TEMPLATE = ( + "**Type:** {type}\n" + "**Expires:** {expires}\n" + "**Reason:** {reason}\n" +) + async def post_user(ctx: Context, user: UserSnowflake) -> t.Optional[dict]: """ @@ -41,7 +52,6 @@ async def post_user(ctx: Context, user: UserSnowflake) -> t.Optional[dict]: log.debug("The user being added to the DB is not a Member or User object.") payload = { - 'avatar_hash': getattr(user, 'avatar', 0), 'discriminator': int(getattr(user, 'discriminator', 0)), 'id': user.id, 'in_guild': False, @@ -71,7 +81,7 @@ async def post_infraction( log.trace(f"Posting {infr_type} infraction for {user} to the API.") payload = { - "actor": ctx.message.author.id, + "actor": ctx.author.id, # Don't use ctx.message.author; antispam only patches ctx.author. "hidden": hidden, "reason": reason, "type": infr_type, @@ -97,8 +107,19 @@ async def post_infraction( return -async def has_active_infraction(ctx: Context, user: UserSnowflake, infr_type: str) -> bool: - """Checks if a user already has an active infraction of the given type.""" +async def get_active_infraction( + ctx: Context, + user: UserSnowflake, + infr_type: str, + send_msg: bool = True +) -> t.Optional[dict]: + """ + Retrieves an active infraction of the given type for the user. + + If `send_msg` is True and the user has an active infraction matching the `infr_type` parameter, + then a message for the moderator will be sent to the context channel letting them know. + Otherwise, no message will be sent. + """ log.trace(f"Checking if {user} has active infractions of type {infr_type}.") active_infractions = await ctx.bot.api_client.get( @@ -110,15 +131,16 @@ async def has_active_infraction(ctx: Context, user: UserSnowflake, infr_type: st } ) if active_infractions: - log.trace(f"{user} has active infractions of type {infr_type}.") - await ctx.send( - f":x: According to my records, this user already has a {infr_type} infraction. " - f"See infraction **#{active_infractions[0]['id']}**." - ) - return True + # Checks to see if the moderator should be told there is an active infraction + if send_msg: + log.trace(f"{user} has active infractions of type {infr_type}.") + await ctx.send( + f":x: According to my records, this user already has a {infr_type} infraction. " + f"See infraction **#{active_infractions[0]['id']}**." + ) + return active_infractions[0] else: log.trace(f"{user} does not have active infractions of type {infr_type}.") - return False async def notify_infraction( @@ -131,23 +153,27 @@ async def notify_infraction( """DM a user about their new infraction and return True if the DM is successful.""" log.trace(f"Sending {user} a DM about their {infr_type} infraction.") + text = INFRACTION_DESCRIPTION_TEMPLATE.format( + type=infr_type.capitalize(), + expires=expires_at or "N/A", + reason=reason or "No reason provided." + ) + + # For case when other fields than reason is too long and this reach limit, then force-shorten string + if len(text) > 2048: + text = f"{text[:2045]}..." + embed = discord.Embed( - description=textwrap.dedent(f""" - **Type:** {infr_type.capitalize()} - **Expires:** {expires_at or "N/A"} - **Reason:** {reason or "No reason provided."} - """), + description=text, colour=Colours.soft_red ) - embed.set_author(name="Infraction information", icon_url=icon_url, url=RULES_URL) - embed.title = f"Please review our rules over at {RULES_URL}" + embed.set_author(name=INFRACTION_AUTHOR_NAME, icon_url=icon_url, url=RULES_URL) + embed.title = INFRACTION_TITLE embed.url = RULES_URL if infr_type in APPEALABLE_INFRACTIONS: - embed.set_footer( - text="To appeal this infraction, send an e-mail to [email protected]" - ) + embed.set_footer(text=INFRACTION_APPEAL_FOOTER) return await send_private_embed(user, embed) diff --git a/bot/cogs/moderation/infractions.py b/bot/exts/moderation/infraction/infractions.py index efa19f59e..ef6f6e3c6 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/exts/moderation/infraction/infractions.py @@ -1,4 +1,5 @@ import logging +import textwrap import typing as t import discord @@ -11,10 +12,10 @@ from bot.bot import Bot from bot.constants import Event from bot.converters import Expiry, FetchedMember from bot.decorators import respect_role_hierarchy -from bot.utils.checks import with_role_check -from . import utils -from .scheduler import InfractionScheduler -from .utils import UserSnowflake +from bot.exts.moderation.infraction import _utils +from bot.exts.moderation.infraction._scheduler import InfractionScheduler +from bot.exts.moderation.infraction._utils import UserSnowflake +from bot.utils.messages import format_user log = logging.getLogger(__name__) @@ -52,21 +53,21 @@ class Infractions(InfractionScheduler, commands.Cog): # region: Permanent infractions @command() - async def warn(self, ctx: Context, user: Member, *, reason: str = None) -> None: + async def warn(self, ctx: Context, user: Member, *, reason: t.Optional[str] = None) -> None: """Warn a user for the given reason.""" - infraction = await utils.post_infraction(ctx, user, "warning", reason, active=False) + infraction = await _utils.post_infraction(ctx, user, "warning", reason, active=False) if infraction is None: return await self.apply_infraction(ctx, infraction, user) @command() - async def kick(self, ctx: Context, user: Member, *, reason: str = None) -> None: + async def kick(self, ctx: Context, user: Member, *, reason: t.Optional[str] = None) -> None: """Kick a user for the given reason.""" - await self.apply_kick(ctx, user, reason, active=False) + await self.apply_kick(ctx, user, reason) @command() - async def ban(self, ctx: Context, user: FetchedMember, *, reason: str = None) -> None: + async def ban(self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None) -> None: """Permanently ban a user for the given reason and stop watching them with Big Brother.""" await self.apply_ban(ctx, user, reason) @@ -74,7 +75,7 @@ class Infractions(InfractionScheduler, commands.Cog): # region: Temporary infractions @command(aliases=["mute"]) - async def tempmute(self, ctx: Context, user: Member, duration: Expiry, *, reason: str = None) -> None: + async def tempmute(self, ctx: Context, user: Member, duration: Expiry, *, reason: t.Optional[str] = None) -> None: """ Temporarily mute a user for the given reason and duration. @@ -93,7 +94,14 @@ class Infractions(InfractionScheduler, commands.Cog): await self.apply_mute(ctx, user, reason, expires_at=duration) @command() - async def tempban(self, ctx: Context, user: FetchedMember, duration: Expiry, *, reason: str = None) -> None: + async def tempban( + self, + ctx: Context, + user: FetchedMember, + duration: Expiry, + *, + reason: t.Optional[str] = None + ) -> None: """ Temporarily ban a user for the given reason and duration. @@ -115,21 +123,21 @@ class Infractions(InfractionScheduler, commands.Cog): # region: Permanent shadow infractions @command(hidden=True) - async def note(self, ctx: Context, user: FetchedMember, *, reason: str = None) -> None: + async def note(self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None) -> None: """Create a private note for a user with the given reason without notifying the user.""" - infraction = await utils.post_infraction(ctx, user, "note", reason, hidden=True, active=False) + infraction = await _utils.post_infraction(ctx, user, "note", reason, hidden=True, active=False) if infraction is None: return await self.apply_infraction(ctx, infraction, user) @command(hidden=True, aliases=['shadowkick', 'skick']) - async def shadow_kick(self, ctx: Context, user: Member, *, reason: str = None) -> None: + async def shadow_kick(self, ctx: Context, user: Member, *, reason: t.Optional[str] = None) -> None: """Kick a user for the given reason without notifying the user.""" - await self.apply_kick(ctx, user, reason, hidden=True, active=False) + await self.apply_kick(ctx, user, reason, hidden=True) @command(hidden=True, aliases=['shadowban', 'sban']) - async def shadow_ban(self, ctx: Context, user: FetchedMember, *, reason: str = None) -> None: + async def shadow_ban(self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None) -> None: """Permanently ban a user for the given reason without notifying the user.""" await self.apply_ban(ctx, user, reason, hidden=True) @@ -137,7 +145,13 @@ class Infractions(InfractionScheduler, commands.Cog): # region: Temporary shadow infractions @command(hidden=True, aliases=["shadowtempmute, stempmute", "shadowmute", "smute"]) - async def shadow_tempmute(self, ctx: Context, user: Member, duration: Expiry, *, reason: str = None) -> None: + async def shadow_tempmute( + self, ctx: Context, + user: Member, + duration: Expiry, + *, + reason: t.Optional[str] = None + ) -> None: """ Temporarily mute a user for the given reason and duration without notifying the user. @@ -162,7 +176,7 @@ class Infractions(InfractionScheduler, commands.Cog): user: FetchedMember, duration: Expiry, *, - reason: str = None + reason: t.Optional[str] = None ) -> None: """ Temporarily ban a user for the given reason and duration without notifying the user. @@ -197,12 +211,12 @@ class Infractions(InfractionScheduler, commands.Cog): # endregion # region: Base apply functions - async def apply_mute(self, ctx: Context, user: Member, reason: str, **kwargs) -> None: + async def apply_mute(self, ctx: Context, user: Member, reason: t.Optional[str], **kwargs) -> None: """Apply a mute infraction with kwargs passed to `post_infraction`.""" - if await utils.has_active_infraction(ctx, user, "mute"): + if await _utils.get_active_infraction(ctx, user, "mute"): return - infraction = await utils.post_infraction(ctx, user, "mute", reason, active=True, **kwargs) + infraction = await _utils.post_infraction(ctx, user, "mute", reason, active=True, **kwargs) if infraction is None: return @@ -217,33 +231,53 @@ class Infractions(InfractionScheduler, commands.Cog): await self.apply_infraction(ctx, infraction, user, action()) @respect_role_hierarchy() - async def apply_kick(self, ctx: Context, user: Member, reason: str, **kwargs) -> None: + async def apply_kick(self, ctx: Context, user: Member, reason: t.Optional[str], **kwargs) -> None: """Apply a kick infraction with kwargs passed to `post_infraction`.""" - infraction = await utils.post_infraction(ctx, user, "kick", reason, active=False, **kwargs) + infraction = await _utils.post_infraction(ctx, user, "kick", reason, active=False, **kwargs) if infraction is None: return self.mod_log.ignore(Event.member_remove, user.id) + if reason: + reason = textwrap.shorten(reason, width=512, placeholder="...") + action = user.kick(reason=reason) await self.apply_infraction(ctx, infraction, user, action) @respect_role_hierarchy() - async def apply_ban(self, ctx: Context, user: UserSnowflake, reason: str, **kwargs) -> None: + async def apply_ban(self, ctx: Context, user: UserSnowflake, reason: t.Optional[str], **kwargs) -> None: """ Apply a ban infraction with kwargs passed to `post_infraction`. Will also remove the banned user from the Big Brother watch list if applicable. """ - if await utils.has_active_infraction(ctx, user, "ban"): - return + # In the case of a permanent ban, we don't need get_active_infractions to tell us if one is active + is_temporary = kwargs.get("expires_at") is not None + active_infraction = await _utils.get_active_infraction(ctx, user, "ban", is_temporary) + + if active_infraction: + if is_temporary: + log.trace("Tempban ignored as it cannot overwrite an active ban.") + return + + if active_infraction.get('expires_at') is None: + log.trace("Permaban already exists, notify.") + await ctx.send(f":x: User is already permanently banned (#{active_infraction['id']}).") + return - infraction = await utils.post_infraction(ctx, user, "ban", reason, active=True, **kwargs) + log.trace("Old tempban is being replaced by new permaban.") + await self.pardon_infraction(ctx, "ban", user, is_temporary) + + infraction = await _utils.post_infraction(ctx, user, "ban", reason, active=True, **kwargs) if infraction is None: return self.mod_log.ignore(Event.member_remove, user.id) + if reason: + reason = textwrap.shorten(reason, width=512, placeholder="...") + action = ctx.guild.ban(user, reason=reason, delete_message_days=0) await self.apply_infraction(ctx, infraction, user, action) @@ -264,7 +298,7 @@ class Infractions(InfractionScheduler, commands.Cog): # endregion # region: Base pardon functions - async def pardon_mute(self, user_id: int, guild: discord.Guild, reason: str) -> t.Dict[str, str]: + async def pardon_mute(self, user_id: int, guild: discord.Guild, reason: t.Optional[str]) -> t.Dict[str, str]: """Remove a user's muted role, DM them a notification, and return a log dict.""" user = guild.get_member(user_id) log_text = {} @@ -275,14 +309,14 @@ class Infractions(InfractionScheduler, commands.Cog): await user.remove_roles(self._muted_role, reason=reason) # DM the user about the expiration. - notified = await utils.notify_pardon( + notified = await _utils.notify_pardon( user=user, title="You have been unmuted", content="You may now send messages in the server.", - icon_url=utils.INFRACTION_ICONS["mute"][1] + icon_url=_utils.INFRACTION_ICONS["mute"][1] ) - log_text["Member"] = f"{user.mention}(`{user.id}`)" + log_text["Member"] = format_user(user) log_text["DM"] = "Sent" if notified else "**Failed**" else: log.info(f"Failed to unmute user {user_id}: user not found") @@ -290,7 +324,7 @@ class Infractions(InfractionScheduler, commands.Cog): return log_text - async def pardon_ban(self, user_id: int, guild: discord.Guild, reason: str) -> t.Dict[str, str]: + async def pardon_ban(self, user_id: int, guild: discord.Guild, reason: t.Optional[str]) -> t.Dict[str, str]: """Remove a user's ban on the Discord guild and return a log dict.""" user = discord.Object(user_id) log_text = {} @@ -305,7 +339,7 @@ class Infractions(InfractionScheduler, commands.Cog): return log_text - async def _pardon_action(self, infraction: utils.Infraction) -> t.Optional[t.Dict[str, str]]: + async def _pardon_action(self, infraction: _utils.Infraction) -> t.Optional[t.Dict[str, str]]: """ Execute deactivation steps specific to the infraction's type and return a log dict. @@ -323,9 +357,9 @@ class Infractions(InfractionScheduler, commands.Cog): # endregion # This cannot be static (must have a __func__ attribute). - def cog_check(self, ctx: Context) -> bool: + async def cog_check(self, ctx: Context) -> bool: """Only allow moderators to invoke the commands in this cog.""" - return with_role_check(ctx, *constants.MODERATION_ROLES) + return await commands.has_any_role(*constants.MODERATION_ROLES).predicate(ctx) # This cannot be static (must have a __func__ attribute). async def cog_command_error(self, ctx: Context, error: Exception) -> None: @@ -334,3 +368,8 @@ class Infractions(InfractionScheduler, commands.Cog): if discord.User in error.converters or discord.Member in error.converters: await ctx.send(str(error.errors[0])) error.handled = True + + +def setup(bot: Bot) -> None: + """Load the Infractions cog.""" + bot.add_cog(Infractions(bot)) diff --git a/bot/cogs/moderation/management.py b/bot/exts/moderation/infraction/management.py index 075d45e2d..cdab1a6c7 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/exts/moderation/infraction/management.py @@ -6,16 +6,16 @@ from datetime import datetime import discord from discord.ext import commands from discord.ext.commands import Context +from discord.utils import escape_markdown from bot import constants from bot.bot import Bot -from bot.converters import Expiry, InfractionSearchQuery, allowed_strings, proxy_user +from bot.converters import Expiry, Snowflake, UserMention, allowed_strings, proxy_user +from bot.exts.moderation.infraction.infractions import Infractions +from bot.exts.moderation.modlog import ModLog from bot.pagination import LinePaginator -from bot.utils import time -from bot.utils.checks import in_channel_check, with_role_check -from . import utils -from .infractions import Infractions -from .modlog import ModLog +from bot.utils import messages, time +from bot.utils.checks import in_whitelist_check log = logging.getLogger(__name__) @@ -43,14 +43,14 @@ class ModManagement(commands.Cog): @commands.group(name='infraction', aliases=('infr', 'infractions', 'inf'), invoke_without_command=True) async def infraction_group(self, ctx: Context) -> None: """Infraction manipulation commands.""" - await ctx.send_help("infraction") + await ctx.send_help(ctx.command) @infraction_group.command(name='edit') async def infraction_edit( self, ctx: Context, - infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], - duration: t.Union[Expiry, allowed_strings("p", "permanent"), None], + infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], # noqa: F821 + duration: t.Union[Expiry, allowed_strings("p", "permanent"), None], # noqa: F821 *, reason: str = None ) -> None: @@ -83,14 +83,14 @@ class ModManagement(commands.Cog): "actor__id": ctx.author.id, "ordering": "-inserted_at" } - infractions = await self.bot.api_client.get(f"bot/infractions", params=params) + infractions = await self.bot.api_client.get("bot/infractions", params=params) if infractions: old_infraction = infractions[0] infraction_id = old_infraction["id"] else: await ctx.send( - f":x: Couldn't find most recent infraction; you have never given an infraction." + ":x: Couldn't find most recent infraction; you have never given an infraction." ) return else: @@ -135,11 +135,11 @@ class ModManagement(commands.Cog): if 'expires_at' in request_data: # A scheduled task should only exist if the old infraction wasn't permanent if old_infraction['expires_at']: - self.infractions_cog.cancel_task(new_infraction['id']) + self.infractions_cog.scheduler.cancel(new_infraction['id']) # If the infraction was not marked as permanent, schedule a new expiration task if request_data['expires_at']: - self.infractions_cog.schedule_task(new_infraction['id'], new_infraction) + self.infractions_cog.schedule_expiration(new_infraction) log_text += f""" Previous expiry: {old_infraction['expires_at'] or "Permanent"} @@ -154,16 +154,12 @@ class ModManagement(commands.Cog): user = ctx.guild.get_member(user_id) if user: - user_text = f"{user.mention} (`{user.id}`)" + user_text = messages.format_user(user) thumbnail = user.avatar_url_as(static_format="png") else: - user_text = f"`{user_id}`" + user_text = f"<@{user_id}>" thumbnail = None - # The infraction's actor - actor_id = new_infraction['actor'] - actor = ctx.guild.get_member(actor_id) or f"`{actor_id}`" - await self.mod_log.send_log_message( icon_url=constants.Icons.pencil, colour=discord.Colour.blurple(), @@ -171,8 +167,8 @@ class ModManagement(commands.Cog): thumbnail=thumbnail, text=textwrap.dedent(f""" Member: {user_text} - Actor: {actor} - Edited by: {ctx.message.author}{log_text} + Actor: <@{new_infraction['actor']}> + Edited by: {ctx.message.author.mention}{log_text} """) ) @@ -180,10 +176,10 @@ class ModManagement(commands.Cog): # region: Search infractions @infraction_group.group(name="search", invoke_without_command=True) - async def infraction_search_group(self, ctx: Context, query: InfractionSearchQuery) -> None: + async def infraction_search_group(self, ctx: Context, query: t.Union[UserMention, Snowflake, str]) -> None: """Searches for infractions in the database.""" - if isinstance(query, discord.User): - await self.search_user(ctx, query) + if isinstance(query, int): + await self.search_user(ctx, discord.Object(query)) else: await self.search_reason(ctx, query) @@ -191,9 +187,16 @@ class ModManagement(commands.Cog): async def search_user(self, ctx: Context, user: t.Union[discord.User, proxy_user]) -> None: """Search for infractions by member.""" infraction_list = await self.bot.api_client.get( - 'bot/infractions', + 'bot/infractions/expanded', params={'user__id': str(user.id)} ) + + user = self.bot.get_user(user.id) + if not user and infraction_list: + # Use the user data retrieved from the DB for the username. + user = infraction_list[0]["user"] + user = escape_markdown(user["name"]) + f"#{user['discriminator']:04}" + embed = discord.Embed( title=f"Infractions for {user} ({len(infraction_list)} total)", colour=discord.Colour.orange() @@ -204,7 +207,7 @@ class ModManagement(commands.Cog): async def search_reason(self, ctx: Context, reason: str) -> None: """Search for infractions by their reason. Use Re2 for matching.""" infraction_list = await self.bot.api_client.get( - 'bot/infractions', + 'bot/infractions/expanded', params={'search': reason} ) embed = discord.Embed( @@ -220,11 +223,11 @@ class ModManagement(commands.Cog): self, ctx: Context, embed: discord.Embed, - infractions: t.Iterable[utils.Infraction] + infractions: t.Iterable[t.Dict[str, t.Any]] ) -> None: """Send a paginated embed of infractions for the specified user.""" if not infractions: - await ctx.send(f":warning: No infractions could be found for that query.") + await ctx.send(":warning: No infractions could be found for that query.") return lines = tuple( @@ -241,39 +244,45 @@ class ModManagement(commands.Cog): max_size=1000 ) - def infraction_to_string(self, infraction: utils.Infraction) -> str: + def infraction_to_string(self, infraction: t.Dict[str, t.Any]) -> str: """Convert the infraction object to a string representation.""" - actor_id = infraction["actor"] - guild = self.bot.get_guild(constants.Guild.id) - actor = guild.get_member(actor_id) active = infraction["active"] - user_id = infraction["user"] - hidden = infraction["hidden"] + user = infraction["user"] + expires_at = infraction["expires_at"] created = time.format_infraction(infraction["inserted_at"]) + # Format the user string. + if user_obj := self.bot.get_user(user["id"]): + # The user is in the cache. + user_str = messages.format_user(user_obj) + else: + # Use the user data retrieved from the DB. + name = escape_markdown(user['name']) + user_str = f"<@{user['id']}> ({name}#{user['discriminator']:04})" + if active: - remaining = time.until_expiration(infraction["expires_at"]) or "Expired" + remaining = time.until_expiration(expires_at) or "Expired" else: remaining = "Inactive" - if infraction["expires_at"] is None: + if expires_at is None: expires = "*Permanent*" else: date_from = datetime.strptime(created, time.INFRACTION_FORMAT) - expires = time.format_infraction_with_duration(infraction["expires_at"], date_from) + expires = time.format_infraction_with_duration(expires_at, date_from) lines = textwrap.dedent(f""" {"**===============**" if active else "==============="} Status: {"__**Active**__" if active else "Inactive"} - User: {self.bot.get_user(user_id)} (`{user_id}`) + User: {user_str} Type: **{infraction["type"]}** - Shadow: {hidden} - Reason: {infraction["reason"] or "*None*"} + Shadow: {infraction["hidden"]} Created: {created} Expires: {expires} Remaining: {remaining} - Actor: {actor.mention if actor else actor_id} + Actor: <@{infraction["actor"]["id"]}> ID: `{infraction["id"]}` + Reason: {infraction["reason"] or "*None*"} {"**===============**" if active else "==============="} """) @@ -282,11 +291,17 @@ class ModManagement(commands.Cog): # endregion # This cannot be static (must have a __func__ attribute). - def cog_check(self, ctx: Context) -> bool: - """Only allow moderators from moderator channels to invoke the commands in this cog.""" + async def cog_check(self, ctx: Context) -> bool: + """Only allow moderators inside moderator channels to invoke the commands in this cog.""" checks = [ - with_role_check(ctx, *constants.MODERATION_ROLES), - in_channel_check(ctx, *constants.MODERATION_CHANNELS) + await commands.has_any_role(*constants.MODERATION_ROLES).predicate(ctx), + in_whitelist_check( + ctx, + channels=constants.MODERATION_CHANNELS, + categories=[constants.Categories.modmail], + redirect=None, + fail_silently=True, + ) ] return all(checks) @@ -297,3 +312,8 @@ class ModManagement(commands.Cog): if discord.User in error.converters: await ctx.send(str(error.errors[0])) error.handled = True + + +def setup(bot: Bot) -> None: + """Load the ModManagement cog.""" + bot.add_cog(ModManagement(bot)) diff --git a/bot/cogs/moderation/superstarify.py b/bot/exts/moderation/infraction/superstarify.py index ca3dc4202..eec63f5b3 100644 --- a/bot/cogs/moderation/superstarify.py +++ b/bot/exts/moderation/infraction/superstarify.py @@ -6,15 +6,16 @@ import typing as t from pathlib import Path from discord import Colour, Embed, Member -from discord.ext.commands import Cog, Context, command +from discord.ext.commands import Cog, Context, command, has_any_role +from discord.utils import escape_markdown from bot import constants from bot.bot import Bot from bot.converters import Expiry -from bot.utils.checks import with_role_check +from bot.exts.moderation.infraction import _utils +from bot.exts.moderation.infraction._scheduler import InfractionScheduler +from bot.utils.messages import format_user from bot.utils.time import format_infraction -from . import utils -from .scheduler import InfractionScheduler log = logging.getLogger(__name__) NICKNAME_POLICY_URL = "https://pythondiscord.com/pages/rules/#nickname-policy" @@ -67,7 +68,7 @@ class Superstarify(InfractionScheduler, Cog): reason=f"Superstarified member tried to escape the prison: {infraction['id']}" ) - notified = await utils.notify_infraction( + notified = await _utils.notify_infraction( user=after, infr_type="Superstarify", expires_at=format_infraction(infraction["expires_at"]), @@ -76,7 +77,7 @@ class Superstarify(InfractionScheduler, Cog): f"from **{before.display_name}** to **{after.display_name}**, but as you " "are currently in superstar-prison, you do not have permission to do so." ), - icon_url=utils.INFRACTION_ICONS["superstar"][0] + icon_url=_utils.INFRACTION_ICONS["superstar"][0] ) if not notified: @@ -130,15 +131,14 @@ class Superstarify(InfractionScheduler, Cog): An optional reason can be provided. If no reason is given, the original name will be shown in a generated reason. """ - if await utils.has_active_infraction(ctx, member, "superstar"): + if await _utils.get_active_infraction(ctx, member, "superstar"): return # Post the infraction to the API reason = reason or f"old nick: {member.display_name}" - infraction = await utils.post_infraction(ctx, member, "superstar", reason, duration, active=True) + infraction = await _utils.post_infraction(ctx, member, "superstar", reason, duration, active=True) id_ = infraction["id"] - old_nick = member.display_name forced_nick = self.get_nick(id_, member.id) expiry_str = format_infraction(infraction["expires_at"]) @@ -146,14 +146,17 @@ class Superstarify(InfractionScheduler, Cog): log.debug(f"Changing nickname of {member} to {forced_nick}.") self.mod_log.ignore(constants.Event.member_update, member.id) await member.edit(nick=forced_nick, reason=reason) - self.schedule_task(id_, infraction) + self.schedule_expiration(infraction) + + old_nick = escape_markdown(member.display_name) + forced_nick = escape_markdown(forced_nick) # Send a DM to the user to notify them of their new infraction. - await utils.notify_infraction( + await _utils.notify_infraction( user=member, infr_type="Superstarify", expires_at=expiry_str, - icon_url=utils.INFRACTION_ICONS["superstar"][0], + icon_url=_utils.INFRACTION_ICONS["superstar"][0], reason=f"Your nickname didn't comply with our [nickname policy]({NICKNAME_POLICY_URL})." ) @@ -176,17 +179,17 @@ class Superstarify(InfractionScheduler, Cog): # Log to the mod log channel. log.trace(f"Sending apply mod log for superstar #{id_}.") await self.mod_log.send_log_message( - icon_url=utils.INFRACTION_ICONS["superstar"][0], + icon_url=_utils.INFRACTION_ICONS["superstar"][0], colour=Colour.gold(), title="Member achieved superstardom", thumbnail=member.avatar_url_as(static_format="png"), text=textwrap.dedent(f""" - Member: {member.mention} (`{member.id}`) - Actor: {ctx.message.author} - Reason: {reason} + Member: {member.mention} + Actor: {ctx.message.author.mention} Expires: {expiry_str} Old nickname: `{old_nick}` New nickname: `{forced_nick}` + Reason: {reason} """), footer=f"ID {id_}" ) @@ -196,7 +199,7 @@ class Superstarify(InfractionScheduler, Cog): """Remove the superstarify infraction and allow the user to change their nickname.""" await self.pardon_infraction(ctx, "superstar", member) - async def _pardon_action(self, infraction: utils.Infraction) -> t.Optional[t.Dict[str, str]]: + async def _pardon_action(self, infraction: _utils.Infraction) -> t.Optional[t.Dict[str, str]]: """Pardon a superstar infraction and return a log dict.""" if infraction["type"] != "superstar": return @@ -213,15 +216,15 @@ class Superstarify(InfractionScheduler, Cog): return {} # DM the user about the expiration. - notified = await utils.notify_pardon( + notified = await _utils.notify_pardon( user=user, title="You are no longer superstarified", content="You may now change your nickname on the server.", - icon_url=utils.INFRACTION_ICONS["superstar"][1] + icon_url=_utils.INFRACTION_ICONS["superstar"][1] ) return { - "Member": f"{user.mention}(`{user.id}`)", + "Member": format_user(user), "DM": "Sent" if notified else "**Failed**" } @@ -234,6 +237,11 @@ class Superstarify(InfractionScheduler, Cog): return rng.choice(STAR_NAMES) # This cannot be static (must have a __func__ attribute). - def cog_check(self, ctx: Context) -> bool: + async def cog_check(self, ctx: Context) -> bool: """Only allow moderators to invoke the commands in this cog.""" - return with_role_check(ctx, *constants.MODERATION_ROLES) + return await has_any_role(*constants.MODERATION_ROLES).predicate(ctx) + + +def setup(bot: Bot) -> None: + """Load the Superstarify cog.""" + bot.add_cog(Superstarify(bot)) diff --git a/bot/cogs/moderation/modlog.py b/bot/exts/moderation/modlog.py index c63b4bab9..b01de0ee3 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/exts/moderation/modlog.py @@ -12,10 +12,10 @@ from deepdiff import DeepDiff from discord import Colour from discord.abc import GuildChannel from discord.ext.commands import Cog, Context -from discord.utils import escape_markdown from bot.bot import Bot -from bot.constants import Channels, Colours, Emojis, Event, Guild as GuildConstant, Icons, URLs +from bot.constants import Categories, Channels, Colours, Emojis, Event, Guild as GuildConstant, Icons, URLs +from bot.utils.messages import format_user from bot.utils.time import humanize_delta log = logging.getLogger(__name__) @@ -24,7 +24,6 @@ GUILD_CHANNEL = t.Union[discord.CategoryChannel, discord.TextChannel, discord.Vo CHANNEL_CHANGES_UNSUPPORTED = ("permissions",) CHANNEL_CHANGES_SUPPRESSED = ("_overwrites", "position") -MEMBER_CHANGES_SUPPRESSED = ("status", "activities", "_client_status", "nick") ROLE_CHANGES_UNSUPPORTED = ("colour", "permissions") VOICE_STATE_ATTRIBUTES = { @@ -64,7 +63,7 @@ class ModLog(Cog, name="ModLog"): 'id': message.id, 'author': message.author.id, 'channel_id': message.channel.id, - 'content': message.content, + 'content': message.content.replace("\0", ""), # Null chars cause 400. 'embeds': [embed.to_dict() for embed in message.embeds], 'attachments': attachment, } @@ -98,7 +97,10 @@ class ModLog(Cog, name="ModLog"): footer: t.Optional[str] = None, ) -> Context: """Generate log embed and send to logging channel.""" - embed = discord.Embed(description=text) + # Truncate string directly here to avoid removing newlines + embed = discord.Embed( + description=text[:2045] + "..." if len(text) > 2048 else text + ) if title and icon_url: embed.set_author(name=title, icon_url=icon_url) @@ -118,8 +120,17 @@ class ModLog(Cog, name="ModLog"): else: content = "@everyone" + # Truncate content to 2000 characters and append an ellipsis. + if content and len(content) > 2000: + content = content[:2000 - 3] + "..." + channel = self.bot.get_channel(channel_id) - log_message = await channel.send(content=content, embed=embed, files=files) + log_message = await channel.send( + content=content, + embed=embed, + files=files, + allowed_mentions=discord.AllowedMentions(everyone=True) + ) if additional_embeds: if additional_embeds_msg: @@ -188,6 +199,12 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.guild_channel_update].remove(before.id) return + # Two channel updates are sent for a single edit: 1 for topic and 1 for category change. + # TODO: remove once support is added for ignoring multiple occurrences for the same channel. + help_categories = (Categories.help_available, Categories.help_dormant, Categories.help_in_use) + if after.category and after.category.id in help_categories: + return + diff = DeepDiff(before, after) changes = [] done = [] @@ -379,7 +396,7 @@ class ModLog(Cog, name="ModLog"): await self.send_log_message( Icons.user_ban, Colours.soft_red, - "User banned", f"{member} (`{member.id}`)", + "User banned", format_user(member), thumbnail=member.avatar_url_as(static_format="png"), channel_id=Channels.user_log ) @@ -390,12 +407,10 @@ class ModLog(Cog, name="ModLog"): if member.guild.id != GuildConstant.id: return - member_str = escape_markdown(str(member)) - message = f"{member_str} (`{member.id}`)" now = datetime.utcnow() difference = abs(relativedelta(now, member.created_at)) - message += "\n\n**Account age:** " + humanize_delta(difference) + message = format_user(member) + "\n\n**Account age:** " + humanize_delta(difference) if difference.days < 1 and difference.months < 1 and difference.years < 1: # New user account! message = f"{Emojis.new} {message}" @@ -417,10 +432,9 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_remove].remove(member.id) return - member_str = escape_markdown(str(member)) await self.send_log_message( Icons.sign_out, Colours.soft_red, - "User left", f"{member_str} (`{member.id}`)", + "User left", format_user(member), thumbnail=member.avatar_url_as(static_format="png"), channel_id=Channels.user_log ) @@ -435,14 +449,28 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_unban].remove(member.id) return - member_str = escape_markdown(str(member)) await self.send_log_message( Icons.user_unban, Colour.blurple(), - "User unbanned", f"{member_str} (`{member.id}`)", + "User unbanned", format_user(member), thumbnail=member.avatar_url_as(static_format="png"), channel_id=Channels.mod_log ) + @staticmethod + def get_role_diff(before: t.List[discord.Role], after: t.List[discord.Role]) -> t.List[str]: + """Return a list of strings describing the roles added and removed.""" + changes = [] + before_roles = set(before) + after_roles = set(after) + + for role in (before_roles - after_roles): + changes.append(f"**Role removed:** {role.name} (`{role.id}`)") + + for role in (after_roles - before_roles): + changes.append(f"**Role added:** {role.name} (`{role.id}`)") + + return changes + @Cog.listener() async def on_member_update(self, before: discord.Member, after: discord.Member) -> None: """Log member update event to user log.""" @@ -453,74 +481,27 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_update].remove(before.id) return - diff = DeepDiff(before, after) - changes = [] - done = [] + changes = self.get_role_diff(before.roles, after.roles) - diff_values = {} + # The regex is a simple way to exclude all sequence and mapping types. + diff = DeepDiff(before, after, exclude_regex_paths=r".*\[.*") - diff_values.update(diff.get("values_changed", {})) - diff_values.update(diff.get("type_changes", {})) - diff_values.update(diff.get("iterable_item_removed", {})) - diff_values.update(diff.get("iterable_item_added", {})) - - diff_user = DeepDiff(before._user, after._user) - - diff_values.update(diff_user.get("values_changed", {})) - diff_values.update(diff_user.get("type_changes", {})) - diff_values.update(diff_user.get("iterable_item_removed", {})) - diff_values.update(diff_user.get("iterable_item_added", {})) - - for key, value in diff_values.items(): - if not key: # Not sure why, but it happens - continue - - key = key[5:] # Remove "root." prefix - - if "[" in key: - key = key.split("[", 1)[0] - - if "." in key: - key = key.split(".", 1)[0] + # A type change seems to always take precedent over a value change. Furthermore, it will + # include the value change along with the type change anyway. Therefore, it's OK to + # "overwrite" values_changed; in practice there will never even be anything to overwrite. + diff_values = {**diff.get("values_changed", {}), **diff.get("type_changes", {})} - if key in done or key in MEMBER_CHANGES_SUPPRESSED: + for attr, value in diff_values.items(): + if not attr: # Not sure why, but it happens. continue - if key == "_roles": - new_roles = after.roles - old_roles = before.roles - - for role in old_roles: - if role not in new_roles: - changes.append(f"**Role removed:** {role.name} (`{role.id}`)") - - for role in new_roles: - if role not in old_roles: - changes.append(f"**Role added:** {role.name} (`{role.id}`)") - - else: - new = value.get("new_value") - old = value.get("old_value") - - if new and old: - changes.append(f"**{key.title()}:** `{old}` **→** `{new}`") - - done.append(key) - - if before.name != after.name: - changes.append( - f"**Username:** `{before.name}` **→** `{after.name}`" - ) + attr = attr[5:] # Remove "root." prefix. + attr = attr.replace("_", " ").replace(".", " ").capitalize() - if before.discriminator != after.discriminator: - changes.append( - f"**Discriminator:** `{before.discriminator}` **→** `{after.discriminator}`" - ) + new = value.get("new_value") + old = value.get("old_value") - if before.display_name != after.display_name: - changes.append( - f"**Display name:** `{before.display_name}` **→** `{after.display_name}`" - ) + changes.append(f"**{attr}:** `{old}` **→** `{new}`") if not changes: return @@ -530,12 +511,13 @@ class ModLog(Cog, name="ModLog"): for item in sorted(changes): message += f"{Emojis.bullet} {item}\n" - member_str = escape_markdown(str(after)) - message = f"**{member_str}** (`{after.id}`)\n{message}" + message = f"{format_user(after)}\n{message}" await self.send_log_message( - Icons.user_update, Colour.blurple(), - "Member updated", message, + icon_url=Icons.user_update, + colour=Colour.blurple(), + title="Member updated", + text=message, thumbnail=after.avatar_url_as(static_format="png"), channel_id=Channels.user_log ) @@ -546,6 +528,10 @@ class ModLog(Cog, name="ModLog"): channel = message.channel author = message.author + # Ignore DMs. + if not message.guild: + return + if message.guild.id != GuildConstant.id or channel.id in GuildConstant.modlog_blacklist: return @@ -558,17 +544,16 @@ class ModLog(Cog, name="ModLog"): if author.bot: return - author_str = escape_markdown(str(author)) if channel.category: response = ( - f"**Author:** {author_str} (`{author.id}`)\n" + f"**Author:** {format_user(author)}\n" f"**Channel:** {channel.category}/#{channel.name} (`{channel.id}`)\n" f"**Message ID:** `{message.id}`\n" "\n" ) else: response = ( - f"**Author:** {author_str} (`{author.id}`)\n" + f"**Author:** {format_user(author)}\n" f"**Channel:** #{channel.name} (`{channel.id}`)\n" f"**Message ID:** `{message.id}`\n" "\n" @@ -654,9 +639,6 @@ class ModLog(Cog, name="ModLog"): if msg_before.content == msg_after.content: return - author = msg_before.author - author_str = escape_markdown(str(author)) - channel = msg_before.channel channel_name = f"{channel.category}/#{channel.name}" if channel.category else f"#{channel.name}" @@ -688,7 +670,7 @@ class ModLog(Cog, name="ModLog"): content_after.append(sub) response = ( - f"**Author:** {author_str} (`{author.id}`)\n" + f"**Author:** {format_user(msg_before.author)}\n" f"**Channel:** {channel_name} (`{channel.id}`)\n" f"**Message ID:** `{msg_before.id}`\n" "\n" @@ -740,12 +722,11 @@ class ModLog(Cog, name="ModLog"): self._cached_edits.remove(event.message_id) return - author = message.author channel = message.channel channel_name = f"{channel.category}/#{channel.name}" if channel.category else f"#{channel.name}" before_response = ( - f"**Author:** {author} (`{author.id}`)\n" + f"**Author:** {format_user(message.author)}\n" f"**Channel:** {channel_name} (`{channel.id}`)\n" f"**Message ID:** `{message.id}`\n" "\n" @@ -753,7 +734,7 @@ class ModLog(Cog, name="ModLog"): ) after_response = ( - f"**Author:** {author} (`{author.id}`)\n" + f"**Author:** {format_user(message.author)}\n" f"**Channel:** {channel_name} (`{channel.id}`)\n" f"**Message ID:** `{message.id}`\n" "\n" @@ -831,9 +812,8 @@ class ModLog(Cog, name="ModLog"): if not changes: return - member_str = escape_markdown(str(member)) message = "\n".join(f"{Emojis.bullet} {item}" for item in sorted(changes)) - message = f"**{member_str}** (`{member.id}`)\n{message}" + message = f"{format_user(member)}\n{message}" await self.send_log_message( icon_url=icon, @@ -843,3 +823,8 @@ class ModLog(Cog, name="ModLog"): thumbnail=member.avatar_url_as(static_format="png"), channel_id=Channels.voice_log ) + + +def setup(bot: Bot) -> None: + """Load the ModLog cog.""" + bot.add_cog(ModLog(bot)) diff --git a/bot/cogs/moderation/silence.py b/bot/exts/moderation/silence.py index 1ef3967a9..ac0c1c85e 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/exts/moderation/silence.py @@ -10,7 +10,7 @@ from discord.ext.commands import Context from bot.bot import Bot from bot.constants import Channels, Emojis, Guild, MODERATION_ROLES, Roles from bot.converters import HushDurationConverter -from bot.utils.checks import with_role_check +from bot.utils.scheduling import Scheduler log = logging.getLogger(__name__) @@ -58,7 +58,9 @@ class Silence(commands.Cog): def __init__(self, bot: Bot): self.bot = bot + self.scheduler = Scheduler(self.__class__.__name__) self.muted_channels = set() + self._get_instance_vars_task = self.bot.loop.create_task(self._get_instance_vars()) self._get_instance_vars_event = asyncio.Event() @@ -90,9 +92,8 @@ class Silence(commands.Cog): return await ctx.send(f"{Emojis.check_mark} silenced current channel for {duration} minute(s).") - await asyncio.sleep(duration*60) - log.info(f"Unsilencing channel after set delay.") - await ctx.invoke(self.unsilence) + + self.scheduler.schedule_later(duration * 60, ctx.channel.id, ctx.invoke(self.unsilence)) @commands.command(aliases=("unhush",)) async def unsilence(self, ctx: Context) -> None: @@ -103,7 +104,9 @@ class Silence(commands.Cog): """ await self._get_instance_vars_event.wait() log.debug(f"Unsilencing channel #{ctx.channel} from {ctx.author}'s command.") - if await self._unsilence(ctx.channel): + if not await self._unsilence(ctx.channel): + await ctx.send(f"{Emojis.cross_mark} current channel was not silenced.") + else: await ctx.send(f"{Emojis.check_mark} unsilenced current channel.") async def _silence(self, channel: TextChannel, persistent: bool, duration: Optional[int]) -> bool: @@ -140,6 +143,7 @@ class Silence(commands.Cog): if current_overwrite.send_messages is False: await channel.set_permissions(self._verified_role, **dict(current_overwrite, send_messages=None)) log.info(f"Unsilenced channel #{channel} ({channel.id}).") + self.scheduler.cancel(channel.id) self.notifier.remove_channel(channel) self.muted_channels.discard(channel) return True @@ -147,13 +151,19 @@ class Silence(commands.Cog): return False def cog_unload(self) -> None: - """Send alert with silenced channels on unload.""" + """Send alert with silenced channels and cancel scheduled tasks on unload.""" + self.scheduler.cancel_all() if self.muted_channels: channels_string = ''.join(channel.mention for channel in self.muted_channels) message = f"<@&{Roles.moderators}> channels left silenced on cog unload: {channels_string}" asyncio.create_task(self._mod_alerts_channel.send(message)) # This cannot be static (must have a __func__ attribute). - def cog_check(self, ctx: Context) -> bool: + async def cog_check(self, ctx: Context) -> bool: """Only allow moderators to invoke the commands in this cog.""" - return with_role_check(ctx, *MODERATION_ROLES) + return await commands.has_any_role(*MODERATION_ROLES).predicate(ctx) + + +def setup(bot: Bot) -> None: + """Load the Silence cog.""" + bot.add_cog(Silence(bot)) diff --git a/bot/exts/moderation/slowmode.py b/bot/exts/moderation/slowmode.py new file mode 100644 index 000000000..efd862aa5 --- /dev/null +++ b/bot/exts/moderation/slowmode.py @@ -0,0 +1,96 @@ +import logging +from datetime import datetime +from typing import Optional + +from dateutil.relativedelta import relativedelta +from discord import TextChannel +from discord.ext.commands import Cog, Context, group, has_any_role + +from bot.bot import Bot +from bot.constants import Emojis, MODERATION_ROLES +from bot.converters import DurationDelta +from bot.utils import time + +log = logging.getLogger(__name__) + +SLOWMODE_MAX_DELAY = 21600 # seconds + + +class Slowmode(Cog): + """Commands for getting and setting slowmode delays of text channels.""" + + def __init__(self, bot: Bot) -> None: + self.bot = bot + + @group(name='slowmode', aliases=['sm'], invoke_without_command=True) + async def slowmode_group(self, ctx: Context) -> None: + """Get or set the slowmode delay for the text channel this was invoked in or a given text channel.""" + await ctx.send_help(ctx.command) + + @slowmode_group.command(name='get', aliases=['g']) + async def get_slowmode(self, ctx: Context, channel: Optional[TextChannel]) -> None: + """Get the slowmode delay for a text channel.""" + # Use the channel this command was invoked in if one was not given + if channel is None: + channel = ctx.channel + + delay = relativedelta(seconds=channel.slowmode_delay) + humanized_delay = time.humanize_delta(delay) + + await ctx.send(f'The slowmode delay for {channel.mention} is {humanized_delay}.') + + @slowmode_group.command(name='set', aliases=['s']) + async def set_slowmode(self, ctx: Context, channel: Optional[TextChannel], delay: DurationDelta) -> None: + """Set the slowmode delay for a text channel.""" + # Use the channel this command was invoked in if one was not given + if channel is None: + channel = ctx.channel + + # Convert `dateutil.relativedelta.relativedelta` to `datetime.timedelta` + # Must do this to get the delta in a particular unit of time + utcnow = datetime.utcnow() + slowmode_delay = (utcnow + delay - utcnow).total_seconds() + + humanized_delay = time.humanize_delta(delay) + + # Ensure the delay is within discord's limits + if slowmode_delay <= SLOWMODE_MAX_DELAY: + log.info(f'{ctx.author} set the slowmode delay for #{channel} to {humanized_delay}.') + + await channel.edit(slowmode_delay=slowmode_delay) + await ctx.send( + f'{Emojis.check_mark} The slowmode delay for {channel.mention} is now {humanized_delay}.' + ) + + else: + log.info( + f'{ctx.author} tried to set the slowmode delay of #{channel} to {humanized_delay}, ' + 'which is not between 0 and 6 hours.' + ) + + await ctx.send( + f'{Emojis.cross_mark} The slowmode delay must be between 0 and 6 hours.' + ) + + @slowmode_group.command(name='reset', aliases=['r']) + async def reset_slowmode(self, ctx: Context, channel: Optional[TextChannel]) -> None: + """Reset the slowmode delay for a text channel to 0 seconds.""" + # Use the channel this command was invoked in if one was not given + if channel is None: + channel = ctx.channel + + log.info(f'{ctx.author} reset the slowmode delay for #{channel} to 0 seconds.') + + await channel.edit(slowmode_delay=0) + await ctx.send( + f'{Emojis.check_mark} The slowmode delay for {channel.mention} has been reset to 0 seconds.' + ) + + async def cog_check(self, ctx: Context) -> bool: + """Only allow moderators to invoke the commands in this cog.""" + return await has_any_role(*MODERATION_ROLES).predicate(ctx) + + +def setup(bot: Bot) -> None: + """Load the Slowmode cog.""" + bot.add_cog(Slowmode(bot)) diff --git a/bot/exts/moderation/verification.py b/bot/exts/moderation/verification.py new file mode 100644 index 000000000..206556483 --- /dev/null +++ b/bot/exts/moderation/verification.py @@ -0,0 +1,775 @@ +import asyncio +import logging +import typing as t +from contextlib import suppress +from datetime import datetime, timedelta + +import discord +from async_rediscache import RedisCache +from discord.ext import tasks +from discord.ext.commands import Cog, Context, command, group, has_any_role +from discord.utils import snowflake_time + +from bot import constants +from bot.bot import Bot +from bot.decorators import has_no_roles, in_whitelist +from bot.exts.moderation.modlog import ModLog +from bot.utils.checks import InWhitelistCheckFailure, has_no_roles_check +from bot.utils.messages import format_user + +log = logging.getLogger(__name__) + +# Sent via DMs once user joins the guild +ON_JOIN_MESSAGE = f""" +Welcome to Python Discord! + +To show you what kind of community we are, we've created this video: +https://youtu.be/ZH26PuX3re0 + +As a new user, you have read-only access to a few select channels to give you a taste of what our server is like. \ +In order to see the rest of the channels and to send messages, you first have to accept our rules. + +Please visit <#{constants.Channels.verification}> to get started. Thank you! +""" + +# Sent via DMs once user verifies +VERIFIED_MESSAGE = f""" +Thanks for verifying yourself! + +For your records, these are the documents you accepted: + +`1)` Our rules, here: <https://pythondiscord.com/pages/rules> +`2)` Our privacy policy, here: <https://pythondiscord.com/pages/privacy> - you can find information on how to have \ +your information removed here as well. + +Feel free to review them at any point! + +Additionally, if you'd like to receive notifications for the announcements \ +we post in <#{constants.Channels.announcements}> +from time to time, you can send `!subscribe` to <#{constants.Channels.bot_commands}> at any time \ +to assign yourself the **Announcements** role. We'll mention this role every time we make an announcement. + +If you'd like to unsubscribe from the announcement notifications, simply send `!unsubscribe` to \ +<#{constants.Channels.bot_commands}>. +""" + +# Sent via DMs to users kicked for failing to verify +KICKED_MESSAGE = f""" +Hi! You have been automatically kicked from Python Discord as you have failed to accept our rules \ +within `{constants.Verification.kicked_after}` days. If this was an accident, please feel free to join us again! + +{constants.Guild.invite} +""" + +# Sent periodically in the verification channel +REMINDER_MESSAGE = f""" +<@&{constants.Roles.unverified}> + +Welcome to Python Discord! Please read the documents mentioned above and type `!accept` to gain permissions \ +to send messages in the community! + +You will be kicked if you don't verify within `{constants.Verification.kicked_after}` days. +""".strip() + +# An async function taking a Member param +Request = t.Callable[[discord.Member], t.Awaitable] + + +class StopExecution(Exception): + """Signals that a task should halt immediately & alert admins.""" + + def __init__(self, reason: discord.HTTPException) -> None: + super().__init__() + self.reason = reason + + +class Limit(t.NamedTuple): + """Composition over config for throttling requests.""" + + batch_size: int # Amount of requests after which to pause + sleep_secs: int # Sleep this many seconds after each batch + + +def mention_role(role_id: int) -> discord.AllowedMentions: + """Construct an allowed mentions instance that allows pinging `role_id`.""" + return discord.AllowedMentions(roles=[discord.Object(role_id)]) + + +def is_verified(member: discord.Member) -> bool: + """ + Check whether `member` is considered verified. + + Members are considered verified if they have at least 1 role other than + the default role (@everyone) and the @Unverified role. + """ + unverified_roles = { + member.guild.get_role(constants.Roles.unverified), + member.guild.default_role, + } + return len(set(member.roles) - unverified_roles) > 0 + + +async def safe_dm(coro: t.Coroutine) -> None: + """ + Execute `coro` ignoring disabled DM warnings. + + The 50_0007 error code indicates that the target user does not accept DMs. + As it turns out, this error code can appear on both 400 and 403 statuses, + we therefore catch any Discord exception. + + If the request fails on any other error code, the exception propagates, + and must be handled by the caller. + """ + try: + await coro + except discord.HTTPException as discord_exc: + log.trace(f"DM dispatch failed on status {discord_exc.status} with code: {discord_exc.code}") + if discord_exc.code != 50_007: # If any reason other than disabled DMs + raise + + +class Verification(Cog): + """ + User verification and role management. + + There are two internal tasks in this cog: + + * `update_unverified_members` + * Unverified members are given the @Unverified role after configured `unverified_after` days + * Unverified members are kicked after configured `kicked_after` days + * `ping_unverified` + * Periodically ping the @Unverified role in the verification channel + + Statistics are collected in the 'verification.' namespace. + + Moderators+ can use the `verification` command group to start or stop both internal + tasks, if necessary. Settings are persisted in Redis across sessions. + + Additionally, this cog offers the !accept, !subscribe and !unsubscribe commands, + and keeps the verification channel clean by deleting messages. + """ + + # Persist task settings & last sent `REMINDER_MESSAGE` id + # RedisCache[ + # "tasks_running": int (0 or 1), + # "last_reminder": int (discord.Message.id), + # ] + task_cache = RedisCache() + + def __init__(self, bot: Bot) -> None: + """Start internal tasks.""" + self.bot = bot + self.bot.loop.create_task(self._maybe_start_tasks()) + + def cog_unload(self) -> None: + """ + Cancel internal tasks. + + This is necessary, as tasks are not automatically cancelled on cog unload. + """ + self._stop_tasks(gracefully=False) + + @property + def mod_log(self) -> ModLog: + """Get currently loaded ModLog cog instance.""" + return self.bot.get_cog("ModLog") + + async def _maybe_start_tasks(self) -> None: + """ + Poll Redis to check whether internal tasks should start. + + Redis must be interfaced with from an async function. + """ + log.trace("Checking whether background tasks should begin") + setting: t.Optional[int] = await self.task_cache.get("tasks_running") # This can be None if never set + + if setting: + log.trace("Background tasks will be started") + self.update_unverified_members.start() + self.ping_unverified.start() + + def _stop_tasks(self, *, gracefully: bool) -> None: + """ + Stop the update users & ping @Unverified tasks. + + If `gracefully` is True, the tasks will be able to finish their current iteration. + Otherwise, they are cancelled immediately. + """ + log.info(f"Stopping internal tasks ({gracefully=})") + if gracefully: + self.update_unverified_members.stop() + self.ping_unverified.stop() + else: + self.update_unverified_members.cancel() + self.ping_unverified.cancel() + + # region: automatically update unverified users + + async def _verify_kick(self, n_members: int) -> bool: + """ + Determine whether `n_members` is a reasonable amount of members to kick. + + First, `n_members` is checked against the size of the PyDis guild. If `n_members` are + more than the configured `kick_confirmation_threshold` of the guild, the operation + must be confirmed by staff in #core-dev. Otherwise, the operation is seen as safe. + """ + log.debug(f"Checking whether {n_members} members are safe to kick") + + await self.bot.wait_until_guild_available() # Ensure cache is populated before we grab the guild + pydis = self.bot.get_guild(constants.Guild.id) + + percentage = n_members / len(pydis.members) + if percentage < constants.Verification.kick_confirmation_threshold: + log.debug(f"Kicking {percentage:.2%} of the guild's population is seen as safe") + return True + + # Since `n_members` is a suspiciously large number, we will ask for confirmation + log.debug("Amount of users is too large, requesting staff confirmation") + + core_dev_channel = pydis.get_channel(constants.Channels.dev_core) + core_dev_ping = f"<@&{constants.Roles.core_developers}>" + + confirmation_msg = await core_dev_channel.send( + f"{core_dev_ping} Verification determined that `{n_members}` members should be kicked as they haven't " + f"verified in `{constants.Verification.kicked_after}` days. This is `{percentage:.2%}` of the guild's " + f"population. Proceed?", + allowed_mentions=mention_role(constants.Roles.core_developers), + ) + + options = (constants.Emojis.incident_actioned, constants.Emojis.incident_unactioned) + for option in options: + await confirmation_msg.add_reaction(option) + + core_dev_ids = [member.id for member in pydis.get_role(constants.Roles.core_developers).members] + + def check(reaction: discord.Reaction, user: discord.User) -> bool: + """Check whether `reaction` is a valid reaction to `confirmation_msg`.""" + return ( + reaction.message.id == confirmation_msg.id # Reacted to `confirmation_msg` + and str(reaction.emoji) in options # With one of `options` + and user.id in core_dev_ids # By a core developer + ) + + timeout = 60 * 5 # Seconds, i.e. 5 minutes + try: + choice, _ = await self.bot.wait_for("reaction_add", check=check, timeout=timeout) + except asyncio.TimeoutError: + log.debug("Staff prompt not answered, aborting operation") + return False + finally: + with suppress(discord.HTTPException): + await confirmation_msg.clear_reactions() + + result = str(choice) == constants.Emojis.incident_actioned + log.debug(f"Received answer: {choice}, result: {result}") + + # Edit the prompt message to reflect the final choice + if result is True: + result_msg = f":ok_hand: {core_dev_ping} Request to kick `{n_members}` members was authorized!" + else: + result_msg = f":warning: {core_dev_ping} Request to kick `{n_members}` members was denied!" + + with suppress(discord.HTTPException): + await confirmation_msg.edit(content=result_msg) + + return result + + async def _alert_admins(self, exception: discord.HTTPException) -> None: + """ + Ping @Admins with information about `exception`. + + This is used when a critical `exception` caused a verification task to abort. + """ + await self.bot.wait_until_guild_available() + log.info(f"Sending admin alert regarding exception: {exception}") + + admins_channel = self.bot.get_guild(constants.Guild.id).get_channel(constants.Channels.admins) + ping = f"<@&{constants.Roles.admins}>" + + await admins_channel.send( + f"{ping} Aborted updating unverified users due to the following exception:\n" + f"```{exception}```\n" + f"Internal tasks will be stopped.", + allowed_mentions=mention_role(constants.Roles.admins), + ) + + async def _send_requests(self, members: t.Collection[discord.Member], request: Request, limit: Limit) -> int: + """ + Pass `members` one by one to `request` handling Discord exceptions. + + This coroutine serves as a generic `request` executor for kicking members and adding + roles, as it allows us to define the error handling logic in one place only. + + Any `request` has the ability to completely abort the execution by raising `StopExecution`. + In such a case, the @Admins will be alerted of the reason attribute. + + To avoid rate-limits, pass a `limit` configuring the batch size and the amount of seconds + to sleep between batches. + + Returns the amount of successful requests. Failed requests are logged at info level. + """ + log.trace(f"Sending {len(members)} requests") + n_success, bad_statuses = 0, set() + + for progress, member in enumerate(members, start=1): + if is_verified(member): # Member could have verified in the meantime + continue + try: + await request(member) + except StopExecution as stop_execution: + await self._alert_admins(stop_execution.reason) + await self.task_cache.set("tasks_running", 0) + self._stop_tasks(gracefully=True) # Gracefully finish current iteration, then stop + break + except discord.HTTPException as http_exc: + bad_statuses.add(http_exc.status) + else: + n_success += 1 + + if progress % limit.batch_size == 0: + log.trace(f"Processed {progress} requests, pausing for {limit.sleep_secs} seconds") + await asyncio.sleep(limit.sleep_secs) + + if bad_statuses: + log.info(f"Failed to send {len(members) - n_success} requests due to following statuses: {bad_statuses}") + + return n_success + + async def _kick_members(self, members: t.Collection[discord.Member]) -> int: + """ + Kick `members` from the PyDis guild. + + Due to strict ratelimits on sending messages (120 requests / 60 secs), we sleep for a second + after each 2 requests to allow breathing room for other features. + + Note that this is a potentially destructive operation. Returns the amount of successful requests. + """ + log.info(f"Kicking {len(members)} members (not verified after {constants.Verification.kicked_after} days)") + + async def kick_request(member: discord.Member) -> None: + """Send `KICKED_MESSAGE` to `member` and kick them from the guild.""" + try: + await safe_dm(member.send(KICKED_MESSAGE)) # Suppress disabled DMs + except discord.HTTPException as suspicious_exception: + raise StopExecution(reason=suspicious_exception) + await member.kick(reason=f"User has not verified in {constants.Verification.kicked_after} days") + + n_kicked = await self._send_requests(members, kick_request, Limit(batch_size=2, sleep_secs=1)) + self.bot.stats.incr("verification.kicked", count=n_kicked) + + return n_kicked + + async def _give_role(self, members: t.Collection[discord.Member], role: discord.Role) -> int: + """ + Give `role` to all `members`. + + We pause for a second after batches of 25 requests to ensure ratelimits aren't exceeded. + + Returns the amount of successful requests. + """ + log.info( + f"Assigning {role} role to {len(members)} members (not verified " + f"after {constants.Verification.unverified_after} days)" + ) + + async def role_request(member: discord.Member) -> None: + """Add `role` to `member`.""" + await member.add_roles(role, reason=f"Not verified after {constants.Verification.unverified_after} days") + + return await self._send_requests(members, role_request, Limit(batch_size=25, sleep_secs=1)) + + async def _check_members(self) -> t.Tuple[t.Set[discord.Member], t.Set[discord.Member]]: + """ + Check in on the verification status of PyDis members. + + This coroutine finds two sets of users: + * Not verified after configured `unverified_after` days, should be given the @Unverified role + * Not verified after configured `kicked_after` days, should be kicked from the guild + + These sets are always disjoint, i.e. share no common members. + """ + await self.bot.wait_until_guild_available() # Ensure cache is ready + pydis = self.bot.get_guild(constants.Guild.id) + + unverified = pydis.get_role(constants.Roles.unverified) + current_dt = datetime.utcnow() # Discord timestamps are UTC + + # Users to be given the @Unverified role, and those to be kicked, these should be entirely disjoint + for_role, for_kick = set(), set() + + log.debug("Checking verification status of guild members") + for member in pydis.members: + + # Skip verified members, bots, and members for which we do not know their join date, + # this should be extremely rare but docs mention that it can happen + if is_verified(member) or member.bot or member.joined_at is None: + continue + + # At this point, we know that `member` is an unverified user, and we will decide what + # to do with them based on time passed since their join date + since_join = current_dt - member.joined_at + + if since_join > timedelta(days=constants.Verification.kicked_after): + for_kick.add(member) # User should be removed from the guild + + elif ( + since_join > timedelta(days=constants.Verification.unverified_after) + and unverified not in member.roles + ): + for_role.add(member) # User should be given the @Unverified role + + log.debug(f"Found {len(for_role)} users for {unverified} role, {len(for_kick)} users to be kicked") + return for_role, for_kick + + @tasks.loop(minutes=30) + async def update_unverified_members(self) -> None: + """ + Periodically call `_check_members` and update unverified members accordingly. + + After each run, a summary will be sent to the modlog channel. If a suspiciously high + amount of members to be kicked is found, the operation is guarded by `_verify_kick`. + """ + log.info("Updating unverified guild members") + + await self.bot.wait_until_guild_available() + unverified = self.bot.get_guild(constants.Guild.id).get_role(constants.Roles.unverified) + + for_role, for_kick = await self._check_members() + + if not for_role: + role_report = f"Found no users to be assigned the {unverified.mention} role." + else: + n_roles = await self._give_role(for_role, unverified) + role_report = f"Assigned {unverified.mention} role to `{n_roles}`/`{len(for_role)}` members." + + if not for_kick: + kick_report = "Found no users to be kicked." + elif not await self._verify_kick(len(for_kick)): + kick_report = f"Not authorized to kick `{len(for_kick)}` members." + else: + n_kicks = await self._kick_members(for_kick) + kick_report = f"Kicked `{n_kicks}`/`{len(for_kick)}` members from the guild." + + await self.mod_log.send_log_message( + icon_url=self.bot.user.avatar_url, + colour=discord.Colour.blurple(), + title="Verification system", + text=f"{kick_report}\n{role_report}", + ) + + # endregion + # region: periodically ping @Unverified + + @tasks.loop(hours=constants.Verification.reminder_frequency) + async def ping_unverified(self) -> None: + """ + Delete latest `REMINDER_MESSAGE` and send it again. + + This utilizes RedisCache to persist the latest reminder message id. + """ + await self.bot.wait_until_guild_available() + verification = self.bot.get_guild(constants.Guild.id).get_channel(constants.Channels.verification) + + last_reminder: t.Optional[int] = await self.task_cache.get("last_reminder") + + if last_reminder is not None: + log.trace(f"Found verification reminder message in cache, deleting: {last_reminder}") + + with suppress(discord.HTTPException): # If something goes wrong, just ignore it + await self.bot.http.delete_message(verification.id, last_reminder) + + log.trace("Sending verification reminder") + new_reminder = await verification.send( + REMINDER_MESSAGE, allowed_mentions=mention_role(constants.Roles.unverified), + ) + + await self.task_cache.set("last_reminder", new_reminder.id) + + @ping_unverified.before_loop + async def _before_first_ping(self) -> None: + """ + Sleep until `REMINDER_MESSAGE` should be sent again. + + If latest reminder is not cached, exit instantly. Otherwise, wait wait until the + configured `reminder_frequency` has passed. + """ + last_reminder: t.Optional[int] = await self.task_cache.get("last_reminder") + + if last_reminder is None: + log.trace("Latest verification reminder message not cached, task will not wait") + return + + # Convert cached message id into a timestamp + time_since = datetime.utcnow() - snowflake_time(last_reminder) + log.trace(f"Time since latest verification reminder: {time_since}") + + to_sleep = timedelta(hours=constants.Verification.reminder_frequency) - time_since + log.trace(f"Time to sleep until next ping: {to_sleep}") + + # Delta can be negative if `reminder_frequency` has already passed + secs = max(to_sleep.total_seconds(), 0) + await asyncio.sleep(secs) + + # endregion + # region: listeners + + @Cog.listener() + async def on_member_join(self, member: discord.Member) -> None: + """Attempt to send initial direct message to each new member.""" + if member.guild.id != constants.Guild.id: + return # Only listen for PyDis events + + log.trace(f"Sending on join message to new member: {member.id}") + try: + await safe_dm(member.send(ON_JOIN_MESSAGE)) + except discord.HTTPException: + log.exception("DM dispatch failed on unexpected error code") + + @Cog.listener() + async def on_message(self, message: discord.Message) -> None: + """Check new message event for messages to the checkpoint channel & process.""" + if message.channel.id != constants.Channels.verification: + return # Only listen for #checkpoint messages + + if message.content == REMINDER_MESSAGE: + return # Ignore bots own verification reminder + + if message.author.bot: + # They're a bot, delete their message after the delay. + await message.delete(delay=constants.Verification.bot_message_delete_delay) + return + + # if a user mentions a role or guild member + # alert the mods in mod-alerts channel + if message.mentions or message.role_mentions: + log.debug( + f"{message.author} mentioned one or more users " + f"and/or roles in {message.channel.name}" + ) + + embed_text = ( + f"{format_user(message.author)} sent a message in " + f"{message.channel.mention} that contained user and/or role mentions." + f"\n\n**Original message:**\n>>> {message.content}" + ) + + # Send pretty mod log embed to mod-alerts + await self.mod_log.send_log_message( + icon_url=constants.Icons.filtering, + colour=discord.Colour(constants.Colours.soft_red), + title=f"User/Role mentioned in {message.channel.name}", + text=embed_text, + thumbnail=message.author.avatar_url_as(static_format="png"), + channel_id=constants.Channels.mod_alerts, + ) + + ctx: Context = await self.bot.get_context(message) + if ctx.command is not None and ctx.command.name == "accept": + return + + if any(r.id == constants.Roles.verified for r in ctx.author.roles): + log.info( + f"{ctx.author} posted '{ctx.message.content}' " + "in the verification channel, but is already verified." + ) + return + + log.debug( + f"{ctx.author} posted '{ctx.message.content}' in the verification " + "channel. We are providing instructions how to verify." + ) + await ctx.send( + f"{ctx.author.mention} Please type `!accept` to verify that you accept our rules, " + f"and gain access to the rest of the server.", + delete_after=20 + ) + + log.trace(f"Deleting the message posted by {ctx.author}") + with suppress(discord.NotFound): + await ctx.message.delete() + + # endregion + # region: task management commands + + @has_any_role(*constants.MODERATION_ROLES) + @group(name="verification") + async def verification_group(self, ctx: Context) -> None: + """Manage internal verification tasks.""" + if ctx.invoked_subcommand is None: + await ctx.send_help(ctx.command) + + @verification_group.command(name="status") + async def status_cmd(self, ctx: Context) -> None: + """Check whether verification tasks are running.""" + log.trace("Checking status of verification tasks") + + if self.update_unverified_members.is_running(): + update_status = f"{constants.Emojis.incident_actioned} Member update task is running." + else: + update_status = f"{constants.Emojis.incident_unactioned} Member update task is **not** running." + + mention = f"<@&{constants.Roles.unverified}>" + if self.ping_unverified.is_running(): + ping_status = f"{constants.Emojis.incident_actioned} Ping {mention} task is running." + else: + ping_status = f"{constants.Emojis.incident_unactioned} Ping {mention} task is **not** running." + + embed = discord.Embed( + title="Verification system", + description=f"{update_status}\n{ping_status}", + colour=discord.Colour.blurple(), + ) + await ctx.send(embed=embed) + + @verification_group.command(name="start") + async def start_cmd(self, ctx: Context) -> None: + """Start verification tasks if they are not already running.""" + log.info("Starting verification tasks") + + if not self.update_unverified_members.is_running(): + self.update_unverified_members.start() + + if not self.ping_unverified.is_running(): + self.ping_unverified.start() + + await self.task_cache.set("tasks_running", 1) + + colour = discord.Colour.blurple() + await ctx.send(embed=discord.Embed(title="Verification system", description="Done. :ok_hand:", colour=colour)) + + @verification_group.command(name="stop", aliases=["kill"]) + async def stop_cmd(self, ctx: Context) -> None: + """Stop verification tasks.""" + log.info("Stopping verification tasks") + + self._stop_tasks(gracefully=False) + await self.task_cache.set("tasks_running", 0) + + colour = discord.Colour.blurple() + await ctx.send(embed=discord.Embed(title="Verification system", description="Tasks canceled.", colour=colour)) + + # endregion + # region: accept and subscribe commands + + def _bump_verified_stats(self, verified_member: discord.Member) -> None: + """ + Increment verification stats for `verified_member`. + + Each member falls into one of the three categories: + * Verified within 24 hours after joining + * Does not have @Unverified role yet + * Does have @Unverified role + + Stats for member kicking are handled separately. + """ + if verified_member.joined_at is None: # Docs mention this can happen + return + + if (datetime.utcnow() - verified_member.joined_at) < timedelta(hours=24): + category = "accepted_on_day_one" + elif constants.Roles.unverified not in [role.id for role in verified_member.roles]: + category = "accepted_before_unverified" + else: + category = "accepted_after_unverified" + + log.trace(f"Bumping verification stats in category: {category}") + self.bot.stats.incr(f"verification.{category}") + + @command(name='accept', aliases=('verify', 'verified', 'accepted'), hidden=True) + @has_no_roles(constants.Roles.verified) + @in_whitelist(channels=(constants.Channels.verification,)) + async def accept_command(self, ctx: Context, *_) -> None: # We don't actually care about the args + """Accept our rules and gain access to the rest of the server.""" + log.debug(f"{ctx.author} called !accept. Assigning the 'Developer' role.") + await ctx.author.add_roles(discord.Object(constants.Roles.verified), reason="Accepted the rules") + + self._bump_verified_stats(ctx.author) # This checks for @Unverified so make sure it's not yet removed + + if constants.Roles.unverified in [role.id for role in ctx.author.roles]: + log.debug(f"Removing Unverified role from: {ctx.author}") + await ctx.author.remove_roles(discord.Object(constants.Roles.unverified)) + + try: + await safe_dm(ctx.author.send(VERIFIED_MESSAGE)) + except discord.HTTPException: + log.exception(f"Sending welcome message failed for {ctx.author}.") + finally: + log.trace(f"Deleting accept message by {ctx.author}.") + with suppress(discord.NotFound): + self.mod_log.ignore(constants.Event.message_delete, ctx.message.id) + await ctx.message.delete() + + @command(name='subscribe') + @in_whitelist(channels=(constants.Channels.bot_commands,)) + async def subscribe_command(self, ctx: Context, *_) -> None: # We don't actually care about the args + """Subscribe to announcement notifications by assigning yourself the role.""" + has_role = False + + for role in ctx.author.roles: + if role.id == constants.Roles.announcements: + has_role = True + break + + if has_role: + await ctx.send(f"{ctx.author.mention} You're already subscribed!") + return + + log.debug(f"{ctx.author} called !subscribe. Assigning the 'Announcements' role.") + await ctx.author.add_roles(discord.Object(constants.Roles.announcements), reason="Subscribed to announcements") + + log.trace(f"Deleting the message posted by {ctx.author}.") + + await ctx.send( + f"{ctx.author.mention} Subscribed to <#{constants.Channels.announcements}> notifications.", + ) + + @command(name='unsubscribe') + @in_whitelist(channels=(constants.Channels.bot_commands,)) + async def unsubscribe_command(self, ctx: Context, *_) -> None: # We don't actually care about the args + """Unsubscribe from announcement notifications by removing the role from yourself.""" + has_role = False + + for role in ctx.author.roles: + if role.id == constants.Roles.announcements: + has_role = True + break + + if not has_role: + await ctx.send(f"{ctx.author.mention} You're already unsubscribed!") + return + + log.debug(f"{ctx.author} called !unsubscribe. Removing the 'Announcements' role.") + await ctx.author.remove_roles( + discord.Object(constants.Roles.announcements), reason="Unsubscribed from announcements" + ) + + log.trace(f"Deleting the message posted by {ctx.author}.") + + await ctx.send( + f"{ctx.author.mention} Unsubscribed from <#{constants.Channels.announcements}> notifications." + ) + + # endregion + # region: miscellaneous + + # This cannot be static (must have a __func__ attribute). + async def cog_command_error(self, ctx: Context, error: Exception) -> None: + """Check for & ignore any InWhitelistCheckFailure.""" + if isinstance(error, InWhitelistCheckFailure): + error.handled = True + + @staticmethod + async def bot_check(ctx: Context) -> bool: + """Block any command within the verification channel that is not !accept.""" + is_verification = ctx.channel.id == constants.Channels.verification + if is_verification and await has_no_roles_check(ctx, *constants.MODERATION_ROLES): + return ctx.command.name == "accept" + else: + return True + + # endregion + + +def setup(bot: Bot) -> None: + """Load the Verification cog.""" + bot.add_cog(Verification(bot)) diff --git a/bot/exts/moderation/watchchannels/__init__.py b/bot/exts/moderation/watchchannels/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/bot/exts/moderation/watchchannels/__init__.py diff --git a/bot/cogs/watchchannels/watchchannel.py b/bot/exts/moderation/watchchannels/_watchchannel.py index 479820444..7118dee02 100644 --- a/bot/cogs/watchchannels/watchchannel.py +++ b/bot/exts/moderation/watchchannels/_watchchannel.py @@ -14,8 +14,10 @@ from discord.ext.commands import Cog, Context from bot.api import ResponseCodeError from bot.bot import Bot -from bot.cogs.moderation import ModLog from bot.constants import BigBrother as BigBrotherConfig, Guild as GuildConfig, Icons +from bot.exts.filters.token_remover import TokenRemover +from bot.exts.filters.webhook_remover import WEBHOOK_URL_RE +from bot.exts.moderation.modlog import ModLog from bot.pagination import LinePaginator from bot.utils import CogABCMeta, messages from bot.utils.time import time_since @@ -82,7 +84,7 @@ class WatchChannel(metaclass=CogABCMeta): exc = self._consume_task.exception() if exc: self.log.exception( - f"The message queue consume task has failed with:", + "The message queue consume task has failed with:", exc_info=exc ) return False @@ -146,7 +148,7 @@ class WatchChannel(metaclass=CogABCMeta): try: data = await self.bot.api_client.get(self.api_endpoint, params=self.api_default_params) except ResponseCodeError as err: - self.log.exception(f"Failed to fetch the watched users from the API", exc_info=err) + self.log.exception("Failed to fetch the watched users from the API", exc_info=err) return False self.watched_users = defaultdict(dict) @@ -173,7 +175,7 @@ class WatchChannel(metaclass=CogABCMeta): self.log.trace(f"Sleeping {BigBrotherConfig.log_delay} seconds before consuming message queue") await asyncio.sleep(BigBrotherConfig.log_delay) - self.log.trace(f"Started consuming the message queue") + self.log.trace("Started consuming the message queue") # If the previous consumption Task failed, first consume the existing comsumption_queue if not self.consumption_queue: @@ -204,11 +206,12 @@ class WatchChannel(metaclass=CogABCMeta): embed: Optional[Embed] = None, ) -> None: """Sends a message to the webhook with the specified kwargs.""" + username = messages.sub_clyde(username) try: await self.webhook.send(content=content, username=username, avatar_url=avatar_url, embed=embed) except discord.HTTPException as exc: self.log.exception( - f"Failed to send a message to the webhook", + "Failed to send a message to the webhook", exc_info=exc ) @@ -225,14 +228,16 @@ class WatchChannel(metaclass=CogABCMeta): await self.send_header(msg) - cleaned_content = msg.clean_content - - if cleaned_content: + if TokenRemover.find_token_in_message(msg) or WEBHOOK_URL_RE.search(msg.content): + cleaned_content = "Content is censored because it contains a bot or webhook token." + elif cleaned_content := msg.clean_content: # Put all non-media URLs in a code block to prevent embeds media_urls = {embed.url for embed in msg.embeds if embed.type in ("image", "video")} for url in URL_RE.findall(cleaned_content): if url not in media_urls: cleaned_content = cleaned_content.replace(url, f"`{url}`") + + if cleaned_content: await self.webhook_send( cleaned_content, username=msg.author.display_name, @@ -254,7 +259,7 @@ class WatchChannel(metaclass=CogABCMeta): ) except discord.HTTPException as exc: self.log.exception( - f"Failed to send an attachment to the webhook", + "Failed to send an attachment to the webhook", exc_info=exc ) @@ -280,15 +285,20 @@ class WatchChannel(metaclass=CogABCMeta): else: message_jump = f"in [#{msg.channel.name}]({msg.jump_url})" + footer = f"Added {time_delta} by {actor} | Reason: {reason}" embed = Embed(description=f"{msg.author.mention} {message_jump}") - embed.set_footer(text=f"Added {time_delta} by {actor} | Reason: {reason}") + embed.set_footer(text=textwrap.shorten(footer, width=128, placeholder="...")) await self.webhook_send(embed=embed, username=msg.author.display_name, avatar_url=msg.author.avatar_url) - async def list_watched_users(self, ctx: Context, update_cache: bool = True) -> None: + async def list_watched_users( + self, ctx: Context, oldest_first: bool = False, update_cache: bool = True + ) -> None: """ Gives an overview of the watched user list for this channel. + The optional kwarg `oldest_first` orders the list by oldest entry. + The optional kwarg `update_cache` specifies whether the cache should be refreshed by polling the API. """ @@ -303,7 +313,11 @@ class WatchChannel(metaclass=CogABCMeta): time_delta = self._get_time_delta(inserted_at) lines.append(f"• <@{user_id}> (added {time_delta})") + if oldest_first: + lines.reverse() + lines = lines or ("There's nothing here yet.",) + embed = Embed( title=f"{self.__class__.__name__} watched users ({'updated' if update_cache else 'cached'})", color=Color.blue() @@ -326,13 +340,13 @@ class WatchChannel(metaclass=CogABCMeta): def cog_unload(self) -> None: """Takes care of unloading the cog and canceling the consumption task.""" - self.log.trace(f"Unloading the cog") + self.log.trace("Unloading the cog") if self._consume_task and not self._consume_task.done(): self._consume_task.cancel() try: self._consume_task.result() except asyncio.CancelledError as e: self.log.exception( - f"The consume task was canceled. Messages may be lost.", + "The consume task was canceled. Messages may be lost.", exc_info=e ) diff --git a/bot/cogs/watchchannels/bigbrother.py b/bot/exts/moderation/watchchannels/bigbrother.py index 37f2d2b9d..3b44056d3 100644 --- a/bot/cogs/watchchannels/bigbrother.py +++ b/bot/exts/moderation/watchchannels/bigbrother.py @@ -1,14 +1,14 @@ import logging +import textwrap from collections import ChainMap -from discord.ext.commands import Cog, Context, group +from discord.ext.commands import Cog, Context, group, has_any_role from bot.bot import Bot -from bot.cogs.moderation.utils import post_infraction from bot.constants import Channels, MODERATION_ROLES, Webhooks from bot.converters import FetchedMember -from bot.decorators import with_role -from .watchchannel import WatchChannel +from bot.exts.moderation.infraction._utils import post_infraction +from bot.exts.moderation.watchchannels._watchchannel import WatchChannel log = logging.getLogger(__name__) @@ -27,24 +27,39 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): ) @group(name='bigbrother', aliases=('bb',), invoke_without_command=True) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def bigbrother_group(self, ctx: Context) -> None: """Monitors users by relaying their messages to the Big Brother watch channel.""" - await ctx.send_help("bigbrother") + await ctx.send_help(ctx.command) @bigbrother_group.command(name='watched', aliases=('all', 'list')) - @with_role(*MODERATION_ROLES) - async def watched_command(self, ctx: Context, update_cache: bool = True) -> None: + @has_any_role(*MODERATION_ROLES) + async def watched_command( + self, ctx: Context, oldest_first: bool = False, update_cache: bool = True + ) -> None: """ Shows the users that are currently being monitored by Big Brother. + The optional kwarg `oldest_first` can be used to order the list by oldest watched. + The optional kwarg `update_cache` can be used to update the user cache using the API before listing the users. """ - await self.list_watched_users(ctx, update_cache) + await self.list_watched_users(ctx, oldest_first=oldest_first, update_cache=update_cache) + + @bigbrother_group.command(name='oldest') + @has_any_role(*MODERATION_ROLES) + async def oldest_command(self, ctx: Context, update_cache: bool = True) -> None: + """ + Shows Big Brother monitored users ordered by oldest watched. - @bigbrother_group.command(name='watch', aliases=('w',)) - @with_role(*MODERATION_ROLES) + The optional kwarg `update_cache` can be used to update the user + cache using the API before listing the users. + """ + await ctx.invoke(self.watched_command, oldest_first=True, update_cache=update_cache) + + @bigbrother_group.command(name='watch', aliases=('w',), root_aliases=('watch',)) + @has_any_role(*MODERATION_ROLES) async def watch_command(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: """ Relay messages sent by the given `user` to the `#big-brother` channel. @@ -54,8 +69,8 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): """ await self.apply_watch(ctx, user, reason) - @bigbrother_group.command(name='unwatch', aliases=('uw',)) - @with_role(*MODERATION_ROLES) + @bigbrother_group.command(name='unwatch', aliases=('uw',), root_aliases=('unwatch',)) + @has_any_role(*MODERATION_ROLES) async def unwatch_command(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: """Stop relaying messages by the given `user`.""" await self.apply_unwatch(ctx, user, reason) @@ -97,8 +112,8 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): if len(history) > 1: total = f"({len(history) // 2} previous infractions in total)" - end_reason = history[0]["reason"] - start_reason = f"Watched: {history[1]['reason']}" + end_reason = textwrap.shorten(history[0]["reason"], width=500, placeholder="...") + start_reason = f"Watched: {textwrap.shorten(history[1]['reason'], width=500, placeholder='...')}" msg += f"\n\nUser's previous watch reasons {total}:```{start_reason}\n\n{end_reason}```" else: msg = ":x: Failed to post the infraction: response was empty." @@ -115,8 +130,8 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): active_watches = await self.bot.api_client.get( self.api_endpoint, params=ChainMap( + {"user__id": str(user.id)}, self.api_default_params, - {"user__id": str(user.id)} ) ) if active_watches: @@ -147,3 +162,8 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): message = ":x: The specified user is currently not being watched." await ctx.send(message) + + +def setup(bot: Bot) -> None: + """Load the BigBrother cog.""" + bot.add_cog(BigBrother(bot)) diff --git a/bot/cogs/watchchannels/talentpool.py b/bot/exts/moderation/watchchannels/talentpool.py index b8473963d..a77dbe156 100644 --- a/bot/cogs/watchchannels/talentpool.py +++ b/bot/exts/moderation/watchchannels/talentpool.py @@ -1,18 +1,18 @@ import logging import textwrap from collections import ChainMap +from typing import Union -from discord import Color, Embed, Member -from discord.ext.commands import Cog, Context, group +from discord import Color, Embed, Member, User +from discord.ext.commands import Cog, Context, group, has_any_role from bot.api import ResponseCodeError from bot.bot import Bot from bot.constants import Channels, Guild, MODERATION_ROLES, STAFF_ROLES, Webhooks from bot.converters import FetchedMember -from bot.decorators import with_role +from bot.exts.moderation.watchchannels._watchchannel import WatchChannel from bot.pagination import LinePaginator from bot.utils import time -from .watchchannel import WatchChannel log = logging.getLogger(__name__) @@ -31,24 +31,39 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): ) @group(name='talentpool', aliases=('tp', 'talent', 'nomination', 'n'), invoke_without_command=True) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def nomination_group(self, ctx: Context) -> None: """Highlights the activity of helper nominees by relaying their messages to the talent pool channel.""" - await ctx.send_help("talentpool") + await ctx.send_help(ctx.command) - @nomination_group.command(name='watched', aliases=('all', 'list')) - @with_role(*MODERATION_ROLES) - async def watched_command(self, ctx: Context, update_cache: bool = True) -> None: + @nomination_group.command(name='watched', aliases=('all', 'list'), root_aliases=("nominees",)) + @has_any_role(*MODERATION_ROLES) + async def watched_command( + self, ctx: Context, oldest_first: bool = False, update_cache: bool = True + ) -> None: """ Shows the users that are currently being monitored in the talent pool. + The optional kwarg `oldest_first` can be used to order the list by oldest nomination. + The optional kwarg `update_cache` can be used to update the user cache using the API before listing the users. """ - await self.list_watched_users(ctx, update_cache) + await self.list_watched_users(ctx, oldest_first=oldest_first, update_cache=update_cache) + + @nomination_group.command(name='oldest') + @has_any_role(*MODERATION_ROLES) + async def oldest_command(self, ctx: Context, update_cache: bool = True) -> None: + """ + Shows talent pool monitored users ordered by oldest nomination. - @nomination_group.command(name='watch', aliases=('w', 'add', 'a')) - @with_role(*STAFF_ROLES) + The optional kwarg `update_cache` can be used to update the user + cache using the API before listing the users. + """ + await ctx.invoke(self.watched_command, oldest_first=True, update_cache=update_cache) + + @nomination_group.command(name='watch', aliases=('w', 'add', 'a'), root_aliases=("nominate",)) + @has_any_role(*STAFF_ROLES) async def watch_command(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: """ Relay messages sent by the given `user` to the `#talent-pool` channel. @@ -61,7 +76,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): return if isinstance(user, Member) and any(role.id in STAFF_ROLES for role in user.roles): - await ctx.send(f":x: Nominating staff members, eh? Here's a cookie :cookie:") + await ctx.send(":x: Nominating staff members, eh? Here's a cookie :cookie:") return if not await self.fetch_user_cache(): @@ -106,14 +121,14 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): if history: total = f"({len(history)} previous nominations in total)" - start_reason = f"Watched: {history[0]['reason']}" - end_reason = f"Unwatched: {history[0]['end_reason']}" + start_reason = f"Watched: {textwrap.shorten(history[0]['reason'], width=500, placeholder='...')}" + end_reason = f"Unwatched: {textwrap.shorten(history[0]['end_reason'], width=500, placeholder='...')}" msg += f"\n\nUser's previous watch reasons {total}:```{start_reason}\n\n{end_reason}```" await ctx.send(msg) @nomination_group.command(name='history', aliases=('info', 'search')) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def history_command(self, ctx: Context, user: FetchedMember) -> None: """Shows the specified user's nomination history.""" result = await self.bot.api_client.get( @@ -141,42 +156,27 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): max_size=1000 ) - @nomination_group.command(name='unwatch', aliases=('end', )) - @with_role(*MODERATION_ROLES) + @nomination_group.command(name='unwatch', aliases=('end', ), root_aliases=("unnominate",)) + @has_any_role(*MODERATION_ROLES) async def unwatch_command(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: """ Ends the active nomination of the specified user with the given reason. Providing a `reason` is required. """ - active_nomination = await self.bot.api_client.get( - self.api_endpoint, - params=ChainMap( - self.api_default_params, - {"user__id": str(user.id)} - ) - ) - - if not active_nomination: + if await self.unwatch(user.id, reason): + await ctx.send(f":white_check_mark: Messages sent by {user} will no longer be relayed") + else: await ctx.send(":x: The specified user does not have an active nomination") - return - - [nomination] = active_nomination - await self.bot.api_client.patch( - f"{self.api_endpoint}/{nomination['id']}", - json={'end_reason': reason, 'active': False} - ) - await ctx.send(f":white_check_mark: Messages sent by {user} will no longer be relayed") - self._remove_user(user.id) @nomination_group.group(name='edit', aliases=('e',), invoke_without_command=True) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def nomination_edit_group(self, ctx: Context) -> None: """Commands to edit nominations.""" - await ctx.send_help("talentpool edit") + await ctx.send_help(ctx.command) @nomination_edit_group.command(name='reason') - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def edit_reason_command(self, ctx: Context, nomination_id: int, *, reason: str) -> None: """ Edits the reason/unnominate reason for the nomination with the given `id` depending on the status. @@ -205,6 +205,36 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): await ctx.send(f":white_check_mark: Updated the {field} of the nomination!") + @Cog.listener() + async def on_member_ban(self, guild: Guild, user: Union[User, Member]) -> None: + """Remove `user` from the talent pool after they are banned.""" + await self.unwatch(user.id, "User was banned.") + + async def unwatch(self, user_id: int, reason: str) -> bool: + """End the active nomination of a user with the given reason and return True on success.""" + active_nomination = await self.bot.api_client.get( + self.api_endpoint, + params=ChainMap( + {"user__id": str(user_id)}, + self.api_default_params, + ) + ) + + if not active_nomination: + log.debug(f"No active nominate exists for {user_id=}") + return False + + log.info(f"Ending nomination: {user_id=} {reason=}") + + nomination = active_nomination[0] + await self.bot.api_client.patch( + f"{self.api_endpoint}/{nomination['id']}", + json={'end_reason': reason, 'active': False} + ) + self._remove_user(user_id) + + return True + def _nomination_to_string(self, nomination_object: dict) -> str: """Creates a string representation of a nomination.""" guild = self.bot.get_guild(Guild.id) @@ -247,3 +277,8 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): ) return lines.strip() + + +def setup(bot: Bot) -> None: + """Load the TalentPool cog.""" + bot.add_cog(TalentPool(bot)) diff --git a/bot/exts/utils/__init__.py b/bot/exts/utils/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/bot/exts/utils/__init__.py diff --git a/bot/cogs/bot.py b/bot/exts/utils/bot.py index 963dc4926..7ed487d47 100644 --- a/bot/cogs/bot.py +++ b/bot/exts/utils/bot.py @@ -5,12 +5,12 @@ import time from typing import Optional, Tuple from discord import Embed, Message, RawMessageUpdateEvent, TextChannel -from discord.ext.commands import Cog, Context, command, group +from discord.ext.commands import Cog, Context, command, group, has_any_role from bot.bot import Bot -from bot.cogs.token_remover import TokenRemover -from bot.constants import Channels, DEBUG_MODE, Guild, MODERATION_ROLES, Roles, URLs -from bot.decorators import with_role +from bot.constants import Categories, Channels, DEBUG_MODE, Guild, MODERATION_ROLES, Roles, URLs +from bot.exts.filters.token_remover import TokenRemover +from bot.exts.filters.webhook_remover import WEBHOOK_URL_RE from bot.utils.messages import wait_for_deletion log = logging.getLogger(__name__) @@ -26,14 +26,6 @@ class BotCog(Cog, name="Bot"): # Stores allowed channels plus epoch time since last call. self.channel_cooldowns = { - Channels.help_0: 0, - Channels.help_1: 0, - Channels.help_2: 0, - Channels.help_3: 0, - Channels.help_4: 0, - Channels.help_5: 0, - Channels.help_6: 0, - Channels.help_7: 0, Channels.python_discussion: 0, } @@ -46,13 +38,13 @@ class BotCog(Cog, name="Bot"): self.codeblock_message_ids = {} @group(invoke_without_command=True, name="bot", hidden=True) - @with_role(Roles.verified) + @has_any_role(Roles.verified) async def botinfo_group(self, ctx: Context) -> None: """Bot informational commands.""" - await ctx.send_help("bot") + await ctx.send_help(ctx.command) @botinfo_group.command(name='about', aliases=('info',), hidden=True) - @with_role(Roles.verified) + @has_any_role(Roles.verified) async def about_command(self, ctx: Context) -> None: """Get information about the bot.""" embed = Embed( @@ -70,7 +62,7 @@ class BotCog(Cog, name="Bot"): await ctx.send(embed=embed) @command(name='echo', aliases=('print',)) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def echo_command(self, ctx: Context, channel: Optional[TextChannel], *, text: str) -> None: """Repeat the given message in either a specified channel or the current channel.""" if channel is None: @@ -79,11 +71,15 @@ class BotCog(Cog, name="Bot"): await channel.send(text) @command(name='embed') - @with_role(*MODERATION_ROLES) - async def embed_command(self, ctx: Context, *, text: str) -> None: - """Send the input within an embed to the current channel.""" + @has_any_role(*MODERATION_ROLES) + async def embed_command(self, ctx: Context, channel: Optional[TextChannel], *, text: str) -> None: + """Send the input within an embed to either a specified channel or the current channel.""" embed = Embed(description=text) - await ctx.send(embed=embed) + + if channel is None: + await ctx.send(embed=embed) + else: + await channel.send(embed=embed) def codeblock_stripping(self, msg: str, bad_ticks: bool) -> Optional[Tuple[Tuple[str, ...], str]]: """ @@ -231,14 +227,20 @@ class BotCog(Cog, name="Bot"): If poorly formatted code is detected, send the user a helpful message explaining how to do properly formatted Python syntax highlighting codeblocks. """ + is_help_channel = ( + getattr(msg.channel, "category", None) + and msg.channel.category.id in (Categories.help_available, Categories.help_in_use) + ) parse_codeblock = ( ( - msg.channel.id in self.channel_cooldowns + is_help_channel + or msg.channel.id in self.channel_cooldowns or msg.channel.id in self.channel_whitelist ) and not msg.author.bot and len(msg.content.splitlines()) > 3 - and not TokenRemover.is_token_in_message(msg) + and not TokenRemover.find_token_in_message(msg) + and not WEBHOOK_URL_RE.search(msg.content) ) if parse_codeblock: # no token in the msg @@ -329,12 +331,14 @@ class BotCog(Cog, name="Bot"): log.trace("The code consists only of expressions, not sending instructions") if howto != "": + # Increase amount of codeblock correction in stats + self.bot.stats.incr("codeblock_corrections") howto_embed = Embed(description=howto) bot_message = await msg.channel.send(f"Hey {msg.author.mention}!", embed=howto_embed) self.codeblock_message_ids[msg.id] = bot_message.id self.bot.loop.create_task( - wait_for_deletion(bot_message, user_ids=(msg.author.id,), client=self.bot) + wait_for_deletion(bot_message, (msg.author.id,), self.bot) ) else: return diff --git a/bot/cogs/clean.py b/bot/exts/utils/clean.py index 5cdf0b048..bf25cb4c2 100644 --- a/bot/cogs/clean.py +++ b/bot/exts/utils/clean.py @@ -1,18 +1,17 @@ import logging import random import re -from typing import Optional +from typing import Iterable, Optional from discord import Colour, Embed, Message, TextChannel, User -from discord.ext.commands import Cog, Context, group +from discord.ext import commands +from discord.ext.commands import Cog, Context, group, has_any_role from bot.bot import Bot -from bot.cogs.moderation import ModLog from bot.constants import ( - Channels, CleanMessages, Colours, Event, - Icons, MODERATION_ROLES, NEGATIVE_REPLIES + Channels, CleanMessages, Colours, Event, Icons, MODERATION_ROLES, NEGATIVE_REPLIES ) -from bot.decorators import with_role +from bot.exts.moderation.modlog import ModLog log = logging.getLogger(__name__) @@ -41,10 +40,11 @@ class Clean(Cog): self, amount: int, ctx: Context, + channels: Iterable[TextChannel], bots_only: bool = False, user: User = None, regex: Optional[str] = None, - channel: Optional[TextChannel] = None + until_message: Optional[Message] = None, ) -> None: """A helper function that does the actual message cleaning.""" def predicate_bots_only(message: Message) -> bool: @@ -110,48 +110,60 @@ class Clean(Cog): predicate = None # Delete all messages # Default to using the invoking context's channel - if not channel: - channel = ctx.channel + if not channels: + channels = [ctx.channel] + + # Delete the invocation first + self.mod_log.ignore(Event.message_delete, ctx.message.id) + await ctx.message.delete() - # Look through the history and retrieve message data messages = [] message_ids = [] self.cleaning = True - invocation_deleted = False - # To account for the invocation message, we index `amount + 1` messages. - async for message in channel.history(limit=amount + 1): + # Find the IDs of the messages to delete. IDs are needed in order to ignore mod log events. + for channel in channels: + async for message in channel.history(limit=amount): + + # If at any point the cancel command is invoked, we should stop. + if not self.cleaning: + return - # If at any point the cancel command is invoked, we should stop. - if not self.cleaning: - return + # If we are looking for specific message. + if until_message: - # Always start by deleting the invocation - if not invocation_deleted: - self.mod_log.ignore(Event.message_delete, message.id) - await message.delete() - invocation_deleted = True - continue + # we could use ID's here however in case if the message we are looking for gets deleted, + # we won't have a way to figure that out thus checking for datetime should be more reliable + if message.created_at < until_message.created_at: + # means we have found the message until which we were supposed to be deleting. + break - # If the message passes predicate, let's save it. - if predicate is None or predicate(message): - message_ids.append(message.id) - messages.append(message) + # Since we will be using `delete_messages` method of a TextChannel and we need message objects to + # use it as well as to send logs we will start appending messages here instead adding them from + # purge. + messages.append(message) + + # If the message passes predicate, let's save it. + if predicate is None or predicate(message): + message_ids.append(message.id) self.cleaning = False - # We should ignore the ID's we stored, so we don't get mod-log spam. + # Now let's delete the actual messages with purge. self.mod_log.ignore(Event.message_delete, *message_ids) - - # Use bulk delete to actually do the cleaning. It's far faster. - await channel.purge( - limit=amount, - check=predicate - ) + for channel in channels: + if until_message: + for i in range(0, len(messages), 100): + # while purge automatically handles the amount of messages + # delete_messages only allows for up to 100 messages at once + # thus we need to paginate the amount to always be <= 100 + await channel.delete_messages(messages[i:i + 100]) + else: + messages += await channel.purge(limit=amount, check=predicate) # Reverse the list to restore chronological order if messages: - messages = list(reversed(messages)) + messages = reversed(messages) log_url = await self.mod_log.upload_log(messages, ctx.author.id) else: # Can't build an embed, nothing to clean! @@ -163,8 +175,11 @@ class Clean(Cog): return # Build the embed and send it + target_channels = ", ".join(channel.mention for channel in channels) + message = ( - f"**{len(message_ids)}** messages deleted in <#{channel.id}> by **{ctx.author.name}**\n\n" + f"**{len(message_ids)}** messages deleted in {target_channels} by " + f"{ctx.author.mention}\n\n" f"A log of the deleted messages can be found [here]({log_url})." ) @@ -177,59 +192,70 @@ class Clean(Cog): ) @group(invoke_without_command=True, name="clean", aliases=["purge"]) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def clean_group(self, ctx: Context) -> None: """Commands for cleaning messages in channels.""" - await ctx.invoke(self.bot.get_command("help"), "clean") + await ctx.send_help(ctx.command) @clean_group.command(name="user", aliases=["users"]) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def clean_user( self, ctx: Context, user: User, amount: Optional[int] = 10, - channel: TextChannel = None + channels: commands.Greedy[TextChannel] = None ) -> None: """Delete messages posted by the provided user, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, user=user, channel=channel) + await self._clean_messages(amount, ctx, user=user, channels=channels) @clean_group.command(name="all", aliases=["everything"]) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def clean_all( self, ctx: Context, amount: Optional[int] = 10, - channel: TextChannel = None + channels: commands.Greedy[TextChannel] = None ) -> None: """Delete all messages, regardless of poster, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, channel=channel) + await self._clean_messages(amount, ctx, channels=channels) @clean_group.command(name="bots", aliases=["bot"]) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def clean_bots( self, ctx: Context, amount: Optional[int] = 10, - channel: TextChannel = None + channels: commands.Greedy[TextChannel] = None ) -> None: """Delete all messages posted by a bot, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, bots_only=True, channel=channel) + await self._clean_messages(amount, ctx, bots_only=True, channels=channels) @clean_group.command(name="regex", aliases=["word", "expression"]) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def clean_regex( self, ctx: Context, regex: str, amount: Optional[int] = 10, - channel: TextChannel = None + channels: commands.Greedy[TextChannel] = None ) -> None: """Delete all messages that match a certain regex, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, regex=regex, channel=channel) + await self._clean_messages(amount, ctx, regex=regex, channels=channels) + + @clean_group.command(name="message", aliases=["messages"]) + @has_any_role(*MODERATION_ROLES) + async def clean_message(self, ctx: Context, message: Message) -> None: + """Delete all messages until certain message, stop cleaning after hitting the `message`.""" + await self._clean_messages( + CleanMessages.message_limit, + ctx, + channels=[message.channel], + until_message=message + ) @clean_group.command(name="stop", aliases=["cancel", "abort"]) - @with_role(*MODERATION_ROLES) + @has_any_role(*MODERATION_ROLES) async def clean_cancel(self, ctx: Context) -> None: """If there is an ongoing cleaning process, attempt to immediately cancel it.""" self.cleaning = False diff --git a/bot/cogs/eval.py b/bot/exts/utils/eval.py index 2d52197e8..6419b320e 100644 --- a/bot/cogs/eval.py +++ b/bot/exts/utils/eval.py @@ -9,12 +9,12 @@ from io import StringIO from typing import Any, Optional, Tuple import discord -from discord.ext.commands import Cog, Context, group +from discord.ext.commands import Cog, Context, group, has_any_role from bot.bot import Bot from bot.constants import Roles -from bot.decorators import with_role from bot.interpreter import Interpreter +from bot.utils import find_nth_occurrence, send_to_paste_service log = logging.getLogger(__name__) @@ -171,17 +171,41 @@ async def func(): # (None,) -> Any res = traceback.format_exc() out, embed = self._format(code, res) + out = out.rstrip("\n") # Strip empty lines from output + + # Truncate output to max 15 lines or 1500 characters + newline_truncate_index = find_nth_occurrence(out, "\n", 15) + + if newline_truncate_index is None or newline_truncate_index > 1500: + truncate_index = 1500 + else: + truncate_index = newline_truncate_index + + if len(out) > truncate_index: + paste_link = await send_to_paste_service(self.bot.http_session, out, extension="py") + if paste_link is not None: + paste_text = f"full contents at {paste_link}" + else: + paste_text = "failed to upload contents to paste service." + + await ctx.send( + f"```py\n{out[:truncate_index]}\n```" + f"... response truncated; {paste_text}", + embed=embed + ) + return + await ctx.send(f"```py\n{out}```", embed=embed) @group(name='internal', aliases=('int',)) - @with_role(Roles.owners, Roles.admins) + @has_any_role(Roles.owners, Roles.admins) async def internal_group(self, ctx: Context) -> None: """Internal commands. Top secret!""" if not ctx.invoked_subcommand: - await ctx.send_help("internal") + await ctx.send_help(ctx.command) @internal_group.command(name='eval', aliases=('e',)) - @with_role(Roles.admins, Roles.owners) + @has_any_role(Roles.admins, Roles.owners) async def eval(self, ctx: Context, *, code: str) -> None: """Run eval in a REPL-like format.""" code = code.strip("`") diff --git a/bot/cogs/extensions.py b/bot/exts/utils/extensions.py index 4493046e1..418db0150 100644 --- a/bot/cogs/extensions.py +++ b/bot/exts/utils/extensions.py @@ -2,25 +2,22 @@ import functools import logging import typing as t from enum import Enum -from pkgutil import iter_modules from discord import Colour, Embed from discord.ext import commands from discord.ext.commands import Context, group +from bot import exts from bot.bot import Bot from bot.constants import Emojis, MODERATION_ROLES, Roles, URLs from bot.pagination import LinePaginator -from bot.utils.checks import with_role_check +from bot.utils.extensions import EXTENSIONS, unqualify log = logging.getLogger(__name__) -UNLOAD_BLACKLIST = {"bot.cogs.extensions", "bot.cogs.modlog"} -EXTENSIONS = frozenset( - ext.name - for ext in iter_modules(("bot/cogs",), "bot.cogs.") - if ext.name[-1] != "_" -) + +UNLOAD_BLACKLIST = {f"{exts.__name__}.utils.extensions", f"{exts.__name__}.moderation.modlog"} +BASE_PATH_LEN = len(exts.__name__.split(".")) class Action(Enum): @@ -47,11 +44,25 @@ class Extension(commands.Converter): argument = argument.lower() - if "." not in argument: - argument = f"bot.cogs.{argument}" - if argument in EXTENSIONS: return argument + elif (qualified_arg := f"{exts.__name__}.{argument}") in EXTENSIONS: + return qualified_arg + + matches = [] + for ext in EXTENSIONS: + if argument == unqualify(ext): + matches.append(ext) + + if len(matches) > 1: + matches.sort() + names = "\n".join(matches) + raise commands.BadArgument( + f":x: `{argument}` is an ambiguous extension name. " + f"Please use one of the following fully-qualified names.```\n{names}```" + ) + elif matches: + return matches[0] else: raise commands.BadArgument(f":x: Could not find the extension `{argument}`.") @@ -65,7 +76,7 @@ class Extensions(commands.Cog): @group(name="extensions", aliases=("ext", "exts", "c", "cogs"), invoke_without_command=True) async def extensions_group(self, ctx: Context) -> None: """Load, unload, reload, and list loaded extensions.""" - await ctx.send_help("extensions") + await ctx.send_help(ctx.command) @extensions_group.command(name="load", aliases=("l",)) async def load_command(self, ctx: Context, *extensions: Extension) -> None: @@ -75,7 +86,7 @@ class Extensions(commands.Cog): If '\*' or '\*\*' is given as the name, all unloaded extensions will be loaded. """ # noqa: W605 if not extensions: - await ctx.send_help("extensions load") + await ctx.send_help(ctx.command) return if "*" in extensions or "**" in extensions: @@ -92,7 +103,7 @@ class Extensions(commands.Cog): If '\*' or '\*\*' is given as the name, all loaded extensions will be unloaded. """ # noqa: W605 if not extensions: - await ctx.send_help("extensions unload") + await ctx.send_help(ctx.command) return blacklisted = "\n".join(UNLOAD_BLACKLIST & set(extensions)) @@ -107,7 +118,7 @@ class Extensions(commands.Cog): await ctx.send(msg) - @extensions_group.command(name="reload", aliases=("r",)) + @extensions_group.command(name="reload", aliases=("r",), root_aliases=("reload",)) async def reload_command(self, ctx: Context, *extensions: Extension) -> None: r""" Reload extensions given their fully qualified or unqualified names. @@ -118,7 +129,7 @@ class Extensions(commands.Cog): If '\*\*' is given as the name, all extensions, including unloaded ones, will be reloaded. """ # noqa: W605 if not extensions: - await ctx.send_help("extensions reload") + await ctx.send_help(ctx.command) return if "**" in extensions: @@ -139,27 +150,44 @@ class Extensions(commands.Cog): Grey indicates that the extension is unloaded. Green indicates that the extension is currently loaded. """ - embed = Embed() - lines = [] - - embed.colour = Colour.blurple() + embed = Embed(colour=Colour.blurple()) embed.set_author( name="Extensions List", url=URLs.github_bot_repo, icon_url=URLs.bot_avatar ) - for ext in sorted(list(EXTENSIONS)): + lines = [] + categories = self.group_extension_statuses() + for category, extensions in sorted(categories.items()): + # Treat each category as a single line by concatenating everything. + # This ensures the paginator will not cut off a page in the middle of a category. + category = category.replace("_", " ").title() + extensions = "\n".join(sorted(extensions)) + lines.append(f"**{category}**\n{extensions}\n") + + log.debug(f"{ctx.author} requested a list of all cogs. Returning a paginated list.") + await LinePaginator.paginate(lines, ctx, embed, scale_to_size=700, empty=False) + + def group_extension_statuses(self) -> t.Mapping[str, str]: + """Return a mapping of extension names and statuses to their categories.""" + categories = {} + + for ext in EXTENSIONS: if ext in self.bot.extensions: status = Emojis.status_online else: status = Emojis.status_offline - ext = ext.rsplit(".", 1)[1] - lines.append(f"{status} {ext}") + path = ext.split(".") + if len(path) > BASE_PATH_LEN + 1: + category = " - ".join(path[BASE_PATH_LEN:-1]) + else: + category = "uncategorised" - log.debug(f"{ctx.author} requested a list of all cogs. Returning a paginated list.") - await LinePaginator.paginate(lines, ctx, embed, max_size=300, empty=False) + categories.setdefault(category, []).append(f"{status} {path[-1]}") + + return categories def batch_manage(self, action: Action, *extensions: str) -> str: """ @@ -219,9 +247,9 @@ class Extensions(commands.Cog): return msg, error_msg # This cannot be static (must have a __func__ attribute). - def cog_check(self, ctx: Context) -> bool: + async def cog_check(self, ctx: Context) -> bool: """Only allow moderators and core developers to invoke the commands in this cog.""" - return with_role_check(ctx, *MODERATION_ROLES, Roles.core_developers) + return await commands.has_any_role(*MODERATION_ROLES, Roles.core_developers).predicate(ctx) # This cannot be static (must have a __func__ attribute). async def cog_command_error(self, ctx: Context, error: Exception) -> None: diff --git a/bot/cogs/jams.py b/bot/exts/utils/jams.py index 1d062b0c2..1c0988343 100644 --- a/bot/cogs/jams.py +++ b/bot/exts/utils/jams.py @@ -1,15 +1,18 @@ import logging +import typing as t -from discord import Member, PermissionOverwrite, utils +from discord import CategoryChannel, Guild, Member, PermissionOverwrite, Role from discord.ext import commands from more_itertools import unique_everseen from bot.bot import Bot from bot.constants import Roles -from bot.decorators import with_role log = logging.getLogger(__name__) +MAX_CHANNELS = 50 +CATEGORY_NAME = "Code Jam" + class CodeJams(commands.Cog): """Manages the code-jam related parts of our server.""" @@ -18,7 +21,7 @@ class CodeJams(commands.Cog): self.bot = bot @commands.command() - @with_role(Roles.admins) + @commands.has_any_role(Roles.admins) async def createteam(self, ctx: commands.Context, team_name: str, members: commands.Greedy[Member]) -> None: """ Create team channels (voice and text) in the Code Jams category, assign roles, and add overwrites for the team. @@ -40,22 +43,47 @@ class CodeJams(commands.Cog): ) return - code_jam_category = utils.get(ctx.guild.categories, name="Code Jam") + team_channel = await self.create_channels(ctx.guild, team_name, members) + await self.add_roles(ctx.guild, members) - if code_jam_category is None: - log.info("Code Jam category not found, creating it.") + await ctx.send( + f":ok_hand: Team created: {team_channel}\n" + f"**Team Leader:** {members[0].mention}\n" + f"**Team Members:** {' '.join(member.mention for member in members[1:])}" + ) - category_overwrites = { - ctx.guild.default_role: PermissionOverwrite(read_messages=False), - ctx.guild.me: PermissionOverwrite(read_messages=True) - } + async def get_category(self, guild: Guild) -> CategoryChannel: + """ + Return a code jam category. - code_jam_category = await ctx.guild.create_category_channel( - "Code Jam", - overwrites=category_overwrites, - reason="It's code jam time!" - ) + If all categories are full or none exist, create a new category. + """ + for category in guild.categories: + # Need 2 available spaces: one for the text channel and one for voice. + if category.name == CATEGORY_NAME and MAX_CHANNELS - len(category.channels) >= 2: + return category + + return await self.create_category(guild) + + @staticmethod + async def create_category(guild: Guild) -> CategoryChannel: + """Create a new code jam category and return it.""" + log.info("Creating a new code jam category.") + + category_overwrites = { + guild.default_role: PermissionOverwrite(read_messages=False), + guild.me: PermissionOverwrite(read_messages=True) + } + + return await guild.create_category_channel( + CATEGORY_NAME, + overwrites=category_overwrites, + reason="It's code jam time!" + ) + @staticmethod + def get_overwrites(members: t.List[Member], guild: Guild) -> t.Dict[t.Union[Member, Role], PermissionOverwrite]: + """Get code jam team channels permission overwrites.""" # First member is always the team leader team_channel_overwrites = { members[0]: PermissionOverwrite( @@ -64,8 +92,8 @@ class CodeJams(commands.Cog): manage_webhooks=True, connect=True ), - ctx.guild.default_role: PermissionOverwrite(read_messages=False, connect=False), - ctx.guild.get_role(Roles.verified): PermissionOverwrite( + guild.default_role: PermissionOverwrite(read_messages=False, connect=False), + guild.get_role(Roles.verified): PermissionOverwrite( read_messages=False, connect=False ) @@ -78,8 +106,16 @@ class CodeJams(commands.Cog): connect=True ) + return team_channel_overwrites + + async def create_channels(self, guild: Guild, team_name: str, members: t.List[Member]) -> str: + """Create team text and voice channels. Return the mention for the text channel.""" + # Get permission overwrites and category + team_channel_overwrites = self.get_overwrites(members, guild) + code_jam_category = await self.get_category(guild) + # Create a text channel for the team - team_channel = await ctx.guild.create_text_channel( + team_channel = await guild.create_text_channel( team_name, overwrites=team_channel_overwrites, category=code_jam_category @@ -88,26 +124,25 @@ class CodeJams(commands.Cog): # Create a voice channel for the team team_voice_name = " ".join(team_name.split("-")).title() - await ctx.guild.create_voice_channel( + await guild.create_voice_channel( team_voice_name, overwrites=team_channel_overwrites, category=code_jam_category ) + return team_channel.mention + + @staticmethod + async def add_roles(guild: Guild, members: t.List[Member]) -> None: + """Assign team leader and jammer roles.""" # Assign team leader role - await members[0].add_roles(ctx.guild.get_role(Roles.team_leaders)) + await members[0].add_roles(guild.get_role(Roles.team_leaders)) # Assign rest of roles - jammer_role = ctx.guild.get_role(Roles.jammers) + jammer_role = guild.get_role(Roles.jammers) for member in members: await member.add_roles(jammer_role) - await ctx.send( - f":ok_hand: Team created: {team_channel.mention}\n" - f"**Team Leader:** {members[0].mention}\n" - f"**Team Members:** {' '.join(member.mention for member in members[1:])}" - ) - def setup(bot: Bot) -> None: """Load the CodeJams cog.""" diff --git a/bot/exts/utils/ping.py b/bot/exts/utils/ping.py new file mode 100644 index 000000000..a9ca3dbeb --- /dev/null +++ b/bot/exts/utils/ping.py @@ -0,0 +1,59 @@ +import socket +from datetime import datetime + +import aioping +from discord import Embed +from discord.ext import commands + +from bot.bot import Bot +from bot.constants import Channels, Emojis, STAFF_ROLES, URLs +from bot.decorators import in_whitelist + +DESCRIPTIONS = ( + "Command processing time", + "Python Discord website latency", + "Discord API latency" +) +ROUND_LATENCY = 3 + + +class Latency(commands.Cog): + """Getting the latency between the bot and websites.""" + + def __init__(self, bot: Bot) -> None: + self.bot = bot + + @commands.command() + @in_whitelist(channels=(Channels.bot_commands,), roles=STAFF_ROLES) + async def ping(self, ctx: commands.Context) -> None: + """ + Gets different measures of latency within the bot. + + Returns bot, Python Discord Site, Discord Protocol latency. + """ + # datetime.datetime objects do not have the "milliseconds" attribute. + # It must be converted to seconds before converting to milliseconds. + bot_ping = (datetime.utcnow() - ctx.message.created_at).total_seconds() / 1000 + bot_ping = f"{bot_ping:.{ROUND_LATENCY}f} ms" + + try: + delay = await aioping.ping(URLs.site, family=socket.AddressFamily.AF_INET) * 1000 + site_ping = f"{delay:.{ROUND_LATENCY}f} ms" + + except TimeoutError: + site_ping = f"{Emojis.cross_mark} Connection timed out." + + # Discord Protocol latency return value is in seconds, must be multiplied by 1000 to get milliseconds. + discord_ping = f"{self.bot.latency * 1000:.{ROUND_LATENCY}f} ms" + + embed = Embed(title="Pong!") + + for desc, latency in zip(DESCRIPTIONS, [bot_ping, site_ping, discord_ping]): + embed.add_field(name=desc, value=latency, inline=False) + + await ctx.send(embed=embed) + + +def setup(bot: Bot) -> None: + """Load the Latency cog.""" + bot.add_cog(Latency(bot)) diff --git a/bot/cogs/reminders.py b/bot/exts/utils/reminders.py index d5f59dd62..efba7ad6e 100644 --- a/bot/cogs/reminders.py +++ b/bot/exts/utils/reminders.py @@ -9,31 +9,38 @@ from operator import itemgetter import discord from dateutil.parser import isoparse from dateutil.relativedelta import relativedelta -from discord.ext.commands import Cog, Context, group +from discord.ext.commands import Cog, Context, Greedy, group from bot.bot import Bot -from bot.constants import Guild, Icons, NEGATIVE_REPLIES, POSITIVE_REPLIES, STAFF_ROLES +from bot.constants import Guild, Icons, MODERATION_ROLES, POSITIVE_REPLIES, Roles, STAFF_ROLES from bot.converters import Duration from bot.pagination import LinePaginator -from bot.utils.checks import without_role_check +from bot.utils.checks import has_any_role_check, has_no_roles_check +from bot.utils.messages import send_denial from bot.utils.scheduling import Scheduler -from bot.utils.time import humanize_delta, wait_until +from bot.utils.time import humanize_delta log = logging.getLogger(__name__) WHITELISTED_CHANNELS = Guild.reminder_whitelist MAXIMUM_REMINDERS = 5 +Mentionable = t.Union[discord.Member, discord.Role] -class Reminders(Scheduler, Cog): + +class Reminders(Cog): """Provide in-channel reminder functionality.""" def __init__(self, bot: Bot): self.bot = bot - super().__init__() + self.scheduler = Scheduler(self.__class__.__name__) self.bot.loop.create_task(self.reschedule_reminders()) + def cog_unload(self) -> None: + """Cancel scheduled tasks.""" + self.scheduler.cancel_all() + async def reschedule_reminders(self) -> None: """Get all current reminders from the API and reschedule them.""" await self.bot.wait_until_guild_available() @@ -56,7 +63,7 @@ class Reminders(Scheduler, Cog): late = relativedelta(now, remind_at) await self.send_reminder(reminder, late) else: - self.schedule_task(reminder["id"], reminder) + self.schedule_reminder(reminder) def ensure_valid_reminder( self, @@ -99,17 +106,58 @@ class Reminders(Scheduler, Cog): await ctx.send(embed=embed) - async def _scheduled_task(self, reminder: dict) -> None: + @staticmethod + async def _check_mentions(ctx: Context, mentions: t.Iterable[Mentionable]) -> t.Tuple[bool, str]: + """ + Returns whether or not the list of mentions is allowed. + + Conditions: + - Role reminders are Mods+ + - Reminders for other users are Helpers+ + + If mentions aren't allowed, also return the type of mention(s) disallowed. + """ + if await has_no_roles_check(ctx, *STAFF_ROLES): + return False, "members/roles" + elif await has_no_roles_check(ctx, *MODERATION_ROLES): + return all(isinstance(mention, discord.Member) for mention in mentions), "roles" + else: + return True, "" + + @staticmethod + async def validate_mentions(ctx: Context, mentions: t.Iterable[Mentionable]) -> bool: + """ + Filter mentions to see if the user can mention, and sends a denial if not allowed. + + Returns whether or not the validation is successful. + """ + mentions_allowed, disallowed_mentions = await Reminders._check_mentions(ctx, mentions) + + if not mentions or mentions_allowed: + return True + else: + await send_denial(ctx, f"You can't mention other {disallowed_mentions} in your reminder!") + return False + + def get_mentionables(self, mention_ids: t.List[int]) -> t.Iterator[Mentionable]: + """Converts Role and Member ids to their corresponding objects if possible.""" + guild = self.bot.get_guild(Guild.id) + for mention_id in mention_ids: + if (mentionable := (guild.get_member(mention_id) or guild.get_role(mention_id))): + yield mentionable + + def schedule_reminder(self, reminder: dict) -> None: """A coroutine which sends the reminder once the time is reached, and cancels the running task.""" reminder_id = reminder["id"] reminder_datetime = isoparse(reminder['expiration']).replace(tzinfo=None) - # Send the reminder message once the desired duration has passed - await wait_until(reminder_datetime) - await self.send_reminder(reminder) + async def _remind() -> None: + await self.send_reminder(reminder) - log.debug(f"Deleting reminder {reminder_id} (the user has been reminded).") - await self._delete_reminder(reminder_id) + log.debug(f"Deleting reminder {reminder_id} (the user has been reminded).") + await self._delete_reminder(reminder_id) + + self.scheduler.schedule_at(reminder_datetime, reminder_id, _remind()) async def _delete_reminder(self, reminder_id: str, cancel_task: bool = True) -> None: """Delete a reminder from the database, given its ID, and cancel the running task.""" @@ -117,15 +165,28 @@ class Reminders(Scheduler, Cog): if cancel_task: # Now we can remove it from the schedule list - self.cancel_task(reminder_id) + self.scheduler.cancel(reminder_id) + + async def _edit_reminder(self, reminder_id: int, payload: dict) -> dict: + """ + Edits a reminder in the database given the ID and payload. + + Returns the edited reminder. + """ + # Send the request to update the reminder in the database + reminder = await self.bot.api_client.patch( + 'bot/reminders/' + str(reminder_id), + json=payload + ) + return reminder async def _reschedule_reminder(self, reminder: dict) -> None: """Reschedule a reminder object.""" log.trace(f"Cancelling old task #{reminder['id']}") - self.cancel_task(reminder["id"]) + self.scheduler.cancel(reminder["id"]) log.trace(f"Scheduling new task #{reminder['id']}") - self.schedule_task(reminder["id"], reminder) + self.schedule_reminder(reminder) async def send_reminder(self, reminder: dict, late: relativedelta = None) -> None: """Send the reminder.""" @@ -152,36 +213,39 @@ class Reminders(Scheduler, Cog): name=f"Sorry it arrived {humanize_delta(late, max_units=2)} late!" ) + additional_mentions = ' '.join( + mentionable.mention for mentionable in self.get_mentionables(reminder["mentions"]) + ) + await channel.send( - content=user.mention, + content=f"{user.mention} {additional_mentions}", embed=embed ) await self._delete_reminder(reminder["id"]) - @group(name="remind", aliases=("reminder", "reminders"), invoke_without_command=True) - async def remind_group(self, ctx: Context, expiration: Duration, *, content: str) -> None: + @group(name="remind", aliases=("reminder", "reminders", "remindme"), invoke_without_command=True) + async def remind_group( + self, ctx: Context, mentions: Greedy[Mentionable], expiration: Duration, *, content: str + ) -> None: """Commands for managing your reminders.""" - await self.new_reminder(ctx, expiration=expiration, content=content) + await self.new_reminder(ctx, mentions=mentions, expiration=expiration, content=content) @remind_group.command(name="new", aliases=("add", "create")) - async def new_reminder(self, ctx: Context, expiration: Duration, *, content: str) -> t.Optional[discord.Message]: + async def new_reminder( + self, ctx: Context, mentions: Greedy[Mentionable], expiration: Duration, *, content: str + ) -> None: """ Set yourself a simple reminder. Expiration is parsed per: http://strftime.org/ """ - embed = discord.Embed() - # If the user is not staff, we need to verify whether or not to make a reminder at all. - if without_role_check(ctx, *STAFF_ROLES): + if await has_no_roles_check(ctx, *STAFF_ROLES): # If they don't have permission to set a reminder in this channel if ctx.channel.id not in WHITELISTED_CHANNELS: - embed.colour = discord.Colour.red() - embed.title = random.choice(NEGATIVE_REPLIES) - embed.description = "Sorry, you can't do that here!" - - return await ctx.send(embed=embed) + await send_denial(ctx, "Sorry, you can't do that here!") + return # Get their current active reminders active_reminders = await self.bot.api_client.get( @@ -194,11 +258,18 @@ class Reminders(Scheduler, Cog): # Let's limit this, so we don't get 10 000 # reminders from kip or something like that :P if len(active_reminders) > MAXIMUM_REMINDERS: - embed.colour = discord.Colour.red() - embed.title = random.choice(NEGATIVE_REPLIES) - embed.description = "You have too many active reminders!" + await send_denial(ctx, "You have too many active reminders!") + return + + # Remove duplicate mentions + mentions = set(mentions) + mentions.discard(ctx.author) - return await ctx.send(embed=embed) + # Filter mentions to see if the user can mention members/roles + if not await self.validate_mentions(ctx, mentions): + return + + mention_ids = [mention.id for mention in mentions] # Now we can attempt to actually set the reminder. reminder = await self.bot.api_client.post( @@ -208,25 +279,30 @@ class Reminders(Scheduler, Cog): 'channel_id': ctx.message.channel.id, 'jump_url': ctx.message.jump_url, 'content': content, - 'expiration': expiration.isoformat() + 'expiration': expiration.isoformat(), + 'mentions': mention_ids, } ) now = datetime.utcnow() - timedelta(seconds=1) humanized_delta = humanize_delta(relativedelta(expiration, now)) + mention_string = ( + f"Your reminder will arrive in {humanized_delta} " + f"and will mention {len(mentions)} other(s)!" + ) # Confirm to the user that it worked. await self._send_confirmation( ctx, - on_success=f"Your reminder will arrive in {humanized_delta}!", + on_success=mention_string, reminder_id=reminder["id"], delivery_dt=expiration, ) - self.schedule_task(reminder["id"], reminder) + self.schedule_reminder(reminder) @remind_group.command(name="list") - async def list_reminders(self, ctx: Context) -> t.Optional[discord.Message]: + async def list_reminders(self, ctx: Context) -> None: """View a paginated embed of all reminders for your user.""" # Get all the user's reminders from the database. data = await self.bot.api_client.get( @@ -239,7 +315,7 @@ class Reminders(Scheduler, Cog): # Make a list of tuples so it can be sorted by time. reminders = sorted( ( - (rem['content'], rem['expiration'], rem['id']) + (rem['content'], rem['expiration'], rem['id'], rem['mentions']) for rem in data ), key=itemgetter(1) @@ -247,13 +323,19 @@ class Reminders(Scheduler, Cog): lines = [] - for content, remind_at, id_ in reminders: + for content, remind_at, id_, mentions in reminders: # Parse and humanize the time, make it pretty :D remind_datetime = isoparse(remind_at).replace(tzinfo=None) time = humanize_delta(relativedelta(remind_datetime, now)) + mentions = ", ".join( + # Both Role and User objects have the `name` attribute + mention.name for mention in self.get_mentionables(mentions) + ) + mention_string = f"\n**Mentions:** {mentions}" if mentions else "" + text = textwrap.dedent(f""" - **Reminder #{id_}:** *expires in {time}* (ID: {id_}) + **Reminder #{id_}:** *expires in {time}* (ID: {id_}){mention_string} {content} """).strip() @@ -266,7 +348,8 @@ class Reminders(Scheduler, Cog): # Remind the user that they have no reminders :^) if not lines: embed.description = "No active reminders could be found." - return await ctx.send(embed=embed) + await ctx.send(embed=embed) + return # Construct the embed and paginate it. embed.colour = discord.Colour.blurple() @@ -281,42 +364,44 @@ class Reminders(Scheduler, Cog): @remind_group.group(name="edit", aliases=("change", "modify"), invoke_without_command=True) async def edit_reminder_group(self, ctx: Context) -> None: """Commands for modifying your current reminders.""" - await ctx.send_help("reminders edit") + await ctx.send_help(ctx.command) @edit_reminder_group.command(name="duration", aliases=("time",)) async def edit_reminder_duration(self, ctx: Context, id_: int, expiration: Duration) -> None: """ - Edit one of your reminder's expiration. + Edit one of your reminder's expiration. Expiration is parsed per: http://strftime.org/ """ - # Send the request to update the reminder in the database - reminder = await self.bot.api_client.patch( - 'bot/reminders/' + str(id_), - json={'expiration': expiration.isoformat()} - ) - - # Send a confirmation message to the channel - await self._send_confirmation( - ctx, - on_success="That reminder has been edited successfully!", - reminder_id=id_, - delivery_dt=expiration, - ) - - await self._reschedule_reminder(reminder) + await self.edit_reminder(ctx, id_, {'expiration': expiration.isoformat()}) @edit_reminder_group.command(name="content", aliases=("reason",)) async def edit_reminder_content(self, ctx: Context, id_: int, *, content: str) -> None: """Edit one of your reminder's content.""" - # Send the request to update the reminder in the database - reminder = await self.bot.api_client.patch( - 'bot/reminders/' + str(id_), - json={'content': content} - ) + await self.edit_reminder(ctx, id_, {"content": content}) + + @edit_reminder_group.command(name="mentions", aliases=("pings",)) + async def edit_reminder_mentions(self, ctx: Context, id_: int, mentions: Greedy[Mentionable]) -> None: + """Edit one of your reminder's mentions.""" + # Remove duplicate mentions + mentions = set(mentions) + mentions.discard(ctx.author) + + # Filter mentions to see if the user can mention members/roles + if not await self.validate_mentions(ctx, mentions): + return + + mention_ids = [mention.id for mention in mentions] + await self.edit_reminder(ctx, id_, {"mentions": mention_ids}) + + async def edit_reminder(self, ctx: Context, id_: int, payload: dict) -> None: + """Edits a reminder with the given payload, then sends a confirmation message.""" + if not await self._can_modify(ctx, id_): + return + reminder = await self._edit_reminder(id_, payload) - # Parse the reminder expiration back into a datetime for the confirmation message - expiration = isoparse(reminder['expiration']).replace(tzinfo=None) + # Parse the reminder expiration back into a datetime + expiration = isoparse(reminder["expiration"]).replace(tzinfo=None) # Send a confirmation message to the channel await self._send_confirmation( @@ -330,6 +415,8 @@ class Reminders(Scheduler, Cog): @remind_group.command("delete", aliases=("remove", "cancel")) async def delete_reminder(self, ctx: Context, id_: int) -> None: """Delete one of your active reminders.""" + if not await self._can_modify(ctx, id_): + return await self._delete_reminder(id_) await self._send_confirmation( ctx, @@ -338,6 +425,24 @@ class Reminders(Scheduler, Cog): delivery_dt=None, ) + async def _can_modify(self, ctx: Context, reminder_id: t.Union[str, int]) -> bool: + """ + Check whether the reminder can be modified by the ctx author. + + The check passes when the user is an admin, or if they created the reminder. + """ + if await has_any_role_check(ctx, Roles.admins): + return True + + api_response = await self.bot.api_client.get(f"bot/reminders/{reminder_id}") + if not api_response["author"] == ctx.author.id: + log.debug(f"{ctx.author} is not the reminder author and does not pass the check.") + await send_denial(ctx, "You can't modify reminders of other users!") + return False + + log.debug(f"{ctx.author} is the reminder author and passes the check.") + return True + def setup(bot: Bot) -> None: """Load the Reminders cog.""" diff --git a/bot/cogs/snekbox.py b/bot/exts/utils/snekbox.py index 99c1a7278..ca6fbf5cb 100644 --- a/bot/cogs/snekbox.py +++ b/bot/exts/utils/snekbox.py @@ -12,8 +12,9 @@ from discord import HTTPException, Message, NotFound, Reaction, User from discord.ext.commands import Cog, Context, command, guild_only from bot.bot import Bot -from bot.constants import Channels, Roles, URLs -from bot.decorators import in_channel +from bot.constants import Categories, Channels, Roles, URLs +from bot.decorators import in_whitelist +from bot.utils import send_to_paste_service from bot.utils.messages import wait_for_deletion log = logging.getLogger(__name__) @@ -38,11 +39,16 @@ RAW_CODE_REGEX = re.compile( ) MAX_PASTE_LEN = 1000 + +# `!eval` command whitelists +EVAL_CHANNELS = (Channels.bot_commands, Channels.esoteric, Channels.code_help_voice) +EVAL_CATEGORIES = (Categories.help_available, Categories.help_in_use) EVAL_ROLES = (Roles.helpers, Roles.moderators, Roles.admins, Roles.owners, Roles.python_community, Roles.partners) SIGKILL = 9 REEVAL_EMOJI = '\U0001f501' # :repeat: +REEVAL_TIMEOUT = 30 class Snekbox(Cog): @@ -66,17 +72,7 @@ class Snekbox(Cog): if len(output) > MAX_PASTE_LEN: log.info("Full output is too long to upload") return "too long to upload" - - url = URLs.paste_service.format(key="documents") - try: - async with self.bot.http_session.post(url, data=output, raise_for_status=True) as resp: - data = await resp.json() - - if "key" in data: - return URLs.paste_service.format(key=data["key"]) - except Exception: - # 400 (Bad Request) means there are too many characters - log.exception("Failed to upload full output to paste service!") + return await send_to_paste_service(self.bot.http_session, output, extension="txt") @staticmethod def prepare_input(code: str) -> str: @@ -154,6 +150,7 @@ class Snekbox(Cog): output = output.replace("<!@", "<!@\u200B") # Zero-width space if ESCAPE_REGEX.findall(output): + paste_link = await self.upload_output(original_output) return "Code block escape attempt detected; will not output result", paste_link truncated = False @@ -197,14 +194,25 @@ class Snekbox(Cog): output, paste_link = await self.format_output(results["stdout"]) icon = self.get_status_emoji(results) - msg = f"{ctx.author.mention} {icon} {msg}.\n\n```py\n{output}\n```" + msg = f"{ctx.author.mention} {icon} {msg}.\n\n```\n{output}\n```" if paste_link: msg = f"{msg}\nFull output: {paste_link}" - response = await ctx.send(msg) - self.bot.loop.create_task( - wait_for_deletion(response, user_ids=(ctx.author.id,), client=ctx.bot) - ) + # Collect stats of eval fails + successes + if icon == ":x:": + self.bot.stats.incr("snekbox.python.fail") + else: + self.bot.stats.incr("snekbox.python.success") + + filter_cog = self.bot.get_cog("Filtering") + filter_triggered = False + if filter_cog: + filter_triggered = await filter_cog.filter_eval(msg, ctx.message) + if filter_triggered: + response = await ctx.send("Attempt to circumvent filter detected. Moderator team has been alerted.") + else: + response = await ctx.send(msg) + self.bot.loop.create_task(wait_for_deletion(response, (ctx.author.id,), ctx.bot)) log.info(f"{ctx.author}'s job had a return code of {results['returncode']}") return response @@ -223,7 +231,7 @@ class Snekbox(Cog): _, new_message = await self.bot.wait_for( 'message_edit', check=_predicate_eval_message_edit, - timeout=10 + timeout=REEVAL_TIMEOUT ) await ctx.message.add_reaction(REEVAL_EMOJI) await self.bot.wait_for( @@ -265,7 +273,7 @@ class Snekbox(Cog): @command(name="eval", aliases=("e",)) @guild_only() - @in_channel(Channels.bot_commands, hidden_channels=(Channels.esoteric,), bypass_roles=EVAL_ROLES) + @in_whitelist(channels=EVAL_CHANNELS, categories=EVAL_CATEGORIES, roles=EVAL_ROLES) async def eval_command(self, ctx: Context, *, code: str = None) -> None: """ Run Python code and get the results. @@ -285,9 +293,21 @@ class Snekbox(Cog): return if not code: # None or empty string - await ctx.send_help("eval") + await ctx.send_help(ctx.command) return + if Roles.helpers in (role.id for role in ctx.author.roles): + self.bot.stats.incr("snekbox_usages.roles.helpers") + else: + self.bot.stats.incr("snekbox_usages.roles.developers") + + if ctx.channel.category_id == Categories.help_in_use: + self.bot.stats.incr("snekbox_usages.channels.help") + elif ctx.channel.id == Channels.bot_commands: + self.bot.stats.incr("snekbox_usages.channels.bot_commands") + else: + self.bot.stats.incr("snekbox_usages.channels.topical") + log.info(f"Received code from {ctx.author} for evaluation:\n{code}") while True: diff --git a/bot/cogs/utils.py b/bot/exts/utils/utils.py index 0d34d4c71..3e9230414 100644 --- a/bot/cogs/utils.py +++ b/bot/exts/utils/utils.py @@ -2,19 +2,18 @@ import difflib import logging import re import unicodedata -from asyncio import TimeoutError, sleep from email.parser import HeaderParser from io import StringIO from typing import Tuple, Union -from dateutil import relativedelta -from discord import Colour, Embed, Message, Role -from discord.ext.commands import BadArgument, Cog, Context, command +from discord import Colour, Embed, utils +from discord.ext.commands import BadArgument, Cog, Context, clean_content, command, has_any_role from bot.bot import Bot -from bot.constants import Channels, MODERATION_ROLES, Mention, STAFF_ROLES -from bot.decorators import in_channel, with_role -from bot.utils.time import humanize_delta +from bot.constants import Channels, MODERATION_ROLES, STAFF_ROLES +from bot.decorators import in_whitelist +from bot.pagination import LinePaginator +from bot.utils import messages log = logging.getLogger(__name__) @@ -58,7 +57,7 @@ class Utils(Cog): if pep_number.isdigit(): pep_number = int(pep_number) else: - await ctx.send_help("pep") + await ctx.send_help(ctx.command) return # Handle PEP 0 directly because it's not in .rst or .txt so it can't be accessed like other PEPs. @@ -85,7 +84,7 @@ class Utils(Cog): # Assemble the embed pep_embed = Embed( title=f"**PEP {pep_number} - {pep_header['Title']}**", - description=f"[Link]({self.base_pep_url}{pep_number:04})", + url=f"{self.base_pep_url}{pep_number:04}" ) pep_embed.set_thumbnail(url=ICON_URL) @@ -118,27 +117,20 @@ class Utils(Cog): await ctx.message.channel.send(embed=pep_embed) @command() - @in_channel(Channels.bot_commands, bypass_roles=STAFF_ROLES) + @in_whitelist(channels=(Channels.bot_commands,), roles=STAFF_ROLES) async def charinfo(self, ctx: Context, *, characters: str) -> None: - """Shows you information on up to 25 unicode characters.""" + """Shows you information on up to 50 unicode characters.""" match = re.match(r"<(a?):(\w+):(\d+)>", characters) if match: - embed = Embed( - title="Non-Character Detected", - description=( - "Only unicode characters can be processed, but a custom Discord emoji " - "was found. Please remove it and try again." - ) + return await messages.send_denial( + ctx, + "**Non-Character Detected**\n" + "Only unicode characters can be processed, but a custom Discord emoji " + "was found. Please remove it and try again." ) - embed.colour = Colour.red() - await ctx.send(embed=embed) - return - if len(characters) > 25: - embed = Embed(title=f"Too many characters ({len(characters)}/25)") - embed.colour = Colour.red() - await ctx.send(embed=embed) - return + if len(characters) > 50: + return await messages.send_denial(ctx, f"Too many characters ({len(characters)}/50)") def get_info(char: str) -> Tuple[str, str]: digit = f"{ord(char):x}" @@ -148,59 +140,17 @@ class Utils(Cog): u_code = f"\\U{digit:>08}" url = f"https://www.compart.com/en/unicode/U+{digit:>04}" name = f"[{unicodedata.name(char, '')}]({url})" - info = f"`{u_code.ljust(10)}`: {name} - {char}" + info = f"`{u_code.ljust(10)}`: {name} - {utils.escape_markdown(char)}" return info, u_code - charlist, rawlist = zip(*(get_info(c) for c in characters)) - - embed = Embed(description="\n".join(charlist)) - embed.set_author(name="Character Info") + char_list, raw_list = zip(*(get_info(c) for c in characters)) + embed = Embed().set_author(name="Character Info") if len(characters) > 1: - embed.add_field(name='Raw', value=f"`{''.join(rawlist)}`", inline=False) - - await ctx.send(embed=embed) - - @command() - @with_role(*MODERATION_ROLES) - async def mention(self, ctx: Context, *, role: Role) -> None: - """Set a role to be mentionable for a limited time.""" - if role.mentionable: - await ctx.send(f"{role} is already mentionable!") - return + # Maximum length possible is 502 out of 1024, so there's no need to truncate. + embed.add_field(name='Full Raw Text', value=f"`{''.join(raw_list)}`", inline=False) - await role.edit(reason=f"Role unlocked by {ctx.author}", mentionable=True) - - human_time = humanize_delta(relativedelta.relativedelta(seconds=Mention.message_timeout)) - await ctx.send( - f"{role} has been made mentionable. I will reset it in {human_time}, or when someone mentions this role." - ) - - def check(m: Message) -> bool: - """Checks that the message contains the role mention.""" - return role in m.role_mentions - - try: - msg = await self.bot.wait_for("message", check=check, timeout=Mention.message_timeout) - except TimeoutError: - await role.edit(mentionable=False, reason="Automatic role lock - timeout.") - await ctx.send(f"{ctx.author.mention}, you took too long. I have reset {role} to be unmentionable.") - return - - if any(r.id in MODERATION_ROLES for r in msg.author.roles): - await sleep(Mention.reset_delay) - await role.edit(mentionable=False, reason=f"Automatic role lock by {msg.author}") - await ctx.send( - f"{ctx.author.mention}, I have reset {role} to be unmentionable as " - f"{msg.author if msg.author != ctx.author else 'you'} sent a message mentioning it." - ) - return - - await role.edit(mentionable=False, reason=f"Automatic role lock - unauthorised use by {msg.author}") - await ctx.send( - f"{ctx.author.mention}, I have reset {role} to be unmentionable " - f"as I detected unauthorised use by {msg.author} (ID: {msg.author.id})." - ) + await LinePaginator.paginate(char_list, ctx, embed, max_lines=10, max_size=2000, empty=False) @command() async def zen(self, ctx: Context, *, search_value: Union[int, str, None] = None) -> None: @@ -274,14 +224,16 @@ class Utils(Cog): await ctx.send(embed=embed) @command(aliases=("poll",)) - @with_role(*MODERATION_ROLES) - async def vote(self, ctx: Context, title: str, *options: str) -> None: + @has_any_role(*MODERATION_ROLES) + async def vote(self, ctx: Context, title: clean_content(fix_channel_mentions=True), *options: str) -> None: """ Build a quick voting poll with matching reactions with the provided options. A maximum of 20 options can be provided, as Discord supports a max of 20 reactions on a single message. """ + if len(title) > 256: + raise BadArgument("The title cannot be longer than 256 characters.") if len(options) < 2: raise BadArgument("Please provide at least 2 options.") if len(options) > 20: @@ -297,8 +249,8 @@ class Utils(Cog): async def send_pep_zero(self, ctx: Context) -> None: """Send information about PEP 0.""" pep_embed = Embed( - title=f"**PEP 0 - Index of Python Enhancement Proposals (PEPs)**", - description=f"[Link](https://www.python.org/dev/peps/)" + title="**PEP 0 - Index of Python Enhancement Proposals (PEPs)**", + url="https://www.python.org/dev/peps/" ) pep_embed.set_thumbnail(url=ICON_URL) pep_embed.add_field(name="Status", value="Active") diff --git a/bot/pagination.py b/bot/pagination.py index 90c8f849c..182b2fa76 100644 --- a/bot/pagination.py +++ b/bot/pagination.py @@ -37,12 +37,19 @@ class LinePaginator(Paginator): The suffix appended at the end of every page. e.g. three backticks. * max_size: `int` The maximum amount of codepoints allowed in a page. + * scale_to_size: `int` + The maximum amount of characters a single line can scale up to. * max_lines: `int` The maximum amount of lines allowed in a page. """ def __init__( - self, prefix: str = '```', suffix: str = '```', max_size: int = 2000, max_lines: int = None + self, + prefix: str = '```', + suffix: str = '```', + max_size: int = 2000, + scale_to_size: int = 2000, + max_lines: t.Optional[int] = None ) -> None: """ This function overrides the Paginator.__init__ from inside discord.ext.commands. @@ -51,7 +58,21 @@ class LinePaginator(Paginator): """ self.prefix = prefix self.suffix = suffix + + # Embeds that exceed 2048 characters will result in an HTTPException + # (Discord API limit), so we've set a limit of 2000 + if max_size > 2000: + raise ValueError(f"max_size must be <= 2,000 characters. ({max_size} > 2000)") + self.max_size = max_size - len(suffix) + + if scale_to_size < max_size: + raise ValueError(f"scale_to_size must be >= max_size. ({scale_to_size} < {max_size})") + + if scale_to_size > 2000: + raise ValueError(f"scale_to_size must be <= 2,000 characters. ({scale_to_size} > 2000)") + + self.scale_to_size = scale_to_size - len(suffix) self.max_lines = max_lines self._current_page = [prefix] self._linecount = 0 @@ -62,23 +83,38 @@ class LinePaginator(Paginator): """ Adds a line to the current page. - If the line exceeds the `self.max_size` then an exception is raised. + If a line on a page exceeds `max_size` characters, then `max_size` will go up to + `scale_to_size` for a single line before creating a new page for the overflow words. If it + is still exceeded, the excess characters are stored and placed on the next pages unti + there are none remaining (by word boundary). The line is truncated if `scale_to_size` is + still exceeded after attempting to continue onto the next page. + + In the case that the page already contains one or more lines and the new lines would cause + `max_size` to be exceeded, a new page is created. This is done in order to make a best + effort to avoid breaking up single lines across pages, while keeping the total length of the + page at a reasonable size. This function overrides the `Paginator.add_line` from inside `discord.ext.commands`. It overrides in order to allow us to configure the maximum number of lines per page. """ - if len(line) > self.max_size - len(self.prefix) - 2: - raise RuntimeError('Line exceeds maximum page size %s' % (self.max_size - len(self.prefix) - 2)) - - if self.max_lines is not None: - if self._linecount >= self.max_lines: - self._linecount = 0 - self.close_page() - - self._linecount += 1 - if self._count + len(line) + 1 > self.max_size: - self.close_page() + remaining_words = None + if len(line) > (max_chars := self.max_size - len(self.prefix) - 2): + if len(line) > self.scale_to_size: + line, remaining_words = self._split_remaining_words(line, max_chars) + if len(line) > self.scale_to_size: + log.debug("Could not continue to next page, truncating line.") + line = line[:self.scale_to_size] + + # Check if we should start a new page or continue the line on the current one + if self.max_lines is not None and self._linecount >= self.max_lines: + log.debug("max_lines exceeded, creating new page.") + self._new_page() + elif self._count + len(line) + 1 > self.max_size and self._linecount > 0: + log.debug("max_size exceeded on page with lines, creating new page.") + self._new_page() + + self._linecount += 1 self._count += len(line) + 1 self._current_page.append(line) @@ -87,6 +123,65 @@ class LinePaginator(Paginator): self._current_page.append('') self._count += 1 + # Start a new page if there were any overflow words + if remaining_words: + self._new_page() + self.add_line(remaining_words) + + def _new_page(self) -> None: + """ + Internal: start a new page for the paginator. + + This closes the current page and resets the counters for the new page's line count and + character count. + """ + self._linecount = 0 + self._count = len(self.prefix) + 1 + self.close_page() + + def _split_remaining_words(self, line: str, max_chars: int) -> t.Tuple[str, t.Optional[str]]: + """ + Internal: split a line into two strings -- reduced_words and remaining_words. + + reduced_words: the remaining words in `line`, after attempting to remove all words that + exceed `max_chars` (rounding down to the nearest word boundary). + + remaining_words: the words in `line` which exceed `max_chars`. This value is None if + no words could be split from `line`. + + If there are any remaining_words, an ellipses is appended to reduced_words and a + continuation header is inserted before remaining_words to visually communicate the line + continuation. + + Return a tuple in the format (reduced_words, remaining_words). + """ + reduced_words = [] + remaining_words = [] + + # "(Continued)" is used on a line by itself to indicate the continuation of last page + continuation_header = "(Continued)\n-----------\n" + reduced_char_count = 0 + is_full = False + + for word in line.split(" "): + if not is_full: + if len(word) + reduced_char_count <= max_chars: + reduced_words.append(word) + reduced_char_count += len(word) + 1 + else: + # If reduced_words is empty, we were unable to split the words across pages + if not reduced_words: + return line, None + is_full = True + remaining_words.append(word) + else: + remaining_words.append(word) + + return ( + " ".join(reduced_words) + "..." if remaining_words else "", + continuation_header + " ".join(remaining_words) if remaining_words else None + ) + @classmethod async def paginate( cls, @@ -97,12 +192,13 @@ class LinePaginator(Paginator): suffix: str = "", max_lines: t.Optional[int] = None, max_size: int = 500, + scale_to_size: int = 2000, empty: bool = True, restrict_to_user: User = None, timeout: int = 300, footer_text: str = None, url: str = None, - exception_on_empty_embed: bool = False + exception_on_empty_embed: bool = False, ) -> t.Optional[discord.Message]: """ Use a paginator and set of reactions to provide pagination over a set of lines. @@ -142,12 +238,13 @@ class LinePaginator(Paginator): )) ) - paginator = cls(prefix=prefix, suffix=suffix, max_size=max_size, max_lines=max_lines) + paginator = cls(prefix=prefix, suffix=suffix, max_size=max_size, max_lines=max_lines, + scale_to_size=scale_to_size) current_page = 0 if not lines: if exception_on_empty_embed: - log.exception(f"Pagination asked for empty lines iterable") + log.exception("Pagination asked for empty lines iterable") raise EmptyPaginatorEmbed("No lines to paginate") log.debug("No lines to add to paginator, adding '(nothing to display)' message") @@ -216,8 +313,6 @@ class LinePaginator(Paginator): log.debug(f"Got first page reaction - changing to page 1/{len(paginator.pages)}") - embed.description = "" - await message.edit(embed=embed) embed.description = paginator.pages[current_page] if footer_text: embed.set_footer(text=f"{footer_text} (Page {current_page + 1}/{len(paginator.pages)})") @@ -231,8 +326,6 @@ class LinePaginator(Paginator): log.debug(f"Got last page reaction - changing to page {current_page + 1}/{len(paginator.pages)}") - embed.description = "" - await message.edit(embed=embed) embed.description = paginator.pages[current_page] if footer_text: embed.set_footer(text=f"{footer_text} (Page {current_page + 1}/{len(paginator.pages)})") @@ -250,8 +343,6 @@ class LinePaginator(Paginator): current_page -= 1 log.debug(f"Got previous page reaction - changing to page {current_page + 1}/{len(paginator.pages)}") - embed.description = "" - await message.edit(embed=embed) embed.description = paginator.pages[current_page] if footer_text: @@ -271,8 +362,6 @@ class LinePaginator(Paginator): current_page += 1 log.debug(f"Got next page reaction - changing to page {current_page + 1}/{len(paginator.pages)}") - embed.description = "" - await message.edit(embed=embed) embed.description = paginator.pages[current_page] if footer_text: @@ -285,169 +374,3 @@ class LinePaginator(Paginator): log.debug("Ending pagination and clearing reactions.") with suppress(discord.NotFound): await message.clear_reactions() - - -class ImagePaginator(Paginator): - """ - Helper class that paginates images for embeds in messages. - - Close resemblance to LinePaginator, except focuses on images over text. - - Refer to ImagePaginator.paginate for documentation on how to use. - """ - - def __init__(self, prefix: str = "", suffix: str = ""): - super().__init__(prefix, suffix) - self._current_page = [prefix] - self.images = [] - self._pages = [] - self._count = 0 - - def add_line(self, line: str = '', *, empty: bool = False) -> None: - """Adds a line to each page.""" - if line: - self._count = len(line) - else: - self._count = 0 - self._current_page.append(line) - self.close_page() - - def add_image(self, image: str = None) -> None: - """Adds an image to a page.""" - self.images.append(image) - - @classmethod - async def paginate( - cls, - pages: t.List[t.Tuple[str, str]], - ctx: Context, embed: discord.Embed, - prefix: str = "", - suffix: str = "", - timeout: int = 300, - exception_on_empty_embed: bool = False - ) -> t.Optional[discord.Message]: - """ - Use a paginator and set of reactions to provide pagination over a set of title/image pairs. - - The reactions are used to switch page, or to finish with pagination. - - When used, this will send a message using `ctx.send()` and apply a set of reactions to it. These reactions may - be used to change page, or to remove pagination from the message. - - Note: Pagination will be removed automatically if no reaction is added for five minutes (300 seconds). - - Example: - >>> embed = discord.Embed() - >>> embed.set_author(name="Some Operation", url=url, icon_url=icon) - >>> await ImagePaginator.paginate(pages, ctx, embed) - """ - def check_event(reaction_: discord.Reaction, member: discord.Member) -> bool: - """Checks each reaction added, if it matches our conditions pass the wait_for.""" - return all(( - # Reaction is on the same message sent - reaction_.message.id == message.id, - # The reaction is part of the navigation menu - str(reaction_.emoji) in PAGINATION_EMOJI, - # The reactor is not a bot - not member.bot - )) - - paginator = cls(prefix=prefix, suffix=suffix) - current_page = 0 - - if not pages: - if exception_on_empty_embed: - log.exception(f"Pagination asked for empty image list") - raise EmptyPaginatorEmbed("No images to paginate") - - log.debug("No images to add to paginator, adding '(no images to display)' message") - pages.append(("(no images to display)", "")) - - for text, image_url in pages: - paginator.add_line(text) - paginator.add_image(image_url) - - embed.description = paginator.pages[current_page] - image = paginator.images[current_page] - - if image: - embed.set_image(url=image) - - if len(paginator.pages) <= 1: - return await ctx.send(embed=embed) - - embed.set_footer(text=f"Page {current_page + 1}/{len(paginator.pages)}") - message = await ctx.send(embed=embed) - - for emoji in PAGINATION_EMOJI: - await message.add_reaction(emoji) - - while True: - # Start waiting for reactions - try: - reaction, user = await ctx.bot.wait_for("reaction_add", timeout=timeout, check=check_event) - except asyncio.TimeoutError: - log.debug("Timed out waiting for a reaction") - break # We're done, no reactions for the last 5 minutes - - # Deletes the users reaction - await message.remove_reaction(reaction.emoji, user) - - # Delete reaction press - [:trashcan:] - if str(reaction.emoji) == DELETE_EMOJI: - log.debug("Got delete reaction") - return await message.delete() - - # First reaction press - [:track_previous:] - if reaction.emoji == FIRST_EMOJI: - if current_page == 0: - log.debug("Got first page reaction, but we're on the first page - ignoring") - continue - - current_page = 0 - reaction_type = "first" - - # Last reaction press - [:track_next:] - if reaction.emoji == LAST_EMOJI: - if current_page >= len(paginator.pages) - 1: - log.debug("Got last page reaction, but we're on the last page - ignoring") - continue - - current_page = len(paginator.pages) - 1 - reaction_type = "last" - - # Previous reaction press - [:arrow_left: ] - if reaction.emoji == LEFT_EMOJI: - if current_page <= 0: - log.debug("Got previous page reaction, but we're on the first page - ignoring") - continue - - current_page -= 1 - reaction_type = "previous" - - # Next reaction press - [:arrow_right:] - if reaction.emoji == RIGHT_EMOJI: - if current_page >= len(paginator.pages) - 1: - log.debug("Got next page reaction, but we're on the last page - ignoring") - continue - - current_page += 1 - reaction_type = "next" - - # Magic happens here, after page and reaction_type is set - embed.description = "" - await message.edit(embed=embed) - embed.description = paginator.pages[current_page] - - image = paginator.images[current_page] - if image: - embed.set_image(url=image) - - embed.set_footer(text=f"Page {current_page + 1}/{len(paginator.pages)}") - log.debug(f"Got {reaction_type} page reaction - changing to page {current_page + 1}/{len(paginator.pages)}") - - await message.edit(embed=embed) - - log.debug("Ending pagination and clearing reactions.") - with suppress(discord.NotFound): - await message.clear_reactions() diff --git a/bot/resources/elements.json b/bot/resources/elements.json new file mode 100644 index 000000000..2dc9b6fd6 --- /dev/null +++ b/bot/resources/elements.json @@ -0,0 +1,120 @@ +[ + "hydrogen", + "helium", + "lithium", + "beryllium", + "boron", + "carbon", + "nitrogen", + "oxygen", + "fluorine", + "neon", + "sodium", + "magnesium", + "aluminium", + "silicon", + "phosphorus", + "sulfur", + "chlorine", + "argon", + "potassium", + "calcium", + "scandium", + "titanium", + "vanadium", + "chromium", + "manganese", + "iron", + "cobalt", + "nickel", + "copper", + "zinc", + "gallium", + "germanium", + "arsenic", + "selenium", + "bromine", + "krypton", + "rubidium", + "strontium", + "yttrium", + "zirconium", + "niobium", + "molybdenum", + "technetium", + "ruthenium", + "rhodium", + "palladium", + "silver", + "cadmium", + "indium", + "tin", + "antimony", + "tellurium", + "iodine", + "xenon", + "caesium", + "barium", + "lanthanum", + "cerium", + "praseodymium", + "neodymium", + "promethium", + "samarium", + "europium", + "gadolinium", + "terbium", + "dysprosium", + "holmium", + "erbium", + "thulium", + "ytterbium", + "lutetium", + "hafnium", + "tantalum", + "tungsten", + "rhenium", + "osmium", + "iridium", + "platinum", + "gold", + "mercury", + "thallium", + "lead", + "bismuth", + "polonium", + "astatine", + "radon", + "francium", + "radium", + "actinium", + "thorium", + "protactinium", + "uranium", + "neptunium", + "plutonium", + "americium", + "curium", + "berkelium", + "californium", + "einsteinium", + "fermium", + "mendelevium", + "nobelium", + "lawrencium", + "rutherfordium", + "dubnium", + "seaborgium", + "bohrium", + "hassium", + "meitnerium", + "darmstadtium", + "roentgenium", + "copernicium", + "nihonium", + "flerovium", + "moscovium", + "livermorium", + "tennessine", + "oganesson" +] diff --git a/bot/resources/tags/ask.md b/bot/resources/tags/ask.md deleted file mode 100644 index e2c2a88f6..000000000 --- a/bot/resources/tags/ask.md +++ /dev/null @@ -1,9 +0,0 @@ -Asking good questions will yield a much higher chance of a quick response: - -• Don't ask to ask your question, just go ahead and tell us your problem. -• Don't ask if anyone is knowledgeable in some area, filtering serves no purpose. -• Try to solve the problem on your own first, we're not going to write code for you. -• Show us the code you've tried and any errors or unexpected results it's giving. -• Be patient while we're helping you. - -You can find a much more detailed explanation [on our website](https://pythondiscord.com/pages/asking-good-questions/). diff --git a/bot/resources/tags/customcooldown.md b/bot/resources/tags/customcooldown.md new file mode 100644 index 000000000..ac7e70aee --- /dev/null +++ b/bot/resources/tags/customcooldown.md @@ -0,0 +1,20 @@ +**Cooldowns in discord.py** + +Cooldowns can be used in discord.py to rate-limit. In this example, we're using it in an on_message. + +```python +from discord.ext import commands + +message_cooldown = commands.CooldownMapping.from_cooldown(1.0, 60.0, commands.BucketType.user) + +async def on_message(message): + bucket = message_cooldown.get_bucket(message) + retry_after = bucket.update_rate_limit() + if retry_after: + await message.channel.send(f"Slow down! Try again in {retry_after} seconds.") + else: + await message.channel.send("Not ratelimited!") +``` + +`from_cooldown` takes the amount of `update_rate_limit()`s needed to trigger the cooldown, the time in which the cooldown is triggered, and a [`BucketType`](https://discordpy.readthedocs.io/en/latest/ext/commands/api.html#discord.discord.ext.commands.BucketType). diff --git a/bot/resources/tags/free.md b/bot/resources/tags/free.md new file mode 100644 index 000000000..1493076c7 --- /dev/null +++ b/bot/resources/tags/free.md @@ -0,0 +1,5 @@ +**We have a new help channel system!** + +Please see <#704250143020417084> for further information. + +A more detailed guide can be found on [our website](https://pythondiscord.com/pages/resources/guides/help-channels/). diff --git a/bot/resources/tags/kindling-projects.md b/bot/resources/tags/kindling-projects.md new file mode 100644 index 000000000..54ed8c961 --- /dev/null +++ b/bot/resources/tags/kindling-projects.md @@ -0,0 +1,3 @@ +**Kindling Projects** + +The [Kindling projects page](https://nedbatchelder.com/text/kindling.html) on Ned Batchelder's website contains a list of projects and ideas programmers can tackle to build their skills and knowledge. diff --git a/bot/resources/tags/modmail.md b/bot/resources/tags/modmail.md new file mode 100644 index 000000000..7545419ee --- /dev/null +++ b/bot/resources/tags/modmail.md @@ -0,0 +1,9 @@ +**Contacting the moderation team via ModMail** + +<@!683001325440860340> is a bot that will relay your messages to our moderation team, so that you can start a conversation with the moderation team. Your messages will be relayed to the entire moderator team, who will be able to respond to you via the bot. + +It supports attachments, codeblocks, and reactions. As communication happens over direct messages, the conversation will stay between you and the mod team. + +**To use it, simply send a direct message to the bot.** + +Should there be an urgent and immediate need for a moderator or admin to look at a channel, feel free to ping the <@&267629731250176001> or <@&267628507062992896> role instead. diff --git a/bot/resources/tags/mutability.md b/bot/resources/tags/mutability.md new file mode 100644 index 000000000..bde9b5e7e --- /dev/null +++ b/bot/resources/tags/mutability.md @@ -0,0 +1,37 @@ +**Mutable vs immutable objects** + +Imagine that you want to make all letters in a string upper case. Conveniently, strings have an `.upper()` method. + +You might think that this would work: +```python +>>> greeting = "hello" +>>> greeting.upper() +'HELLO' +>>> greeting +'hello' +``` + +`greeting` didn't change. Why is that so? + +That's because strings in Python are _immutable_. You can't change them, you can only pass around existing strings or create new ones. + +```python +>>> greeting = "hello" +>>> greeting = greeting.upper() +>>> greeting +'HELLO' +``` + +`greeting.upper()` creates and returns a new string which is like the old one, but with all the letters turned to upper case. + +`int`, `float`, `complex`, `tuple`, `frozenset` are other examples of immutable data types in Python. + +Mutable data types like `list`, on the other hand, can be changed in-place: +```python +>>> my_list = [1, 2, 3] +>>> my_list.append(4) +>>> my_list +[1, 2, 3, 4] +``` + +Other examples of mutable data types in Python are `dict` and `set`. Instances of user-defined classes are also mutable. diff --git a/bot/resources/tags/or-gotcha.md b/bot/resources/tags/or-gotcha.md index 00c2db1f8..d75a73d78 100644 --- a/bot/resources/tags/or-gotcha.md +++ b/bot/resources/tags/or-gotcha.md @@ -3,7 +3,7 @@ When checking if something is equal to one thing or another, you might think tha if favorite_fruit == 'grapefruit' or 'lemon': print("That's a weird favorite fruit to have.") ``` -After all, that's how you would normally phrase it in plain English. In Python, however, you have to have _complete instructions on both sides of the logical operator_. +While this makes sense in English, it may not behave the way you would expect. In Python, you should have _[complete instructions on both sides of the logical operator](https://docs.python.org/3/reference/expressions.html#boolean-operations)_. So, if you want to check if something is equal to one thing or another, there are two common ways: ```py diff --git a/bot/resources/tags/range-len.md b/bot/resources/tags/range-len.md new file mode 100644 index 000000000..65665eccf --- /dev/null +++ b/bot/resources/tags/range-len.md @@ -0,0 +1,11 @@ +Iterating over `range(len(...))` is a common approach to accessing each item in an ordered collection. +```py +for i in range(len(my_list)): + do_something(my_list[i]) +``` +The pythonic syntax is much simpler, and is guaranteed to produce elements in the same order: +```py +for item in my_list: + do_something(item) +``` +Python has other solutions for cases when the index itself might be needed. To get the element at the same index from two or more lists, use [zip](https://docs.python.org/3/library/functions.html#zip). To get both the index and the element at that index, use [enumerate](https://docs.python.org/3/library/functions.html#enumerate). diff --git a/bot/resources/tags/traceback.md b/bot/resources/tags/traceback.md index 46ef40aa1..e770fa86d 100644 --- a/bot/resources/tags/traceback.md +++ b/bot/resources/tags/traceback.md @@ -11,7 +11,7 @@ ZeroDivisionError: integer division or modulo by zero ``` The best way to read your traceback is bottom to top. -• Identify the exception raised (e.g. ZeroDivisonError) +• Identify the exception raised (e.g. ZeroDivisionError) • Make note of the line number, and navigate there in your program. • Try to understand why the error occurred. diff --git a/bot/rules/burst_shared.py b/bot/rules/burst_shared.py index bbe9271b3..0e66df69c 100644 --- a/bot/rules/burst_shared.py +++ b/bot/rules/burst_shared.py @@ -2,11 +2,20 @@ from typing import Dict, Iterable, List, Optional, Tuple from discord import Member, Message +from bot.constants import Channels + async def apply( last_message: Message, recent_messages: List[Message], config: Dict[str, int] ) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: - """Detects repeated messages sent by multiple users.""" + """ + Detects repeated messages sent by multiple users. + + This filter never triggers in the verification channel. + """ + if last_message.channel.id == Channels.verification: + return + total_recent = len(recent_messages) if total_recent > config['max']: diff --git a/bot/rules/discord_emojis.py b/bot/rules/discord_emojis.py index 5bab514f2..6e47f0197 100644 --- a/bot/rules/discord_emojis.py +++ b/bot/rules/discord_emojis.py @@ -5,6 +5,7 @@ from discord import Member, Message DISCORD_EMOJI_RE = re.compile(r"<:\w+:\d+>") +CODE_BLOCK_RE = re.compile(r"```.*?```", flags=re.DOTALL) async def apply( @@ -17,8 +18,9 @@ async def apply( if msg.author == last_message.author ) + # Get rid of code blocks in the message before searching for emojis. total_emojis = sum( - len(DISCORD_EMOJI_RE.findall(msg.content)) + len(DISCORD_EMOJI_RE.findall(CODE_BLOCK_RE.sub("", msg.content))) for msg in relevant_messages ) diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 9b32e515d..60170a88f 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -1,9 +1,4 @@ -from abc import ABCMeta +from bot.utils.helpers import CogABCMeta, find_nth_occurrence, pad_base64 +from bot.utils.services import send_to_paste_service -from discord.ext.commands import CogMeta - - -class CogABCMeta(CogMeta, ABCMeta): - """Metaclass for ABCs meant to be implemented as Cogs.""" - - pass +__all__ = ['CogABCMeta', 'find_nth_occurrence', 'pad_base64', 'send_to_paste_service'] diff --git a/bot/utils/checks.py b/bot/utils/checks.py index db56c347c..460a937d8 100644 --- a/bot/utils/checks.py +++ b/bot/utils/checks.py @@ -1,49 +1,122 @@ import datetime import logging -from typing import Callable, Iterable - -from discord.ext.commands import BucketType, Cog, Command, CommandOnCooldown, Context, Cooldown, CooldownMapping +from typing import Callable, Container, Iterable, Optional, Union + +from discord.ext.commands import ( + BucketType, + CheckFailure, + Cog, + Command, + CommandOnCooldown, + Context, + Cooldown, + CooldownMapping, + NoPrivateMessage, + has_any_role, +) + +from bot import constants log = logging.getLogger(__name__) -def with_role_check(ctx: Context, *role_ids: int) -> bool: - """Returns True if the user has any one of the roles in role_ids.""" - if not ctx.guild: # Return False in a DM - log.trace(f"{ctx.author} tried to use the '{ctx.command.name}'command from a DM. " - "This command is restricted by the with_role decorator. Rejecting request.") - return False +class InWhitelistCheckFailure(CheckFailure): + """Raised when the `in_whitelist` check fails.""" + + def __init__(self, redirect_channel: Optional[int]) -> None: + self.redirect_channel = redirect_channel + + if redirect_channel: + redirect_message = f" here. Please use the <#{redirect_channel}> channel instead" + else: + redirect_message = "" + + error_message = f"You are not allowed to use that command{redirect_message}." + + super().__init__(error_message) + + +def in_whitelist_check( + ctx: Context, + channels: Container[int] = (), + categories: Container[int] = (), + roles: Container[int] = (), + redirect: Optional[int] = constants.Channels.bot_commands, + fail_silently: bool = False, +) -> bool: + """ + Check if a command was issued in a whitelisted context. - for role in ctx.author.roles: - if role.id in role_ids: - log.trace(f"{ctx.author} has the '{role.name}' role, and passes the check.") - return True + The whitelists that can be provided are: - log.trace(f"{ctx.author} does not have the required role to use " - f"the '{ctx.command.name}' command, so the request is rejected.") + - `channels`: a container with channel ids for whitelisted channels + - `categories`: a container with category ids for whitelisted categories + - `roles`: a container with with role ids for whitelisted roles + + If the command was invoked in a context that was not whitelisted, the member is either + redirected to the `redirect` channel that was passed (default: #bot-commands) or simply + told that they're not allowed to use this particular command (if `None` was passed). + """ + if redirect and redirect not in channels: + # It does not make sense for the channel whitelist to not contain the redirection + # channel (if applicable). That's why we add the redirection channel to the `channels` + # container if it's not already in it. As we allow any container type to be passed, + # we first create a tuple in order to safely add the redirection channel. + # + # Note: It's possible for the redirect channel to be in a whitelisted category, but + # there's no easy way to check that and as a channel can easily be moved in and out of + # categories, it's probably not wise to rely on its category in any case. + channels = tuple(channels) + (redirect,) + + if channels and ctx.channel.id in channels: + log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted channel.") + return True + + # Only check the category id if we have a category whitelist and the channel has a `category_id` + if categories and hasattr(ctx.channel, "category_id") and ctx.channel.category_id in categories: + log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted category.") + return True + + # Only check the roles whitelist if we have one and ensure the author's roles attribute returns + # an iterable to prevent breakage in DM channels (for if we ever decide to enable commands there). + if roles and any(r.id in roles for r in getattr(ctx.author, "roles", ())): + log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they have a whitelisted role.") + return True + + log.trace(f"{ctx.author} may not use the `{ctx.command.name}` command within this context.") + + # Some commands are secret, and should produce no feedback at all. + if not fail_silently: + raise InWhitelistCheckFailure(redirect) return False -def without_role_check(ctx: Context, *role_ids: int) -> bool: - """Returns True if the user does not have any of the roles in role_ids.""" - if not ctx.guild: # Return False in a DM - log.trace(f"{ctx.author} tried to use the '{ctx.command.name}' command from a DM. " - "This command is restricted by the without_role decorator. Rejecting request.") +async def has_any_role_check(ctx: Context, *roles: Union[str, int]) -> bool: + """ + Returns True if the context's author has any of the specified roles. + + `roles` are the names or IDs of the roles for which to check. + False is always returns if the context is outside a guild. + """ + try: + return await has_any_role(*roles).predicate(ctx) + except CheckFailure: return False - author_roles = [role.id for role in ctx.author.roles] - check = all(role not in author_roles for role in role_ids) - log.trace(f"{ctx.author} tried to call the '{ctx.command.name}' command. " - f"The result of the without_role check was {check}.") - return check +async def has_no_roles_check(ctx: Context, *roles: Union[str, int]) -> bool: + """ + Returns True if the context's author doesn't have any of the specified roles. -def in_channel_check(ctx: Context, *channel_ids: int) -> bool: - """Checks if the command was executed inside the list of specified channels.""" - check = ctx.channel.id in channel_ids - log.trace(f"{ctx.author} tried to call the '{ctx.command.name}' command. " - f"The result of the in_channel check was {check}.") - return check + `roles` are the names or IDs of the roles for which to check. + False is always returns if the context is outside a guild. + """ + try: + return not await has_any_role(*roles).predicate(ctx) + except NoPrivateMessage: + return False + except CheckFailure: + return True def cooldown_with_role_bypass(rate: int, per: float, type: BucketType = BucketType.default, *, diff --git a/bot/utils/extensions.py b/bot/utils/extensions.py new file mode 100644 index 000000000..50350ea8d --- /dev/null +++ b/bot/utils/extensions.py @@ -0,0 +1,34 @@ +import importlib +import inspect +import pkgutil +from typing import Iterator, NoReturn + +from bot import exts + + +def unqualify(name: str) -> str: + """Return an unqualified name given a qualified module/package `name`.""" + return name.rsplit(".", maxsplit=1)[-1] + + +def walk_extensions() -> Iterator[str]: + """Yield extension names from the bot.exts subpackage.""" + + def on_error(name: str) -> NoReturn: + raise ImportError(name=name) # pragma: no cover + + for module in pkgutil.walk_packages(exts.__path__, f"{exts.__name__}.", onerror=on_error): + if unqualify(module.name).startswith("_"): + # Ignore module/package names starting with an underscore. + continue + + if module.ispkg: + imported = importlib.import_module(module.name) + if not inspect.isfunction(getattr(imported, "setup", None)): + # If it lacks a setup function, it's not an extension. + continue + + yield module.name + + +EXTENSIONS = frozenset(walk_extensions()) diff --git a/bot/utils/helpers.py b/bot/utils/helpers.py new file mode 100644 index 000000000..d9b60af07 --- /dev/null +++ b/bot/utils/helpers.py @@ -0,0 +1,23 @@ +from abc import ABCMeta +from typing import Optional + +from discord.ext.commands import CogMeta + + +class CogABCMeta(CogMeta, ABCMeta): + """Metaclass for ABCs meant to be implemented as Cogs.""" + + +def find_nth_occurrence(string: str, substring: str, n: int) -> Optional[int]: + """Return index of `n`th occurrence of `substring` in `string`, or None if not found.""" + index = 0 + for _ in range(n): + index = string.find(substring, index+1) + if index == -1: + return None + return index + + +def pad_base64(data: str) -> str: + """Return base64 `data` with padding characters to ensure its length is a multiple of 4.""" + return data + "=" * (-len(data) % 4) diff --git a/bot/utils/messages.py b/bot/utils/messages.py index e969ee590..d0b2342b3 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -1,45 +1,46 @@ import asyncio import contextlib import logging +import random +import re from io import BytesIO from typing import List, Optional, Sequence, Union -from discord import Client, Embed, File, Member, Message, Reaction, TextChannel, Webhook -from discord.abc import Snowflake +import discord from discord.errors import HTTPException +from discord.ext.commands import Context -from bot.constants import Emojis +from bot.constants import Emojis, NEGATIVE_REPLIES log = logging.getLogger(__name__) async def wait_for_deletion( - message: Message, - user_ids: Sequence[Snowflake], + message: discord.Message, + user_ids: Sequence[discord.abc.Snowflake], + client: discord.Client, deletion_emojis: Sequence[str] = (Emojis.trashcan,), timeout: float = 60 * 5, attach_emojis: bool = True, - client: Optional[Client] = None ) -> None: """ Wait for up to `timeout` seconds for a reaction by any of the specified `user_ids` to delete the message. An `attach_emojis` bool may be specified to determine whether to attach the given - `deletion_emojis` to the message in the given `context` - - A `client` instance may be optionally specified, otherwise client will be taken from the - guild of the message. + `deletion_emojis` to the message in the given `context`. """ - if message.guild is None and client is None: + if message.guild is None: raise ValueError("Message must be sent on a guild") - bot = client or message.guild.me - if attach_emojis: for emoji in deletion_emojis: - await message.add_reaction(emoji) + try: + await message.add_reaction(emoji) + except discord.NotFound: + log.trace(f"Aborting wait_for_deletion: message {message.id} deleted prematurely.") + return - def check(reaction: Reaction, user: Member) -> bool: + def check(reaction: discord.Reaction, user: discord.Member) -> bool: """Check that the deletion emoji is reacted by the appropriate user.""" return ( reaction.message.id == message.id @@ -48,13 +49,13 @@ async def wait_for_deletion( ) with contextlib.suppress(asyncio.TimeoutError): - await bot.wait_for('reaction_add', check=check, timeout=timeout) + await client.wait_for('reaction_add', check=check, timeout=timeout) await message.delete() async def send_attachments( - message: Message, - destination: Union[TextChannel, Webhook], + message: discord.Message, + destination: Union[discord.TextChannel, discord.Webhook], link_large: bool = True ) -> List[str]: """ @@ -78,15 +79,15 @@ async def send_attachments( if attachment.size <= destination.guild.filesize_limit - 512: with BytesIO() as file: await attachment.save(file, use_cached=True) - attachment_file = File(file, filename=attachment.filename) + attachment_file = discord.File(file, filename=attachment.filename) - if isinstance(destination, TextChannel): + if isinstance(destination, discord.TextChannel): msg = await destination.send(file=attachment_file) urls.append(msg.attachments[0].url) else: await destination.send( file=attachment_file, - username=message.author.display_name, + username=sub_clyde(message.author.display_name), avatar_url=message.author.avatar_url ) elif link_large: @@ -97,20 +98,52 @@ async def send_attachments( if link_large and e.status == 413: large.append(attachment) else: - log.warning(f"{failure_msg} with status {e.status}.") + log.warning(f"{failure_msg} with status {e.status}.", exc_info=e) if link_large and large: - desc = f"\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large) - embed = Embed(description=desc) + desc = "\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large) + embed = discord.Embed(description=desc) embed.set_footer(text="Attachments exceed upload size limit.") - if isinstance(destination, TextChannel): + if isinstance(destination, discord.TextChannel): await destination.send(embed=embed) else: await destination.send( embed=embed, - username=message.author.display_name, + username=sub_clyde(message.author.display_name), avatar_url=message.author.avatar_url ) return urls + + +def sub_clyde(username: Optional[str]) -> Optional[str]: + """ + Replace "e"/"E" in any "clyde" in `username` with a Cyrillic "е"/"E" and return the new string. + + Discord disallows "clyde" anywhere in the username for webhooks. It will return a 400. + Return None only if `username` is None. + """ + def replace_e(match: re.Match) -> str: + char = "е" if match[2] == "e" else "Е" + return match[1] + char + + if username: + return re.sub(r"(clyd)(e)", replace_e, username, flags=re.I) + else: + return username # Empty string or None + + +async def send_denial(ctx: Context, reason: str) -> None: + """Send an embed denying the user with the given reason.""" + embed = discord.Embed() + embed.colour = discord.Colour.red() + embed.title = random.choice(NEGATIVE_REPLIES) + embed.description = reason + + await ctx.send(embed=embed) + + +def format_user(user: discord.abc.User) -> str: + """Return a string for `user` which has their mention and ID.""" + return f"{user.mention} (`{user.id}`)" diff --git a/bot/utils/regex.py b/bot/utils/regex.py new file mode 100644 index 000000000..0d2068f90 --- /dev/null +++ b/bot/utils/regex.py @@ -0,0 +1,12 @@ +import re + +INVITE_RE = re.compile( + r"(?:discord(?:[\.,]|dot)gg|" # Could be discord.gg/ + r"discord(?:[\.,]|dot)com(?:\/|slash)invite|" # or discord.com/invite/ + r"discordapp(?:[\.,]|dot)com(?:\/|slash)invite|" # or discordapp.com/invite/ + r"discord(?:[\.,]|dot)me|" # or discord.me + r"discord(?:[\.,]|dot)io" # or discord.io. + r")(?:[\/]|slash)" # / or 'slash' + r"([a-zA-Z0-9\-]+)", # the invite code itself + flags=re.IGNORECASE +) diff --git a/bot/utils/scheduling.py b/bot/utils/scheduling.py index 5760ec2d4..03f31d78f 100644 --- a/bot/utils/scheduling.py +++ b/bot/utils/scheduling.py @@ -1,69 +1,126 @@ import asyncio import contextlib +import inspect import logging import typing as t -from abc import abstractmethod +from datetime import datetime from functools import partial -from bot.utils import CogABCMeta -log = logging.getLogger(__name__) +class Scheduler: + """ + Schedule the execution of coroutines and keep track of them. + When instantiating a Scheduler, a name must be provided. This name is used to distinguish the + instance's log messages from other instances. Using the name of the class or module containing + the instance is suggested. -class Scheduler(metaclass=CogABCMeta): - """Task scheduler.""" + Coroutines can be scheduled immediately with `schedule` or in the future with `schedule_at` + or `schedule_later`. A unique ID is required to be given in order to keep track of the + resulting Tasks. Any scheduled task can be cancelled prematurely using `cancel` by providing + the same ID used to schedule it. The `in` operator is supported for checking if a task with a + given ID is currently scheduled. - def __init__(self): - # Keep track of the child cog's name so the logs are clear. - self.cog_name = self.__class__.__name__ + Any exception raised in a scheduled task is logged when the task is done. + """ - self._scheduled_tasks: t.Dict[t.Hashable, asyncio.Task] = {} + def __init__(self, name: str): + self.name = name - @abstractmethod - async def _scheduled_task(self, task_object: t.Any) -> None: - """ - A coroutine which handles the scheduling. + self._log = logging.getLogger(f"{__name__}.{name}") + self._scheduled_tasks: t.Dict[t.Hashable, asyncio.Task] = {} - This is added to the scheduled tasks, and should wait the task duration, execute the desired - code, then clean up the task. + def __contains__(self, task_id: t.Hashable) -> bool: + """Return True if a task with the given `task_id` is currently scheduled.""" + return task_id in self._scheduled_tasks - For example, in Reminders this will wait for the reminder duration, send the reminder, - then make a site API request to delete the reminder from the database. + def schedule(self, task_id: t.Hashable, coroutine: t.Coroutine) -> None: """ + Schedule the execution of a `coroutine`. - def schedule_task(self, task_id: t.Hashable, task_data: t.Any) -> None: + If a task with `task_id` already exists, close `coroutine` instead of scheduling it. This + prevents unawaited coroutine warnings. Don't pass a coroutine that'll be re-used elsewhere. """ - Schedules a task. + self._log.trace(f"Scheduling task #{task_id}...") - `task_data` is passed to the `Scheduler._scheduled_task()` coroutine. - """ - log.trace(f"{self.cog_name}: scheduling task #{task_id}...") + msg = f"Cannot schedule an already started coroutine for #{task_id}" + assert inspect.getcoroutinestate(coroutine) == "CORO_CREATED", msg if task_id in self._scheduled_tasks: - log.debug( - f"{self.cog_name}: did not schedule task #{task_id}; task was already scheduled." - ) + self._log.debug(f"Did not schedule task #{task_id}; task was already scheduled.") + coroutine.close() return - task = asyncio.create_task(self._scheduled_task(task_data)) + task = asyncio.create_task(coroutine, name=f"{self.name}_{task_id}") task.add_done_callback(partial(self._task_done_callback, task_id)) self._scheduled_tasks[task_id] = task - log.debug(f"{self.cog_name}: scheduled task #{task_id} {id(task)}.") + self._log.debug(f"Scheduled task #{task_id} {id(task)}.") + + def schedule_at(self, time: datetime, task_id: t.Hashable, coroutine: t.Coroutine) -> None: + """ + Schedule `coroutine` to be executed at the given naïve UTC `time`. - def cancel_task(self, task_id: t.Hashable) -> None: - """Unschedule the task identified by `task_id`.""" - log.trace(f"{self.cog_name}: cancelling task #{task_id}...") - task = self._scheduled_tasks.get(task_id) + If `time` is in the past, schedule `coroutine` immediately. - if not task: - log.warning(f"{self.cog_name}: failed to unschedule {task_id} (no task found).") - return + If a task with `task_id` already exists, close `coroutine` instead of scheduling it. This + prevents unawaited coroutine warnings. Don't pass a coroutine that'll be re-used elsewhere. + """ + delay = (time - datetime.utcnow()).total_seconds() + if delay > 0: + coroutine = self._await_later(delay, task_id, coroutine) - task.cancel() - del self._scheduled_tasks[task_id] + self.schedule(task_id, coroutine) - log.debug(f"{self.cog_name}: unscheduled task #{task_id} {id(task)}.") + def schedule_later(self, delay: t.Union[int, float], task_id: t.Hashable, coroutine: t.Coroutine) -> None: + """ + Schedule `coroutine` to be executed after the given `delay` number of seconds. + + If a task with `task_id` already exists, close `coroutine` instead of scheduling it. This + prevents unawaited coroutine warnings. Don't pass a coroutine that'll be re-used elsewhere. + """ + self.schedule(task_id, self._await_later(delay, task_id, coroutine)) + + def cancel(self, task_id: t.Hashable) -> None: + """Unschedule the task identified by `task_id`. Log a warning if the task doesn't exist.""" + self._log.trace(f"Cancelling task #{task_id}...") + + try: + task = self._scheduled_tasks.pop(task_id) + except KeyError: + self._log.warning(f"Failed to unschedule {task_id} (no task found).") + else: + task.cancel() + + self._log.debug(f"Unscheduled task #{task_id} {id(task)}.") + + def cancel_all(self) -> None: + """Unschedule all known tasks.""" + self._log.debug("Unscheduling all tasks") + + for task_id in self._scheduled_tasks.copy(): + self.cancel(task_id) + + async def _await_later(self, delay: t.Union[int, float], task_id: t.Hashable, coroutine: t.Coroutine) -> None: + """Await `coroutine` after the given `delay` number of seconds.""" + try: + self._log.trace(f"Waiting {delay} seconds before awaiting coroutine for #{task_id}.") + await asyncio.sleep(delay) + + # Use asyncio.shield to prevent the coroutine from cancelling itself. + self._log.trace(f"Done waiting for #{task_id}; now awaiting the coroutine.") + await asyncio.shield(coroutine) + finally: + # Close it to prevent unawaited coroutine warnings, + # which would happen if the task was cancelled during the sleep. + # Only close it if it's not been awaited yet. This check is important because the + # coroutine may cancel this task, which would also trigger the finally block. + state = inspect.getcoroutinestate(coroutine) + if state == "CORO_CREATED": + self._log.debug(f"Explicitly closing the coroutine for #{task_id}.") + coroutine.close() + else: + self._log.debug(f"Finally block reached for #{task_id}; {state=}") def _task_done_callback(self, task_id: t.Hashable, done_task: asyncio.Task) -> None: """ @@ -72,24 +129,24 @@ class Scheduler(metaclass=CogABCMeta): If `done_task` and the task associated with `task_id` are different, then the latter will not be deleted. In this case, a new task was likely rescheduled with the same ID. """ - log.trace(f"{self.cog_name}: performing done callback for task #{task_id} {id(done_task)}.") + self._log.trace(f"Performing done callback for task #{task_id} {id(done_task)}.") scheduled_task = self._scheduled_tasks.get(task_id) if scheduled_task and done_task is scheduled_task: - # A task for the ID exists and its the same as the done task. + # A task for the ID exists and is the same as the done task. # Since this is the done callback, the task is already done so no need to cancel it. - log.trace(f"{self.cog_name}: deleting task #{task_id} {id(done_task)}.") + self._log.trace(f"Deleting task #{task_id} {id(done_task)}.") del self._scheduled_tasks[task_id] elif scheduled_task: # A new task was likely rescheduled with the same ID. - log.debug( - f"{self.cog_name}: the scheduled task #{task_id} {id(scheduled_task)} " + self._log.debug( + f"The scheduled task #{task_id} {id(scheduled_task)} " f"and the done task {id(done_task)} differ." ) elif not done_task.cancelled(): - log.warning( - f"{self.cog_name}: task #{task_id} not found while handling task {id(done_task)}! " + self._log.warning( + f"Task #{task_id} not found while handling task {id(done_task)}! " f"A task somehow got unscheduled improperly (i.e. deleted but not cancelled)." ) @@ -97,7 +154,4 @@ class Scheduler(metaclass=CogABCMeta): exception = done_task.exception() # Log the exception if one exists. if exception: - log.error( - f"{self.cog_name}: error in task #{task_id} {id(scheduled_task)}!", - exc_info=exception - ) + self._log.error(f"Error in task #{task_id} {id(done_task)}!", exc_info=exception) diff --git a/bot/utils/services.py b/bot/utils/services.py new file mode 100644 index 000000000..087b9f969 --- /dev/null +++ b/bot/utils/services.py @@ -0,0 +1,54 @@ +import logging +from typing import Optional + +from aiohttp import ClientConnectorError, ClientSession + +from bot.constants import URLs + +log = logging.getLogger(__name__) + +FAILED_REQUEST_ATTEMPTS = 3 + + +async def send_to_paste_service(http_session: ClientSession, contents: str, *, extension: str = "") -> Optional[str]: + """ + Upload `contents` to the paste service. + + `http_session` should be the current running ClientSession from aiohttp + `extension` is added to the output URL + + When an error occurs, `None` is returned, otherwise the generated URL with the suffix. + """ + extension = extension and f".{extension}" + log.debug(f"Sending contents of size {len(contents.encode())} bytes to paste service.") + paste_url = URLs.paste_service.format(key="documents") + for attempt in range(1, FAILED_REQUEST_ATTEMPTS + 1): + try: + async with http_session.post(paste_url, data=contents) as response: + response_json = await response.json() + except ClientConnectorError: + log.warning( + f"Failed to connect to paste service at url {paste_url}, " + f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." + ) + continue + except Exception: + log.exception( + f"An unexpected error has occurred during handling of the request, " + f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." + ) + continue + + if "message" in response_json: + log.warning( + f"Paste service returned error {response_json['message']} with status code {response.status}, " + f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." + ) + continue + elif "key" in response_json: + log.info(f"Successfully uploaded contents to paste service behind key {response_json['key']}.") + return URLs.paste_service.format(key=response_json['key']) + extension + log.warning( + f"Got unexpected JSON response from paste service: {response_json}\n" + f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." + ) diff --git a/bot/utils/time.py b/bot/utils/time.py index 77060143c..47e49904b 100644 --- a/bot/utils/time.py +++ b/bot/utils/time.py @@ -20,7 +20,9 @@ def _stringify_time_unit(value: int, unit: str) -> str: >>> _stringify_time_unit(0, "minutes") "less than a minute" """ - if value == 1: + if unit == "seconds" and value == 0: + return "0 seconds" + elif value == 1: return f"{value} {unit[:-1]}" elif value == 0: return f"less than a {unit[:-1]}" diff --git a/bot/utils/webhooks.py b/bot/utils/webhooks.py new file mode 100644 index 000000000..66f82ec66 --- /dev/null +++ b/bot/utils/webhooks.py @@ -0,0 +1,34 @@ +import logging +from typing import Optional + +import discord +from discord import Embed + +from bot.utils.messages import sub_clyde + +log = logging.getLogger(__name__) + + +async def send_webhook( + webhook: discord.Webhook, + content: Optional[str] = None, + username: Optional[str] = None, + avatar_url: Optional[str] = None, + embed: Optional[Embed] = None, + wait: Optional[bool] = False +) -> discord.Message: + """ + Send a message using the provided webhook. + + This uses sub_clyde() and tries for an HTTPException to ensure it doesn't crash. + """ + try: + return await webhook.send( + content=content, + username=sub_clyde(username), + avatar_url=avatar_url, + embed=embed, + wait=wait, + ) + except discord.HTTPException: + log.exception("Failed to send a message to the webhook!") diff --git a/config-default.yml b/config-default.yml index a9578d9bb..4f7b1e217 100644 --- a/config-default.yml +++ b/config-default.yml @@ -3,6 +3,16 @@ bot: token: !ENV "BOT_TOKEN" sentry_dsn: !ENV "BOT_SENTRY_DSN" + redis: + host: "redis" + port: 6379 + password: !ENV "REDIS_PASSWORD" + use_fakeredis: false + + stats: + statsd_host: "graphite" + presence_update_timeout: 300 + cooldowns: # Per channel, per tag. tags: 60 @@ -28,6 +38,21 @@ style: status_dnd: "<:status_dnd:470326272082313216>" status_offline: "<:status_offline:470326266537705472>" + badge_staff: "<:discord_staff:743882896498098226>" + badge_partner: "<:partner:748666453242413136>" + badge_hypesquad: "<:hypesquad_events:743882896892362873>" + badge_bug_hunter: "<:bug_hunter_lvl1:743882896372269137>" + badge_hypesquad_bravery: "<:hypesquad_bravery:743882896745693335>" + badge_hypesquad_brilliance: "<:hypesquad_brilliance:743882896938631248>" + badge_hypesquad_balance: "<:hypesquad_balance:743882896460480625>" + badge_early_supporter: "<:early_supporter:743882896909140058>" + badge_bug_hunter_level_2: "<:bug_hunter_lvl2:743882896611344505>" + badge_verified_bot_developer: "<:verified_bot_dev:743882897299210310>" + + incident_actioned: "<:incident_actioned:719645530128646266>" + incident_unactioned: "<:incident_unactioned:719645583245180960>" + incident_investigating: "<:incident_investigating:719645658671480924>" + failmail: "<:failmail:633660039931887616>" trashcan: "<:trashcan:637136429717389331>" @@ -37,23 +62,10 @@ style: cross_mark: "\u274C" check_mark: "\u2705" - ducky_yellow: &DUCKY_YELLOW 574951975574175744 - ducky_blurple: &DUCKY_BLURPLE 574951975310065675 - ducky_regal: &DUCKY_REGAL 637883439185395712 - ducky_camo: &DUCKY_CAMO 637914731566596096 - ducky_ninja: &DUCKY_NINJA 637923502535606293 - ducky_devil: &DUCKY_DEVIL 637925314982576139 - ducky_tube: &DUCKY_TUBE 637881368008851456 - ducky_hunt: &DUCKY_HUNT 639355090909528084 - ducky_wizard: &DUCKY_WIZARD 639355996954689536 - ducky_party: &DUCKY_PARTY 639468753440210977 - ducky_angel: &DUCKY_ANGEL 640121935610511361 - ducky_maul: &DUCKY_MAUL 640137724958867467 - ducky_santa: &DUCKY_SANTA 655360331002019870 - - upvotes: "<:upvotes:638729835245731840>" - comments: "<:comments:638729835073765387>" - user: "<:user:638729835442602003>" + # emotes used for #reddit + upvotes: "<:reddit_upvotes:755845219890757644>" + comments: "<:reddit_comments:755845255001014384>" + user: "<:reddit_users:755845303822974997>" icons: crown_blurple: "https://cdn.discordapp.com/emojis/469964153289965568.png" @@ -109,13 +121,23 @@ style: guild: id: 267624335836053506 + invite: "https://discord.gg/python" categories: - python_help: 356013061213126657 + help_available: 691405807388196926 + help_in_use: 696958401460043776 + help_dormant: 691405908919451718 + modmail: 714494672835444826 channels: - announcements: 354619224620138496 - user_event_announcements: &USER_EVENT_A 592000283102674944 + # Public announcement and news channels + change_log: &CHANGE_LOG 748238795236704388 + announcements: &ANNOUNCEMENTS 354619224620138496 + python_news: &PYNEWS_CHANNEL 704372456592506880 + python_events: &PYEVENTS_CHANNEL 729674110270963822 + mailing_lists: &MAILING_LISTS 704372456592506880 + reddit: &REDDIT_CHANNEL 458224812528238616 + user_event_announcements: &USER_EVENT_A 592000283102674944 # Development dev_contrib: &DEV_CONTRIB 635950537262759947 @@ -126,32 +148,26 @@ guild: meta: 429409067623251969 python_discussion: 267624335836053506 + # Python Help: Available + how_to_get_help: 704250143020417084 + cooldown: 720603994149486673 + # Logs attachment_log: &ATTACH_LOG 649243850006855680 message_log: &MESSAGE_LOG 467752170159079424 mod_log: &MOD_LOG 282638479504965634 user_log: 528976905546760203 voice_log: 640292421988646961 + dm_log: 653713721625018428 # Off-topic off_topic_0: 291284109232308226 off_topic_1: 463035241142026251 off_topic_2: 463035268514185226 - # Python Help - help_0: 303906576991780866 - help_1: 303906556754395136 - help_2: 303906514266226689 - help_3: 439702951246692352 - help_4: 451312046647148554 - help_5: 454941769734422538 - help_6: 587375753306570782 - help_7: 587375768556797982 - # Special bot_commands: &BOT_CMD 267659945086812160 esoteric: 470884583684964352 - reddit: 458224812528238616 verification: 352442727016693763 # Staff @@ -159,13 +175,22 @@ guild: admin_spam: &ADMIN_SPAM 563594791770914816 defcon: &DEFCON 464469101889454091 helpers: &HELPERS 385474242440986624 + incidents: 714214212200562749 + incidents_archive: 720668923636351037 mods: &MODS 305126844661760000 mod_alerts: &MOD_ALERTS 473092532147060736 mod_spam: &MOD_SPAM 620607373828030464 organisation: &ORGANISATION 551789653284356126 staff_lounge: &STAFF_LOUNGE 464905259261755392 + duck_pond: &DUCK_POND 637820308341915648 + + # Staff announcement channels + staff_announcements: &STAFF_ANNOUNCEMENTS 464033278631084042 + mod_announcements: &MOD_ANNOUNCEMENTS 372115205867700225 + admin_announcements: &ADMIN_ANNOUNCEMENTS 749736155569848370 # Voice + code_help_voice: 755154969761677312 admins_voice: &ADMINS_VOICE 500734494840717332 staff_voice: &STAFF_VOICE 412375055910043655 @@ -173,15 +198,6 @@ guild: big_brother_logs: &BB_LOGS 468507907357409333 talent_pool: &TALENT_POOL 534321732593647616 - staff_channels: - - *ADMINS - - *ADMIN_SPAM - - *DEFCON - - *HELPERS - - *MODS - - *MOD_SPAM - - *ORGANISATION - moderation_channels: - *ADMINS - *ADMIN_SPAM @@ -205,12 +221,13 @@ guild: roles: announcements: 463658397560995840 contributors: 295488872404484098 + help_cooldown: 699189276025421825 muted: &MUTED_ROLE 277914926603829249 partners: 323426753857191936 python_community: &PY_COMMUNITY_ROLE 458226413825294336 - # This is the Developers role on PyDis, here named verified for readability reasons - verified: 352427296948486144 + unverified: 739794855945044069 + verified: 352427296948486144 # @Developers on PyDis # Staff admins: &ADMINS_ROLE 267628507062992896 @@ -220,8 +237,8 @@ guild: owners: &OWNERS_ROLE 267627879762755584 # Code Jam - jammers: 591786436651646989 - team_leaders: 501324292341104650 + jammers: 737249140966162473 + team_leaders: 737250302834638889 moderation_roles: - *OWNERS_ROLE @@ -235,108 +252,34 @@ guild: - *HELPERS_ROLE webhooks: - talent_pool: 569145364800602132 - big_brother: 569133704568373283 - reddit: 635408384794951680 - duck_pond: 637821475327311927 - dev_log: 680501655111729222 - + big_brother: 569133704568373283 + dev_log: 680501655111729222 + dm_log: 654567640664244225 + duck_pond: 637821475327311927 + incidents_archive: 720671599790915702 + python_news: &PYNEWS_WEBHOOK 704381182279942324 + reddit: 635408384794951680 + talent_pool: 569145364800602132 filter: - # What do we filter? - filter_zalgo: false - filter_invites: true - filter_domains: true - watch_regex: true - watch_rich_embeds: true + filter_zalgo: false + filter_invites: true + filter_domains: true + filter_everyone_ping: true + watch_regex: true + watch_rich_embeds: true # Notify user on filter? # Notifications are not expected for "watchlist" type filters - notify_user_zalgo: false - notify_user_invites: true - notify_user_domains: false + notify_user_zalgo: false + notify_user_invites: true + notify_user_domains: false + notify_user_everyone_ping: true # Filter configuration - ping_everyone: true # Ping @everyone when we send a mod-alert? - - guild_invite_whitelist: - - 280033776820813825 # Functional Programming - - 267624335836053506 # Python Discord - - 440186186024222721 # Python Discord: ModLog Emojis - - 273944235143593984 # STEM - - 348658686962696195 # RLBot - - 531221516914917387 # Pallets - - 249111029668249601 # Gentoo - - 327254708534116352 # Adafruit - - 544525886180032552 # kennethreitz.org - - 590806733924859943 # Discord Hack Week - - 423249981340778496 # Kivy - - 197038439483310086 # Discord Testers - - 286633898581164032 # Ren'Py - - 349505959032389632 # PyGame - - 438622377094414346 # Pyglet - - 524691714909274162 # Panda3D - - 336642139381301249 # discord.py - - 405403391410438165 # Sentdex - - 172018499005317120 # The Coding Den - - domain_blacklist: - - pornhub.com - - liveleak.com - - grabify.link - - bmwforum.co - - leancoding.co - - spottyfly.com - - stopify.co - - yoütu.be - - discörd.com - - minecräft.com - - freegiftcards.co - - disçordapp.com - - fortnight.space - - fortnitechat.site - - joinmy.site - - curiouscat.club - - catsnthings.fun - - yourtube.site - - youtubeshort.watch - - catsnthing.com - - youtubeshort.pro - - canadianlumberjacks.online - - poweredbydialup.club - - poweredbydialup.online - - poweredbysecurity.org - - poweredbysecurity.online - - word_watchlist: - - goo+ks* - - ky+s+ - - ki+ke+s* - - beaner+s? - - coo+ns* - - nig+lets* - - slant-eyes* - - towe?l-?head+s* - - chi*n+k+s* - - spick*s* - - kill* +(?:yo)?urself+ - - jew+s* - - suicide - - rape - - (re+)tar+(d+|t+)(ed)? - - ta+r+d+ - - cunts* - - trann*y - - shemale - - token_watchlist: - - fa+g+s* - - 卐 - - 卍 - - cuck(?!oo+) - - nigg+(?:e*r+|a+h*?|u+h+)s? - - fag+o+t+s* + ping_everyone: true + offensive_msg_delete_days: 7 # How many days before deleting an offensive message? # Censor doesn't apply to these channel_whitelist: @@ -369,24 +312,7 @@ urls: site_staff: &STAFF !JOIN ["staff.", *DOMAIN] site_schema: &SCHEMA "https://" - site_bigbrother_api: !JOIN [*SCHEMA, *API, "/bot/bigbrother"] - site_docs_api: !JOIN [*SCHEMA, *API, "/bot/docs"] - site_superstarify_api: !JOIN [*SCHEMA, *API, "/bot/superstarify"] - site_infractions: !JOIN [*SCHEMA, *API, "/bot/infractions"] - site_infractions_user: !JOIN [*SCHEMA, *API, "/bot/infractions/user/{user_id}"] - site_infractions_type: !JOIN [*SCHEMA, *API, "/bot/infractions/type/{infraction_type}"] - site_infractions_by_id: !JOIN [*SCHEMA, *API, "/bot/infractions/id/{infraction_id}"] - site_infractions_user_type_current: !JOIN [*SCHEMA, *API, "/bot/infractions/user/{user_id}/{infraction_type}/current"] - site_infractions_user_type: !JOIN [*SCHEMA, *API, "/bot/infractions/user/{user_id}/{infraction_type}"] - site_logs_api: !JOIN [*SCHEMA, *API, "/bot/logs"] site_logs_view: !JOIN [*SCHEMA, *STAFF, "/bot/logs"] - site_off_topic_names_api: !JOIN [*SCHEMA, *API, "/bot/off-topic-names"] - site_reminders_api: !JOIN [*SCHEMA, *API, "/bot/reminders"] - site_reminders_user_api: !JOIN [*SCHEMA, *API, "/bot/reminders/user"] - site_settings_api: !JOIN [*SCHEMA, *API, "/bot/settings"] - site_tags_api: !JOIN [*SCHEMA, *API, "/bot/tags"] - site_user_api: !JOIN [*SCHEMA, *API, "/bot/users"] - site_user_complete_api: !JOIN [*SCHEMA, *API, "/bot/users/complete"] paste_service: !JOIN [*SCHEMA, *PASTE, "/{key}"] # Snekbox @@ -418,9 +344,13 @@ anti_spam: interval: 10 max: 7 - burst_shared: - interval: 10 - max: 20 + # Burst shared it (temporarily) disabled to prevent + # the bug that triggers multiple infractions/DMs per + # user. It also tends to catch a lot of innocent users + # now that we're so big. + # burst_shared: + # interval: 10 + # max: 20 chars: interval: 5 @@ -452,36 +382,6 @@ anti_spam: max: 3 -anti_malware: - whitelist: - - '.3gp' - - '.3g2' - - '.avi' - - '.bmp' - - '.gif' - - '.h264' - - '.jpg' - - '.jpeg' - - '.m4v' - - '.mkv' - - '.mov' - - '.mp4' - - '.mpeg' - - '.mpg' - - '.png' - - '.tiff' - - '.wmv' - - '.svg' - - '.psd' # Photoshop - - '.ai' # Illustrator - - '.aep' # After Effects - - '.xcf' # GIMP - - '.mp3' - - '.wav' - - '.ogg' - - '.md' - - reddit: subreddits: - 'r/Python' @@ -489,13 +389,6 @@ reddit: secret: !ENV "REDDIT_SECRET" -wolfram: - # Max requests per day. - user_limit_day: 10 - guild_limit_day: 67 - key: !ENV "WOLFRAM_API_KEY" - - big_brother: log_delay: 15 header_message_limit: 15 @@ -508,34 +401,86 @@ free: cooldown_rate: 1 cooldown_per: 60.0 -mention: - message_timeout: 300 - reset_delay: 5 + +help_channels: + enable: true + + # Minimum interval before allowing a certain user to claim a new help channel + claim_minutes: 15 + + # Roles which are allowed to use the command which makes channels dormant + cmd_whitelist: + - *HELPERS_ROLE + + # Allowed duration of inactivity before making a channel dormant + idle_minutes: 30 + + # Allowed duration of inactivity when channel is empty (due to deleted messages) + # before message making a channel dormant + deleted_idle_minutes: 5 + + # Maximum number of channels to put in the available category + max_available: 2 + + # Maximum number of channels across all 3 categories + # Note Discord has a hard limit of 50 channels per category, so this shouldn't be > 50 + max_total_channels: 32 + + # Prefix for help channel names + name_prefix: 'help-' + + # Notify if more available channels are needed but there are no more dormant ones + notify: true + + # Channel in which to send notifications + notify_channel: *HELPERS + + # Minimum interval between helper notifications + notify_minutes: 15 + + # Mention these roles in notifications + notify_roles: + - *HELPERS_ROLE redirect_output: delete_invocation: true delete_delay: 15 -sync: - confirm_timeout: 300 - max_diff: 10 - duck_pond: - threshold: 5 - custom_emojis: - - *DUCKY_YELLOW - - *DUCKY_BLURPLE - - *DUCKY_CAMO - - *DUCKY_DEVIL - - *DUCKY_NINJA - - *DUCKY_REGAL - - *DUCKY_TUBE - - *DUCKY_HUNT - - *DUCKY_WIZARD - - *DUCKY_PARTY - - *DUCKY_ANGEL - - *DUCKY_MAUL - - *DUCKY_SANTA + threshold: 4 + channel_blacklist: + - *ANNOUNCEMENTS + - *PYNEWS_CHANNEL + - *PYEVENTS_CHANNEL + - *MAILING_LISTS + - *REDDIT_CHANNEL + - *USER_EVENT_A + - *DUCK_POND + - *CHANGE_LOG + - *STAFF_ANNOUNCEMENTS + - *MOD_ANNOUNCEMENTS + - *ADMIN_ANNOUNCEMENTS + +python_news: + mail_lists: + - 'python-ideas' + - 'python-announce-list' + - 'pypi-announce' + channel: *PYNEWS_CHANNEL + webhook: *PYNEWS_WEBHOOK + + +verification: + unverified_after: 3 # Days after which non-Developers receive the @Unverified role + kicked_after: 30 # Days after which non-Developers get kicked from the guild + reminder_frequency: 28 # Hours between @Unverified pings + bot_message_delete_delay: 10 # Seconds before deleting bots response in #verification + + # Number in range [0, 1] determining the percentage of unverified users that are safe + # to be kicked from the guild in one batch, any larger amount will require staff confirmation, + # set this to 0 to require explicit approval for batches of any size + kick_confirmation_threshold: 0.01 # 1% + config: required_keys: ['bot.token'] diff --git a/docker-compose.yml b/docker-compose.yml index 11deceae8..cff7d33d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,19 @@ services: POSTGRES_PASSWORD: pysite POSTGRES_USER: pysite + redis: + image: redis:5.0.9 + ports: + - "127.0.0.1:6379:6379" + + snekbox: + image: pythondiscord/snekbox:latest + init: true + ipc: none + ports: + - "127.0.0.1:8060:8060" + privileged: true + web: image: pythondiscord/site:latest command: ["run", "--debug"] @@ -41,6 +54,8 @@ services: tty: true depends_on: - web + - redis + - snekbox environment: BOT_TOKEN: ${BOT_TOKEN} BOT_API_KEY: badbot13m0n8f570f942013fc818f234916ca531 diff --git a/tests/bot/cogs/sync/test_base.py b/tests/bot/cogs/sync/test_base.py deleted file mode 100644 index 6ee9dfda6..000000000 --- a/tests/bot/cogs/sync/test_base.py +++ /dev/null @@ -1,403 +0,0 @@ -import unittest -from unittest import mock - -import discord - -from bot import constants -from bot.api import ResponseCodeError -from bot.cogs.sync.syncers import Syncer, _Diff -from tests import helpers - - -class TestSyncer(Syncer): - """Syncer subclass with mocks for abstract methods for testing purposes.""" - - name = "test" - _get_diff = mock.AsyncMock() - _sync = mock.AsyncMock() - - -class SyncerBaseTests(unittest.TestCase): - """Tests for the syncer base class.""" - - def setUp(self): - self.bot = helpers.MockBot() - - def test_instantiation_fails_without_abstract_methods(self): - """The class must have abstract methods implemented.""" - with self.assertRaisesRegex(TypeError, "Can't instantiate abstract class"): - Syncer(self.bot) - - -class SyncerSendPromptTests(unittest.IsolatedAsyncioTestCase): - """Tests for sending the sync confirmation prompt.""" - - def setUp(self): - self.bot = helpers.MockBot() - self.syncer = TestSyncer(self.bot) - - def mock_get_channel(self): - """Fixture to return a mock channel and message for when `get_channel` is used.""" - self.bot.reset_mock() - - mock_channel = helpers.MockTextChannel() - mock_message = helpers.MockMessage() - - mock_channel.send.return_value = mock_message - self.bot.get_channel.return_value = mock_channel - - return mock_channel, mock_message - - def mock_fetch_channel(self): - """Fixture to return a mock channel and message for when `fetch_channel` is used.""" - self.bot.reset_mock() - - mock_channel = helpers.MockTextChannel() - mock_message = helpers.MockMessage() - - self.bot.get_channel.return_value = None - mock_channel.send.return_value = mock_message - self.bot.fetch_channel.return_value = mock_channel - - return mock_channel, mock_message - - async def test_send_prompt_edits_and_returns_message(self): - """The given message should be edited to display the prompt and then should be returned.""" - msg = helpers.MockMessage() - ret_val = await self.syncer._send_prompt(msg) - - msg.edit.assert_called_once() - self.assertIn("content", msg.edit.call_args[1]) - self.assertEqual(ret_val, msg) - - async def test_send_prompt_gets_dev_core_channel(self): - """The dev-core channel should be retrieved if an extant message isn't given.""" - subtests = ( - (self.bot.get_channel, self.mock_get_channel), - (self.bot.fetch_channel, self.mock_fetch_channel), - ) - - for method, mock_ in subtests: - with self.subTest(method=method, msg=mock_.__name__): - mock_() - await self.syncer._send_prompt() - - method.assert_called_once_with(constants.Channels.dev_core) - - async def test_send_prompt_returns_none_if_channel_fetch_fails(self): - """None should be returned if there's an HTTPException when fetching the channel.""" - self.bot.get_channel.return_value = None - self.bot.fetch_channel.side_effect = discord.HTTPException(mock.MagicMock(), "test error!") - - ret_val = await self.syncer._send_prompt() - - self.assertIsNone(ret_val) - - async def test_send_prompt_sends_and_returns_new_message_if_not_given(self): - """A new message mentioning core devs should be sent and returned if message isn't given.""" - for mock_ in (self.mock_get_channel, self.mock_fetch_channel): - with self.subTest(msg=mock_.__name__): - mock_channel, mock_message = mock_() - ret_val = await self.syncer._send_prompt() - - mock_channel.send.assert_called_once() - self.assertIn(self.syncer._CORE_DEV_MENTION, mock_channel.send.call_args[0][0]) - self.assertEqual(ret_val, mock_message) - - async def test_send_prompt_adds_reactions(self): - """The message should have reactions for confirmation added.""" - extant_message = helpers.MockMessage() - subtests = ( - (extant_message, lambda: (None, extant_message)), - (None, self.mock_get_channel), - (None, self.mock_fetch_channel), - ) - - for message_arg, mock_ in subtests: - subtest_msg = "Extant message" if mock_.__name__ == "<lambda>" else mock_.__name__ - - with self.subTest(msg=subtest_msg): - _, mock_message = mock_() - await self.syncer._send_prompt(message_arg) - - calls = [mock.call(emoji) for emoji in self.syncer._REACTION_EMOJIS] - mock_message.add_reaction.assert_has_calls(calls) - - -class SyncerConfirmationTests(unittest.IsolatedAsyncioTestCase): - """Tests for waiting for a sync confirmation reaction on the prompt.""" - - def setUp(self): - self.bot = helpers.MockBot() - self.syncer = TestSyncer(self.bot) - self.core_dev_role = helpers.MockRole(id=constants.Roles.core_developers) - - @staticmethod - def get_message_reaction(emoji): - """Fixture to return a mock message an reaction from the given `emoji`.""" - message = helpers.MockMessage() - reaction = helpers.MockReaction(emoji=emoji, message=message) - - return message, reaction - - def test_reaction_check_for_valid_emoji_and_authors(self): - """Should return True if authors are identical or are a bot and a core dev, respectively.""" - user_subtests = ( - ( - helpers.MockMember(id=77), - helpers.MockMember(id=77), - "identical users", - ), - ( - helpers.MockMember(id=77, bot=True), - helpers.MockMember(id=43, roles=[self.core_dev_role]), - "bot author and core-dev reactor", - ), - ) - - for emoji in self.syncer._REACTION_EMOJIS: - for author, user, msg in user_subtests: - with self.subTest(author=author, user=user, emoji=emoji, msg=msg): - message, reaction = self.get_message_reaction(emoji) - ret_val = self.syncer._reaction_check(author, message, reaction, user) - - self.assertTrue(ret_val) - - def test_reaction_check_for_invalid_reactions(self): - """Should return False for invalid reaction events.""" - valid_emoji = self.syncer._REACTION_EMOJIS[0] - subtests = ( - ( - helpers.MockMember(id=77), - *self.get_message_reaction(valid_emoji), - helpers.MockMember(id=43, roles=[self.core_dev_role]), - "users are not identical", - ), - ( - helpers.MockMember(id=77, bot=True), - *self.get_message_reaction(valid_emoji), - helpers.MockMember(id=43), - "reactor lacks the core-dev role", - ), - ( - helpers.MockMember(id=77, bot=True, roles=[self.core_dev_role]), - *self.get_message_reaction(valid_emoji), - helpers.MockMember(id=77, bot=True, roles=[self.core_dev_role]), - "reactor is a bot", - ), - ( - helpers.MockMember(id=77), - helpers.MockMessage(id=95), - helpers.MockReaction(emoji=valid_emoji, message=helpers.MockMessage(id=26)), - helpers.MockMember(id=77), - "messages are not identical", - ), - ( - helpers.MockMember(id=77), - *self.get_message_reaction("InVaLiD"), - helpers.MockMember(id=77), - "emoji is invalid", - ), - ) - - for *args, msg in subtests: - kwargs = dict(zip(("author", "message", "reaction", "user"), args)) - with self.subTest(**kwargs, msg=msg): - ret_val = self.syncer._reaction_check(*args) - self.assertFalse(ret_val) - - async def test_wait_for_confirmation(self): - """The message should always be edited and only return True if the emoji is a check mark.""" - subtests = ( - (constants.Emojis.check_mark, True, None), - ("InVaLiD", False, None), - (None, False, TimeoutError), - ) - - for emoji, ret_val, side_effect in subtests: - for bot in (True, False): - with self.subTest(emoji=emoji, ret_val=ret_val, side_effect=side_effect, bot=bot): - # Set up mocks - message = helpers.MockMessage() - member = helpers.MockMember(bot=bot) - - self.bot.wait_for.reset_mock() - self.bot.wait_for.return_value = (helpers.MockReaction(emoji=emoji), None) - self.bot.wait_for.side_effect = side_effect - - # Call the function - actual_return = await self.syncer._wait_for_confirmation(member, message) - - # Perform assertions - self.bot.wait_for.assert_called_once() - self.assertIn("reaction_add", self.bot.wait_for.call_args[0]) - - message.edit.assert_called_once() - kwargs = message.edit.call_args[1] - self.assertIn("content", kwargs) - - # Core devs should only be mentioned if the author is a bot. - if bot: - self.assertIn(self.syncer._CORE_DEV_MENTION, kwargs["content"]) - else: - self.assertNotIn(self.syncer._CORE_DEV_MENTION, kwargs["content"]) - - self.assertIs(actual_return, ret_val) - - -class SyncerSyncTests(unittest.IsolatedAsyncioTestCase): - """Tests for main function orchestrating the sync.""" - - def setUp(self): - self.bot = helpers.MockBot(user=helpers.MockMember(bot=True)) - self.syncer = TestSyncer(self.bot) - - async def test_sync_respects_confirmation_result(self): - """The sync should abort if confirmation fails and continue if confirmed.""" - mock_message = helpers.MockMessage() - subtests = ( - (True, mock_message), - (False, None), - ) - - for confirmed, message in subtests: - with self.subTest(confirmed=confirmed): - self.syncer._sync.reset_mock() - self.syncer._get_diff.reset_mock() - - diff = _Diff({1, 2, 3}, {4, 5}, None) - self.syncer._get_diff.return_value = diff - self.syncer._get_confirmation_result = mock.AsyncMock( - return_value=(confirmed, message) - ) - - guild = helpers.MockGuild() - await self.syncer.sync(guild) - - self.syncer._get_diff.assert_called_once_with(guild) - self.syncer._get_confirmation_result.assert_called_once() - - if confirmed: - self.syncer._sync.assert_called_once_with(diff) - else: - self.syncer._sync.assert_not_called() - - async def test_sync_diff_size(self): - """The diff size should be correctly calculated.""" - subtests = ( - (6, _Diff({1, 2}, {3, 4}, {5, 6})), - (5, _Diff({1, 2, 3}, None, {4, 5})), - (0, _Diff(None, None, None)), - (0, _Diff(set(), set(), set())), - ) - - for size, diff in subtests: - with self.subTest(size=size, diff=diff): - self.syncer._get_diff.reset_mock() - self.syncer._get_diff.return_value = diff - self.syncer._get_confirmation_result = mock.AsyncMock(return_value=(False, None)) - - guild = helpers.MockGuild() - await self.syncer.sync(guild) - - self.syncer._get_diff.assert_called_once_with(guild) - self.syncer._get_confirmation_result.assert_called_once() - self.assertEqual(self.syncer._get_confirmation_result.call_args[0][0], size) - - async def test_sync_message_edited(self): - """The message should be edited if one was sent, even if the sync has an API error.""" - subtests = ( - (None, None, False), - (helpers.MockMessage(), None, True), - (helpers.MockMessage(), ResponseCodeError(mock.MagicMock()), True), - ) - - for message, side_effect, should_edit in subtests: - with self.subTest(message=message, side_effect=side_effect, should_edit=should_edit): - self.syncer._sync.side_effect = side_effect - self.syncer._get_confirmation_result = mock.AsyncMock( - return_value=(True, message) - ) - - guild = helpers.MockGuild() - await self.syncer.sync(guild) - - if should_edit: - message.edit.assert_called_once() - self.assertIn("content", message.edit.call_args[1]) - - async def test_sync_confirmation_context_redirect(self): - """If ctx is given, a new message should be sent and author should be ctx's author.""" - mock_member = helpers.MockMember() - subtests = ( - (None, self.bot.user, None), - (helpers.MockContext(author=mock_member), mock_member, helpers.MockMessage()), - ) - - for ctx, author, message in subtests: - with self.subTest(ctx=ctx, author=author, message=message): - if ctx is not None: - ctx.send.return_value = message - - # Make sure `_get_diff` returns a MagicMock, not an AsyncMock - self.syncer._get_diff.return_value = mock.MagicMock() - - self.syncer._get_confirmation_result = mock.AsyncMock(return_value=(False, None)) - - guild = helpers.MockGuild() - await self.syncer.sync(guild, ctx) - - if ctx is not None: - ctx.send.assert_called_once() - - self.syncer._get_confirmation_result.assert_called_once() - self.assertEqual(self.syncer._get_confirmation_result.call_args[0][1], author) - self.assertEqual(self.syncer._get_confirmation_result.call_args[0][2], message) - - @mock.patch.object(constants.Sync, "max_diff", new=3) - async def test_confirmation_result_small_diff(self): - """Should always return True and the given message if the diff size is too small.""" - author = helpers.MockMember() - expected_message = helpers.MockMessage() - - for size in (3, 2): # pragma: no cover - with self.subTest(size=size): - self.syncer._send_prompt = mock.AsyncMock() - self.syncer._wait_for_confirmation = mock.AsyncMock() - - coro = self.syncer._get_confirmation_result(size, author, expected_message) - result, actual_message = await coro - - self.assertTrue(result) - self.assertEqual(actual_message, expected_message) - self.syncer._send_prompt.assert_not_called() - self.syncer._wait_for_confirmation.assert_not_called() - - @mock.patch.object(constants.Sync, "max_diff", new=3) - async def test_confirmation_result_large_diff(self): - """Should return True if confirmed and False if _send_prompt fails or aborted.""" - author = helpers.MockMember() - mock_message = helpers.MockMessage() - - subtests = ( - (True, mock_message, True, "confirmed"), - (False, None, False, "_send_prompt failed"), - (False, mock_message, False, "aborted"), - ) - - for expected_result, expected_message, confirmed, msg in subtests: # pragma: no cover - with self.subTest(msg=msg): - self.syncer._send_prompt = mock.AsyncMock(return_value=expected_message) - self.syncer._wait_for_confirmation = mock.AsyncMock(return_value=confirmed) - - coro = self.syncer._get_confirmation_result(4, author) - actual_result, actual_message = await coro - - self.syncer._send_prompt.assert_called_once_with(None) # message defaults to None - self.assertIs(actual_result, expected_result) - self.assertEqual(actual_message, expected_message) - - if expected_message: - self.syncer._wait_for_confirmation.assert_called_once_with( - author, expected_message - ) diff --git a/tests/bot/cogs/test_duck_pond.py b/tests/bot/cogs/test_duck_pond.py deleted file mode 100644 index 7e6bfc748..000000000 --- a/tests/bot/cogs/test_duck_pond.py +++ /dev/null @@ -1,575 +0,0 @@ -import asyncio -import logging -import typing -import unittest -from unittest.mock import AsyncMock, MagicMock, patch - -import discord - -from bot import constants -from bot.cogs import duck_pond -from tests import base -from tests import helpers - -MODULE_PATH = "bot.cogs.duck_pond" - - -class DuckPondTests(base.LoggingTestsMixin, unittest.IsolatedAsyncioTestCase): - """Tests for DuckPond functionality.""" - - @classmethod - def setUpClass(cls): - """Sets up the objects that only have to be initialized once.""" - cls.nonstaff_member = helpers.MockMember(name="Non-staffer") - - cls.staff_role = helpers.MockRole(name="Staff role", id=constants.STAFF_ROLES[0]) - cls.staff_member = helpers.MockMember(name="staffer", roles=[cls.staff_role]) - - cls.checkmark_emoji = "\N{White Heavy Check Mark}" - cls.thumbs_up_emoji = "\N{Thumbs Up Sign}" - cls.unicode_duck_emoji = "\N{Duck}" - cls.duck_pond_emoji = helpers.MockPartialEmoji(id=constants.DuckPond.custom_emojis[0]) - cls.non_duck_custom_emoji = helpers.MockPartialEmoji(id=123) - - def setUp(self): - """Sets up the objects that need to be refreshed before each test.""" - self.bot = helpers.MockBot(user=helpers.MockMember(id=46692)) - self.cog = duck_pond.DuckPond(bot=self.bot) - - def test_duck_pond_correctly_initializes(self): - """`__init__ should set `bot` and `webhook_id` attributes and schedule `fetch_webhook`.""" - bot = helpers.MockBot() - cog = MagicMock() - - duck_pond.DuckPond.__init__(cog, bot) - - self.assertEqual(cog.bot, bot) - self.assertEqual(cog.webhook_id, constants.Webhooks.duck_pond) - bot.loop.create_loop.called_once_with(cog.fetch_webhook()) - - def test_fetch_webhook_succeeds_without_connectivity_issues(self): - """The `fetch_webhook` method waits until `READY` event and sets the `webhook` attribute.""" - self.bot.fetch_webhook.return_value = "dummy webhook" - self.cog.webhook_id = 1 - - asyncio.run(self.cog.fetch_webhook()) - - self.bot.wait_until_guild_available.assert_called_once() - self.bot.fetch_webhook.assert_called_once_with(1) - self.assertEqual(self.cog.webhook, "dummy webhook") - - def test_fetch_webhook_logs_when_unable_to_fetch_webhook(self): - """The `fetch_webhook` method should log an exception when it fails to fetch the webhook.""" - self.bot.fetch_webhook.side_effect = discord.HTTPException(response=MagicMock(), message="Not found.") - self.cog.webhook_id = 1 - - log = logging.getLogger('bot.cogs.duck_pond') - with self.assertLogs(logger=log, level=logging.ERROR) as log_watcher: - asyncio.run(self.cog.fetch_webhook()) - - self.bot.wait_until_guild_available.assert_called_once() - self.bot.fetch_webhook.assert_called_once_with(1) - - self.assertEqual(len(log_watcher.records), 1) - - record = log_watcher.records[0] - self.assertEqual(record.levelno, logging.ERROR) - - def test_is_staff_returns_correct_values_based_on_instance_passed(self): - """The `is_staff` method should return correct values based on the instance passed.""" - test_cases = ( - (helpers.MockUser(name="User instance"), False), - (helpers.MockMember(name="Member instance without staff role"), False), - (helpers.MockMember(name="Member instance with staff role", roles=[self.staff_role]), True) - ) - - for user, expected_return in test_cases: - actual_return = self.cog.is_staff(user) - with self.subTest(user_type=user.name, expected_return=expected_return, actual_return=actual_return): - self.assertEqual(expected_return, actual_return) - - async def test_has_green_checkmark_correctly_detects_presence_of_green_checkmark_emoji(self): - """The `has_green_checkmark` method should only return `True` if one is present.""" - test_cases = ( - ( - "No reactions", helpers.MockMessage(), False - ), - ( - "No green check mark reactions", - helpers.MockMessage(reactions=[ - helpers.MockReaction(emoji=self.unicode_duck_emoji, users=[self.bot.user]), - helpers.MockReaction(emoji=self.thumbs_up_emoji, users=[self.bot.user]) - ]), - False - ), - ( - "Green check mark reaction, but not from our bot", - helpers.MockMessage(reactions=[ - helpers.MockReaction(emoji=self.unicode_duck_emoji, users=[self.bot.user]), - helpers.MockReaction(emoji=self.checkmark_emoji, users=[self.staff_member]) - ]), - False - ), - ( - "Green check mark reaction, with one from the bot", - helpers.MockMessage(reactions=[ - helpers.MockReaction(emoji=self.unicode_duck_emoji, users=[self.bot.user]), - helpers.MockReaction(emoji=self.checkmark_emoji, users=[self.staff_member, self.bot.user]) - ]), - True - ) - ) - - for description, message, expected_return in test_cases: - actual_return = await self.cog.has_green_checkmark(message) - with self.subTest( - test_case=description, - expected_return=expected_return, - actual_return=actual_return - ): - self.assertEqual(expected_return, actual_return) - - def test_send_webhook_correctly_passes_on_arguments(self): - """The `send_webhook` method should pass the arguments to the webhook correctly.""" - self.cog.webhook = helpers.MockAsyncWebhook() - - content = "fake content" - username = "fake username" - avatar_url = "fake avatar_url" - embed = "fake embed" - - asyncio.run(self.cog.send_webhook(content, username, avatar_url, embed)) - - self.cog.webhook.send.assert_called_once_with( - content=content, - username=username, - avatar_url=avatar_url, - embed=embed - ) - - def test_send_webhook_logs_when_sending_message_fails(self): - """The `send_webhook` method should catch a `discord.HTTPException` and log accordingly.""" - self.cog.webhook = helpers.MockAsyncWebhook() - self.cog.webhook.send.side_effect = discord.HTTPException(response=MagicMock(), message="Something failed.") - - log = logging.getLogger('bot.cogs.duck_pond') - with self.assertLogs(logger=log, level=logging.ERROR) as log_watcher: - asyncio.run(self.cog.send_webhook()) - - self.assertEqual(len(log_watcher.records), 1) - - record = log_watcher.records[0] - self.assertEqual(record.levelno, logging.ERROR) - - def _get_reaction( - self, - emoji: typing.Union[str, helpers.MockEmoji], - staff: int = 0, - nonstaff: int = 0 - ) -> helpers.MockReaction: - staffers = [helpers.MockMember(roles=[self.staff_role]) for _ in range(staff)] - nonstaffers = [helpers.MockMember() for _ in range(nonstaff)] - return helpers.MockReaction(emoji=emoji, users=staffers + nonstaffers) - - async def test_count_ducks_correctly_counts_the_number_of_eligible_duck_emojis(self): - """The `count_ducks` method should return the number of unique staffers who gave a duck.""" - test_cases = ( - # Simple test cases - # A message without reactions should return 0 - ( - "No reactions", - helpers.MockMessage(), - 0 - ), - # A message with a non-duck reaction from a non-staffer should return 0 - ( - "Non-duck reaction from non-staffer", - helpers.MockMessage(reactions=[self._get_reaction(emoji=self.thumbs_up_emoji, nonstaff=1)]), - 0 - ), - # A message with a non-duck reaction from a staffer should return 0 - ( - "Non-duck reaction from staffer", - helpers.MockMessage(reactions=[self._get_reaction(emoji=self.non_duck_custom_emoji, staff=1)]), - 0 - ), - # A message with a non-duck reaction from a non-staffer and staffer should return 0 - ( - "Non-duck reaction from staffer + non-staffer", - helpers.MockMessage(reactions=[self._get_reaction(emoji=self.thumbs_up_emoji, staff=1, nonstaff=1)]), - 0 - ), - # A message with a unicode duck reaction from a non-staffer should return 0 - ( - "Unicode Duck Reaction from non-staffer", - helpers.MockMessage(reactions=[self._get_reaction(emoji=self.unicode_duck_emoji, nonstaff=1)]), - 0 - ), - # A message with a unicode duck reaction from a staffer should return 1 - ( - "Unicode Duck Reaction from staffer", - helpers.MockMessage(reactions=[self._get_reaction(emoji=self.unicode_duck_emoji, staff=1)]), - 1 - ), - # A message with a unicode duck reaction from a non-staffer and staffer should return 1 - ( - "Unicode Duck Reaction from staffer + non-staffer", - helpers.MockMessage(reactions=[self._get_reaction(emoji=self.unicode_duck_emoji, staff=1, nonstaff=1)]), - 1 - ), - # A message with a duckpond duck reaction from a non-staffer should return 0 - ( - "Duckpond Duck Reaction from non-staffer", - helpers.MockMessage(reactions=[self._get_reaction(emoji=self.duck_pond_emoji, nonstaff=1)]), - 0 - ), - # A message with a duckpond duck reaction from a staffer should return 1 - ( - "Duckpond Duck Reaction from staffer", - helpers.MockMessage(reactions=[self._get_reaction(emoji=self.duck_pond_emoji, staff=1)]), - 1 - ), - # A message with a duckpond duck reaction from a non-staffer and staffer should return 1 - ( - "Duckpond Duck Reaction from staffer + non-staffer", - helpers.MockMessage(reactions=[self._get_reaction(emoji=self.duck_pond_emoji, staff=1, nonstaff=1)]), - 1 - ), - - # Complex test cases - # A message with duckpond duck reactions from 3 staffers and 2 non-staffers returns 3 - ( - "Duckpond Duck Reaction from 3 staffers + 2 non-staffers", - helpers.MockMessage(reactions=[self._get_reaction(emoji=self.duck_pond_emoji, staff=3, nonstaff=2)]), - 3 - ), - # A staffer with multiple duck reactions only counts once - ( - "Two different duck reactions from the same staffer", - helpers.MockMessage( - reactions=[ - helpers.MockReaction(emoji=self.duck_pond_emoji, users=[self.staff_member]), - helpers.MockReaction(emoji=self.unicode_duck_emoji, users=[self.staff_member]), - ] - ), - 1 - ), - # A non-string emoji does not count (to test the `isinstance(reaction.emoji, str)` elif) - ( - "Reaction with non-Emoji/str emoij from 3 staffers + 2 non-staffers", - helpers.MockMessage(reactions=[self._get_reaction(emoji=100, staff=3, nonstaff=2)]), - 0 - ), - # We correctly sum when multiple reactions are provided. - ( - "Duckpond Duck Reaction from 3 staffers + 2 non-staffers", - helpers.MockMessage( - reactions=[ - self._get_reaction(emoji=self.duck_pond_emoji, staff=3, nonstaff=2), - self._get_reaction(emoji=self.unicode_duck_emoji, staff=4, nonstaff=9), - ] - ), - 3 + 4 - ), - ) - - for description, message, expected_count in test_cases: - actual_count = await self.cog.count_ducks(message) - with self.subTest(test_case=description, expected_count=expected_count, actual_count=actual_count): - self.assertEqual(expected_count, actual_count) - - async def test_relay_message_correctly_relays_content_and_attachments(self): - """The `relay_message` method should correctly relay message content and attachments.""" - send_webhook_path = f"{MODULE_PATH}.DuckPond.send_webhook" - send_attachments_path = f"{MODULE_PATH}.send_attachments" - - self.cog.webhook = helpers.MockAsyncWebhook() - - test_values = ( - (helpers.MockMessage(clean_content="", attachments=[]), False, False), - (helpers.MockMessage(clean_content="message", attachments=[]), True, False), - (helpers.MockMessage(clean_content="", attachments=["attachment"]), False, True), - (helpers.MockMessage(clean_content="message", attachments=["attachment"]), True, True), - ) - - for message, expect_webhook_call, expect_attachment_call in test_values: - with patch(send_webhook_path, new_callable=AsyncMock) as send_webhook: - with patch(send_attachments_path, new_callable=AsyncMock) as send_attachments: - with self.subTest(clean_content=message.clean_content, attachments=message.attachments): - await self.cog.relay_message(message) - - self.assertEqual(expect_webhook_call, send_webhook.called) - self.assertEqual(expect_attachment_call, send_attachments.called) - - message.add_reaction.assert_called_once_with(self.checkmark_emoji) - - @patch(f"{MODULE_PATH}.send_attachments", new_callable=AsyncMock) - async def test_relay_message_handles_irretrievable_attachment_exceptions(self, send_attachments): - """The `relay_message` method should handle irretrievable attachments.""" - message = helpers.MockMessage(clean_content="message", attachments=["attachment"]) - side_effects = (discord.errors.Forbidden(MagicMock(), ""), discord.errors.NotFound(MagicMock(), "")) - - self.cog.webhook = helpers.MockAsyncWebhook() - log = logging.getLogger("bot.cogs.duck_pond") - - for side_effect in side_effects: # pragma: no cover - send_attachments.side_effect = side_effect - with patch(f"{MODULE_PATH}.DuckPond.send_webhook", new_callable=AsyncMock) as send_webhook: - with self.subTest(side_effect=type(side_effect).__name__): - with self.assertNotLogs(logger=log, level=logging.ERROR): - await self.cog.relay_message(message) - - self.assertEqual(send_webhook.call_count, 2) - - @patch(f"{MODULE_PATH}.DuckPond.send_webhook", new_callable=AsyncMock) - @patch(f"{MODULE_PATH}.send_attachments", new_callable=AsyncMock) - async def test_relay_message_handles_attachment_http_error(self, send_attachments, send_webhook): - """The `relay_message` method should handle irretrievable attachments.""" - message = helpers.MockMessage(clean_content="message", attachments=["attachment"]) - - self.cog.webhook = helpers.MockAsyncWebhook() - log = logging.getLogger("bot.cogs.duck_pond") - - side_effect = discord.HTTPException(MagicMock(), "") - send_attachments.side_effect = side_effect - with self.subTest(side_effect=type(side_effect).__name__): - with self.assertLogs(logger=log, level=logging.ERROR) as log_watcher: - await self.cog.relay_message(message) - - send_webhook.assert_called_once_with( - content=message.clean_content, - username=message.author.display_name, - avatar_url=message.author.avatar_url - ) - - self.assertEqual(len(log_watcher.records), 1) - - record = log_watcher.records[0] - self.assertEqual(record.levelno, logging.ERROR) - - def _mock_payload(self, label: str, is_custom_emoji: bool, id_: int, emoji_name: str): - """Creates a mock `on_raw_reaction_add` payload with the specified emoji data.""" - payload = MagicMock(name=label) - payload.emoji.is_custom_emoji.return_value = is_custom_emoji - payload.emoji.id = id_ - payload.emoji.name = emoji_name - return payload - - async def test_payload_has_duckpond_emoji_correctly_detects_relevant_emojis(self): - """The `on_raw_reaction_add` event handler should ignore irrelevant emojis.""" - test_values = ( - # Custom Emojis - ( - self._mock_payload( - label="Custom Duckpond Emoji", - is_custom_emoji=True, - id_=constants.DuckPond.custom_emojis[0], - emoji_name="" - ), - True - ), - ( - self._mock_payload( - label="Custom Non-Duckpond Emoji", - is_custom_emoji=True, - id_=123, - emoji_name="" - ), - False - ), - # Unicode Emojis - ( - self._mock_payload( - label="Unicode Duck Emoji", - is_custom_emoji=False, - id_=1, - emoji_name=self.unicode_duck_emoji - ), - True - ), - ( - self._mock_payload( - label="Unicode Non-Duck Emoji", - is_custom_emoji=False, - id_=1, - emoji_name=self.thumbs_up_emoji - ), - False - ), - ) - - for payload, expected_return in test_values: - actual_return = self.cog._payload_has_duckpond_emoji(payload) - with self.subTest(case=payload._mock_name, expected_return=expected_return, actual_return=actual_return): - self.assertEqual(expected_return, actual_return) - - @patch(f"{MODULE_PATH}.discord.utils.get") - @patch(f"{MODULE_PATH}.DuckPond._payload_has_duckpond_emoji", new=MagicMock(return_value=False)) - def test_on_raw_reaction_add_returns_early_with_payload_without_duck_emoji(self, utils_get): - """The `on_raw_reaction_add` method should return early if the payload does not contain a duck emoji.""" - self.assertIsNone(asyncio.run(self.cog.on_raw_reaction_add(payload=MagicMock()))) - - # Ensure we've returned before making an unnecessary API call in the lines of code after the emoji check - utils_get.assert_not_called() - - def _raw_reaction_mocks(self, channel_id, message_id, user_id): - """Sets up mocks for tests of the `on_raw_reaction_add` event listener.""" - channel = helpers.MockTextChannel(id=channel_id) - self.bot.get_all_channels.return_value = (channel,) - - message = helpers.MockMessage(id=message_id) - - channel.fetch_message.return_value = message - - member = helpers.MockMember(id=user_id, roles=[self.staff_role]) - message.guild.members = (member,) - - payload = MagicMock(channel_id=channel_id, message_id=message_id, user_id=user_id) - - return channel, message, member, payload - - async def test_on_raw_reaction_add_returns_for_bot_and_non_staff_members(self): - """The `on_raw_reaction_add` event handler should return for bot users or non-staff members.""" - channel_id = 1234 - message_id = 2345 - user_id = 3456 - - channel, message, _, payload = self._raw_reaction_mocks(channel_id, message_id, user_id) - - test_cases = ( - ("non-staff member", helpers.MockMember(id=user_id)), - ("bot staff member", helpers.MockMember(id=user_id, roles=[self.staff_role], bot=True)), - ) - - payload.emoji = self.duck_pond_emoji - - for description, member in test_cases: - message.guild.members = (member, ) - with self.subTest(test_case=description), patch(f"{MODULE_PATH}.DuckPond.has_green_checkmark") as checkmark: - checkmark.side_effect = AssertionError( - "Expected method to return before calling `self.has_green_checkmark`." - ) - self.assertIsNone(await self.cog.on_raw_reaction_add(payload)) - - # Check that we did make it past the payload checks - channel.fetch_message.assert_called_once() - channel.fetch_message.reset_mock() - - @patch(f"{MODULE_PATH}.DuckPond.is_staff") - @patch(f"{MODULE_PATH}.DuckPond.count_ducks", new_callable=AsyncMock) - def test_on_raw_reaction_add_returns_on_message_with_green_checkmark_placed_by_bot(self, count_ducks, is_staff): - """The `on_raw_reaction_add` event should return when the message has a green check mark placed by the bot.""" - channel_id = 31415926535 - message_id = 27182818284 - user_id = 16180339887 - - channel, message, member, payload = self._raw_reaction_mocks(channel_id, message_id, user_id) - - payload.emoji = helpers.MockPartialEmoji(name=self.unicode_duck_emoji) - payload.emoji.is_custom_emoji.return_value = False - - message.reactions = [helpers.MockReaction(emoji=self.checkmark_emoji, users=[self.bot.user])] - - is_staff.return_value = True - count_ducks.side_effect = AssertionError("Expected method to return before calling `self.count_ducks`") - - self.assertIsNone(asyncio.run(self.cog.on_raw_reaction_add(payload))) - - # Assert that we've made it past `self.is_staff` - is_staff.assert_called_once() - - async def test_on_raw_reaction_add_does_not_relay_below_duck_threshold(self): - """The `on_raw_reaction_add` listener should not relay messages or attachments below the duck threshold.""" - test_cases = ( - (constants.DuckPond.threshold - 1, False), - (constants.DuckPond.threshold, True), - (constants.DuckPond.threshold + 1, True), - ) - - channel, message, member, payload = self._raw_reaction_mocks(channel_id=3, message_id=4, user_id=5) - - payload.emoji = self.duck_pond_emoji - - for duck_count, should_relay in test_cases: - with patch(f"{MODULE_PATH}.DuckPond.relay_message", new_callable=AsyncMock) as relay_message: - with patch(f"{MODULE_PATH}.DuckPond.count_ducks", new_callable=AsyncMock) as count_ducks: - count_ducks.return_value = duck_count - with self.subTest(duck_count=duck_count, should_relay=should_relay): - await self.cog.on_raw_reaction_add(payload) - - # Confirm that we've made it past counting - count_ducks.assert_called_once() - - # Did we relay a message? - has_relayed = relay_message.called - self.assertEqual(has_relayed, should_relay) - - if should_relay: - relay_message.assert_called_once_with(message) - - async def test_on_raw_reaction_remove_prevents_removal_of_green_checkmark_depending_on_the_duck_count(self): - """The `on_raw_reaction_remove` listener prevents removal of the check mark on messages with enough ducks.""" - checkmark = helpers.MockPartialEmoji(name=self.checkmark_emoji) - - message = helpers.MockMessage(id=1234) - - channel = helpers.MockTextChannel(id=98765) - channel.fetch_message.return_value = message - - self.bot.get_all_channels.return_value = (channel, ) - - payload = MagicMock(channel_id=channel.id, message_id=message.id, emoji=checkmark) - - test_cases = ( - (constants.DuckPond.threshold - 1, False), - (constants.DuckPond.threshold, True), - (constants.DuckPond.threshold + 1, True), - ) - for duck_count, should_re_add_checkmark in test_cases: - with patch(f"{MODULE_PATH}.DuckPond.count_ducks", new_callable=AsyncMock) as count_ducks: - count_ducks.return_value = duck_count - with self.subTest(duck_count=duck_count, should_re_add_checkmark=should_re_add_checkmark): - await self.cog.on_raw_reaction_remove(payload) - - # Check if we fetched the message - channel.fetch_message.assert_called_once_with(message.id) - - # Check if we actually counted the number of ducks - count_ducks.assert_called_once_with(message) - - has_re_added_checkmark = message.add_reaction.called - self.assertEqual(should_re_add_checkmark, has_re_added_checkmark) - - if should_re_add_checkmark: - message.add_reaction.assert_called_once_with(self.checkmark_emoji) - message.add_reaction.reset_mock() - - # reset mocks - channel.fetch_message.reset_mock() - message.reset_mock() - - def test_on_raw_reaction_remove_ignores_removal_of_non_checkmark_reactions(self): - """The `on_raw_reaction_remove` listener should ignore the removal of non-check mark emojis.""" - channel = helpers.MockTextChannel(id=98765) - - channel.fetch_message.side_effect = AssertionError( - "Expected method to return before calling `channel.fetch_message`" - ) - - self.bot.get_all_channels.return_value = (channel, ) - - payload = MagicMock(emoji=helpers.MockPartialEmoji(name=self.thumbs_up_emoji), channel_id=channel.id) - - self.assertIsNone(asyncio.run(self.cog.on_raw_reaction_remove(payload))) - - channel.fetch_message.assert_not_called() - - -class DuckPondSetupTests(unittest.TestCase): - """Tests setup of the `DuckPond` cog.""" - - def test_setup(self): - """Setup of the extension should call add_cog.""" - bot = helpers.MockBot() - duck_pond.setup(bot) - bot.add_cog.assert_called_once() diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py deleted file mode 100644 index 33d1ec170..000000000 --- a/tests/bot/cogs/test_token_remover.py +++ /dev/null @@ -1,131 +0,0 @@ -import asyncio -import logging -import unittest -from unittest.mock import AsyncMock, MagicMock - -from discord import Colour - -from bot.cogs.token_remover import ( - DELETION_MESSAGE_TEMPLATE, - TokenRemover, - setup as setup_cog, -) -from bot.constants import Channels, Colours, Event, Icons -from tests.helpers import MockBot, MockMessage - - -class TokenRemoverTests(unittest.TestCase): - """Tests the `TokenRemover` cog.""" - - def setUp(self): - """Adds the cog, a bot, and a message to the instance for usage in tests.""" - self.bot = MockBot() - self.bot.get_cog.return_value = MagicMock() - self.bot.get_cog.return_value.send_log_message = AsyncMock() - self.cog = TokenRemover(bot=self.bot) - - self.msg = MockMessage(id=555, content='') - self.msg.author.__str__ = MagicMock() - self.msg.author.__str__.return_value = 'lemon' - self.msg.author.bot = False - self.msg.author.avatar_url_as.return_value = 'picture-lemon.png' - self.msg.author.id = 42 - self.msg.author.mention = '@lemon' - self.msg.channel.mention = "#lemonade-stand" - - def test_is_valid_user_id_is_true_for_numeric_content(self): - """A string decoding to numeric characters is a valid user ID.""" - # MTIz = base64(123) - self.assertTrue(TokenRemover.is_valid_user_id('MTIz')) - - def test_is_valid_user_id_is_false_for_alphabetic_content(self): - """A string decoding to alphabetic characters is not a valid user ID.""" - # YWJj = base64(abc) - self.assertFalse(TokenRemover.is_valid_user_id('YWJj')) - - def test_is_valid_timestamp_is_true_for_valid_timestamps(self): - """A string decoding to a valid timestamp should be recognized as such.""" - self.assertTrue(TokenRemover.is_valid_timestamp('DN9r_A')) - - def test_is_valid_timestamp_is_false_for_invalid_values(self): - """A string not decoding to a valid timestamp should not be recognized as such.""" - # MTIz = base64(123) - self.assertFalse(TokenRemover.is_valid_timestamp('MTIz')) - - def test_mod_log_property(self): - """The `mod_log` property should ask the bot to return the `ModLog` cog.""" - self.bot.get_cog.return_value = 'lemon' - self.assertEqual(self.cog.mod_log, self.bot.get_cog.return_value) - self.bot.get_cog.assert_called_once_with('ModLog') - - def test_ignores_bot_messages(self): - """When the message event handler is called with a bot message, nothing is done.""" - self.msg.author.bot = True - coroutine = self.cog.on_message(self.msg) - self.assertIsNone(asyncio.run(coroutine)) - - def test_ignores_messages_without_tokens(self): - """Messages without anything looking like a token are ignored.""" - for content in ('', 'lemon wins'): - with self.subTest(content=content): - self.msg.content = content - coroutine = self.cog.on_message(self.msg) - self.assertIsNone(asyncio.run(coroutine)) - - def test_ignores_messages_with_invalid_tokens(self): - """Messages with values that are invalid tokens are ignored.""" - for content in ('foo.bar.baz', 'x.y.'): - with self.subTest(content=content): - self.msg.content = content - coroutine = self.cog.on_message(self.msg) - self.assertIsNone(asyncio.run(coroutine)) - - def test_censors_valid_tokens(self): - """Valid tokens are censored.""" - cases = ( - # (content, censored_token) - ('MTIz.DN9R_A.xyz', 'MTIz.DN9R_A.xxx'), - ) - - for content, censored_token in cases: - with self.subTest(content=content, censored_token=censored_token): - self.msg.content = content - coroutine = self.cog.on_message(self.msg) - with self.assertLogs(logger='bot.cogs.token_remover', level=logging.DEBUG) as cm: - self.assertIsNone(asyncio.run(coroutine)) # no return value - - [line] = cm.output - log_message = ( - "Censored a seemingly valid token sent by " - "lemon (`42`) in #lemonade-stand, " - f"token was `{censored_token}`" - ) - self.assertIn(log_message, line) - - self.msg.delete.assert_called_once_with() - self.msg.channel.send.assert_called_once_with( - DELETION_MESSAGE_TEMPLATE.format(mention='@lemon') - ) - self.bot.get_cog.assert_called_with('ModLog') - self.msg.author.avatar_url_as.assert_called_once_with(static_format='png') - - mod_log = self.bot.get_cog.return_value - mod_log.ignore.assert_called_once_with(Event.message_delete, self.msg.id) - mod_log.send_log_message.assert_called_once_with( - icon_url=Icons.token_removed, - colour=Colour(Colours.soft_red), - title="Token removed!", - text=log_message, - thumbnail='picture-lemon.png', - channel_id=Channels.mod_alerts - ) - - -class TokenRemoverSetupTests(unittest.TestCase): - """Tests setup of the `TokenRemover` cog.""" - - def test_setup(self): - """Setup of the extension should call add_cog.""" - bot = MockBot() - setup_cog(bot) - bot.add_cog.assert_called_once() diff --git a/tests/bot/exts/__init__.py b/tests/bot/exts/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/bot/exts/__init__.py diff --git a/tests/bot/exts/backend/__init__.py b/tests/bot/exts/backend/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/bot/exts/backend/__init__.py diff --git a/tests/bot/exts/backend/sync/__init__.py b/tests/bot/exts/backend/sync/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/bot/exts/backend/sync/__init__.py diff --git a/tests/bot/exts/backend/sync/test_base.py b/tests/bot/exts/backend/sync/test_base.py new file mode 100644 index 000000000..4953550f9 --- /dev/null +++ b/tests/bot/exts/backend/sync/test_base.py @@ -0,0 +1,73 @@ +import unittest +from unittest import mock + + +from bot.api import ResponseCodeError +from bot.exts.backend.sync._syncers import Syncer +from tests import helpers + + +class TestSyncer(Syncer): + """Syncer subclass with mocks for abstract methods for testing purposes.""" + + name = "test" + _get_diff = mock.AsyncMock() + _sync = mock.AsyncMock() + + +class SyncerBaseTests(unittest.TestCase): + """Tests for the syncer base class.""" + + def setUp(self): + self.bot = helpers.MockBot() + + def test_instantiation_fails_without_abstract_methods(self): + """The class must have abstract methods implemented.""" + with self.assertRaisesRegex(TypeError, "Can't instantiate abstract class"): + Syncer(self.bot) + + +class SyncerSyncTests(unittest.IsolatedAsyncioTestCase): + """Tests for main function orchestrating the sync.""" + + def setUp(self): + self.bot = helpers.MockBot(user=helpers.MockMember(bot=True)) + self.syncer = TestSyncer(self.bot) + self.guild = helpers.MockGuild() + + # Make sure `_get_diff` returns a MagicMock, not an AsyncMock + self.syncer._get_diff.return_value = mock.MagicMock() + + async def test_sync_message_edited(self): + """The message should be edited if one was sent, even if the sync has an API error.""" + subtests = ( + (None, None, False), + (helpers.MockMessage(), None, True), + (helpers.MockMessage(), ResponseCodeError(mock.MagicMock()), True), + ) + + for message, side_effect, should_edit in subtests: + with self.subTest(message=message, side_effect=side_effect, should_edit=should_edit): + self.syncer._sync.side_effect = side_effect + ctx = helpers.MockContext() + ctx.send.return_value = message + + await self.syncer.sync(self.guild, ctx) + + if should_edit: + message.edit.assert_called_once() + self.assertIn("content", message.edit.call_args[1]) + + async def test_sync_message_sent(self): + """If ctx is given, a new message should be sent.""" + subtests = ( + (None, None), + (helpers.MockContext(), helpers.MockMessage()), + ) + + for ctx, message in subtests: + with self.subTest(ctx=ctx, message=message): + await self.syncer.sync(self.guild, ctx) + + if ctx is not None: + ctx.send.assert_called_once() diff --git a/tests/bot/cogs/sync/test_cog.py b/tests/bot/exts/backend/sync/test_cog.py index a4745f7b4..063a82754 100644 --- a/tests/bot/cogs/sync/test_cog.py +++ b/tests/bot/exts/backend/sync/test_cog.py @@ -5,8 +5,9 @@ import discord from bot import constants from bot.api import ResponseCodeError -from bot.cogs import sync -from bot.cogs.sync.syncers import Syncer +from bot.exts.backend import sync +from bot.exts.backend.sync._cog import Sync +from bot.exts.backend.sync._syncers import Syncer from tests import helpers from tests.base import CommandTestCase @@ -29,19 +30,19 @@ class SyncCogTestCase(unittest.IsolatedAsyncioTestCase): self.bot = helpers.MockBot() self.role_syncer_patcher = mock.patch( - "bot.cogs.sync.syncers.RoleSyncer", + "bot.exts.backend.sync._syncers.RoleSyncer", autospec=Syncer, spec_set=True ) self.user_syncer_patcher = mock.patch( - "bot.cogs.sync.syncers.UserSyncer", + "bot.exts.backend.sync._syncers.UserSyncer", autospec=Syncer, spec_set=True ) self.RoleSyncer = self.role_syncer_patcher.start() self.UserSyncer = self.user_syncer_patcher.start() - self.cog = sync.Sync(self.bot) + self.cog = Sync(self.bot) def tearDown(self): self.role_syncer_patcher.stop() @@ -59,7 +60,7 @@ class SyncCogTestCase(unittest.IsolatedAsyncioTestCase): class SyncCogTests(SyncCogTestCase): """Tests for the Sync cog.""" - @mock.patch.object(sync.Sync, "sync_guild", new_callable=mock.MagicMock) + @mock.patch.object(Sync, "sync_guild", new_callable=mock.MagicMock) def test_sync_cog_init(self, sync_guild): """Should instantiate syncers and run a sync for the guild.""" # Reset because a Sync cog was already instantiated in setUp. @@ -70,7 +71,7 @@ class SyncCogTests(SyncCogTestCase): mock_sync_guild_coro = mock.MagicMock() sync_guild.return_value = mock_sync_guild_coro - sync.Sync(self.bot) + Sync(self.bot) self.RoleSyncer.assert_called_once_with(self.bot) self.UserSyncer.assert_called_once_with(self.bot) @@ -131,6 +132,15 @@ class SyncCogListenerTests(SyncCogTestCase): super().setUp() self.cog.patch_user = mock.AsyncMock(spec_set=self.cog.patch_user) + self.guild_id_patcher = mock.patch("bot.exts.backend.sync._cog.constants.Guild.id", 5) + self.guild_id = self.guild_id_patcher.start() + + self.guild = helpers.MockGuild(id=self.guild_id) + self.other_guild = helpers.MockGuild(id=0) + + def tearDown(self): + self.guild_id_patcher.stop() + async def test_sync_cog_on_guild_role_create(self): """A POST request should be sent with the new role's data.""" self.assertTrue(self.cog.on_guild_role_create.__cog_listener__) @@ -142,20 +152,32 @@ class SyncCogListenerTests(SyncCogTestCase): "permissions": 8, "position": 23, } - role = helpers.MockRole(**role_data) + role = helpers.MockRole(**role_data, guild=self.guild) await self.cog.on_guild_role_create(role) self.bot.api_client.post.assert_called_once_with("bot/roles", json=role_data) + async def test_sync_cog_on_guild_role_create_ignores_guilds(self): + """Events from other guilds should be ignored.""" + role = helpers.MockRole(guild=self.other_guild) + await self.cog.on_guild_role_create(role) + self.bot.api_client.post.assert_not_awaited() + async def test_sync_cog_on_guild_role_delete(self): """A DELETE request should be sent.""" self.assertTrue(self.cog.on_guild_role_delete.__cog_listener__) - role = helpers.MockRole(id=99) + role = helpers.MockRole(id=99, guild=self.guild) await self.cog.on_guild_role_delete(role) self.bot.api_client.delete.assert_called_once_with("bot/roles/99") + async def test_sync_cog_on_guild_role_delete_ignores_guilds(self): + """Events from other guilds should be ignored.""" + role = helpers.MockRole(guild=self.other_guild) + await self.cog.on_guild_role_delete(role) + self.bot.api_client.delete.assert_not_awaited() + async def test_sync_cog_on_guild_role_update(self): """A PUT request should be sent if the colour, name, permissions, or position changes.""" self.assertTrue(self.cog.on_guild_role_update.__cog_listener__) @@ -180,8 +202,8 @@ class SyncCogListenerTests(SyncCogTestCase): after_role_data = role_data.copy() after_role_data[attribute] = 876 - before_role = helpers.MockRole(**role_data) - after_role = helpers.MockRole(**after_role_data) + before_role = helpers.MockRole(**role_data, guild=self.guild) + after_role = helpers.MockRole(**after_role_data, guild=self.guild) await self.cog.on_guild_role_update(before_role, after_role) @@ -193,31 +215,43 @@ class SyncCogListenerTests(SyncCogTestCase): else: self.bot.api_client.put.assert_not_called() + async def test_sync_cog_on_guild_role_update_ignores_guilds(self): + """Events from other guilds should be ignored.""" + role = helpers.MockRole(guild=self.other_guild) + await self.cog.on_guild_role_update(role, role) + self.bot.api_client.put.assert_not_awaited() + async def test_sync_cog_on_member_remove(self): - """Member should patched to set in_guild as False.""" + """Member should be patched to set in_guild as False.""" self.assertTrue(self.cog.on_member_remove.__cog_listener__) - member = helpers.MockMember() + member = helpers.MockMember(guild=self.guild) await self.cog.on_member_remove(member) self.cog.patch_user.assert_called_once_with( member.id, - updated_information={"in_guild": False} + json={"in_guild": False} ) + async def test_sync_cog_on_member_remove_ignores_guilds(self): + """Events from other guilds should be ignored.""" + member = helpers.MockMember(guild=self.other_guild) + await self.cog.on_member_remove(member) + self.cog.patch_user.assert_not_awaited() + async def test_sync_cog_on_member_update_roles(self): """Members should be patched if their roles have changed.""" self.assertTrue(self.cog.on_member_update.__cog_listener__) # Roles are intentionally unsorted. before_roles = [helpers.MockRole(id=12), helpers.MockRole(id=30), helpers.MockRole(id=20)] - before_member = helpers.MockMember(roles=before_roles) - after_member = helpers.MockMember(roles=before_roles[1:]) + before_member = helpers.MockMember(roles=before_roles, guild=self.guild) + after_member = helpers.MockMember(roles=before_roles[1:], guild=self.guild) await self.cog.on_member_update(before_member, after_member) data = {"roles": sorted(role.id for role in after_member.roles)} - self.cog.patch_user.assert_called_once_with(after_member.id, updated_information=data) + self.cog.patch_user.assert_called_once_with(after_member.id, json=data) async def test_sync_cog_on_member_update_other(self): """Members should not be patched if other attributes have changed.""" @@ -233,13 +267,19 @@ class SyncCogListenerTests(SyncCogTestCase): with self.subTest(attribute=attribute): self.cog.patch_user.reset_mock() - before_member = helpers.MockMember(**{attribute: old_value}) - after_member = helpers.MockMember(**{attribute: new_value}) + before_member = helpers.MockMember(**{attribute: old_value}, guild=self.guild) + after_member = helpers.MockMember(**{attribute: new_value}, guild=self.guild) await self.cog.on_member_update(before_member, after_member) self.cog.patch_user.assert_not_called() + async def test_sync_cog_on_member_update_ignores_guilds(self): + """Events from other guilds should be ignored.""" + member = helpers.MockMember(guild=self.other_guild) + await self.cog.on_member_update(member, member) + self.cog.patch_user.assert_not_awaited() + async def test_sync_cog_on_user_update(self): """A user should be patched only if the name, discriminator, or avatar changes.""" self.assertTrue(self.cog.on_user_update.__cog_listener__) @@ -247,14 +287,12 @@ class SyncCogListenerTests(SyncCogTestCase): before_data = { "name": "old name", "discriminator": "1234", - "avatar": "old avatar", "bot": False, } subtests = ( (True, "name", "name", "new name", "new name"), (True, "discriminator", "discriminator", "8765", 8765), - (True, "avatar", "avatar_hash", "9j2e9", "9j2e9"), (False, "bot", "bot", True, True), ) @@ -274,12 +312,15 @@ class SyncCogListenerTests(SyncCogTestCase): # Don't care if *all* keys are present; only the changed one is required call_args = self.cog.patch_user.call_args - self.assertEqual(call_args[0][0], after_user.id) - self.assertIn("updated_information", call_args[1]) + self.assertEqual(call_args.args[0], after_user.id) + self.assertIn("json", call_args.kwargs) + + self.assertIn("ignore_404", call_args.kwargs) + self.assertTrue(call_args.kwargs["ignore_404"]) - updated_information = call_args[1]["updated_information"] - self.assertIn(api_field, updated_information) - self.assertEqual(updated_information[api_field], api_value) + json = call_args.kwargs["json"] + self.assertIn(api_field, json) + self.assertEqual(json[api_field], api_value) else: self.cog.patch_user.assert_not_called() @@ -292,10 +333,10 @@ class SyncCogListenerTests(SyncCogTestCase): member = helpers.MockMember( discriminator="1234", roles=[helpers.MockRole(id=22), helpers.MockRole(id=12)], + guild=self.guild, ) data = { - "avatar_hash": member.avatar, "discriminator": int(member.discriminator), "id": member.id, "in_guild": True, @@ -337,6 +378,13 @@ class SyncCogListenerTests(SyncCogTestCase): self.bot.api_client.post.assert_not_called() + async def test_sync_cog_on_member_join_ignores_guilds(self): + """Events from other guilds should be ignored.""" + member = helpers.MockMember(guild=self.other_guild) + await self.cog.on_member_join(member) + self.bot.api_client.post.assert_not_awaited() + self.bot.api_client.put.assert_not_awaited() + class SyncCogCommandTests(SyncCogTestCase, CommandTestCase): """Tests for the commands in the Sync cog.""" diff --git a/tests/bot/cogs/sync/test_roles.py b/tests/bot/exts/backend/sync/test_roles.py index 79eee98f4..7b9f40cad 100644 --- a/tests/bot/cogs/sync/test_roles.py +++ b/tests/bot/exts/backend/sync/test_roles.py @@ -3,7 +3,7 @@ from unittest import mock import discord -from bot.cogs.sync.syncers import RoleSyncer, _Diff, _Role +from bot.exts.backend.sync._syncers import RoleSyncer, _Diff, _Role from tests import helpers diff --git a/tests/bot/cogs/sync/test_users.py b/tests/bot/exts/backend/sync/test_users.py index 818883012..c0a1da35c 100644 --- a/tests/bot/cogs/sync/test_users.py +++ b/tests/bot/exts/backend/sync/test_users.py @@ -1,7 +1,7 @@ import unittest from unittest import mock -from bot.cogs.sync.syncers import UserSyncer, _Diff, _User +from bot.exts.backend.sync._syncers import UserSyncer, _Diff, _User from tests import helpers @@ -10,7 +10,6 @@ def fake_user(**kwargs): kwargs.setdefault("id", 43) kwargs.setdefault("name", "bob the test man") kwargs.setdefault("discriminator", 1337) - kwargs.setdefault("avatar_hash", None) kwargs.setdefault("roles", (666,)) kwargs.setdefault("in_guild", True) @@ -32,7 +31,6 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): for member in members: member = member.copy() - member["avatar"] = member.pop("avatar_hash") del member["in_guild"] mock_member = helpers.MockMember(**member) diff --git a/tests/bot/exts/backend/test_logging.py b/tests/bot/exts/backend/test_logging.py new file mode 100644 index 000000000..466f207d9 --- /dev/null +++ b/tests/bot/exts/backend/test_logging.py @@ -0,0 +1,32 @@ +import unittest +from unittest.mock import patch + +from bot import constants +from bot.exts.backend.logging import Logging +from tests.helpers import MockBot, MockTextChannel + + +class LoggingTests(unittest.IsolatedAsyncioTestCase): + """Test cases for connected login.""" + + def setUp(self): + self.bot = MockBot() + self.cog = Logging(self.bot) + self.dev_log = MockTextChannel(id=1234, name="dev-log") + + @patch("bot.exts.backend.logging.DEBUG_MODE", False) + async def test_debug_mode_false(self): + """Should send connected message to dev-log.""" + self.bot.get_channel.return_value = self.dev_log + + await self.cog.startup_greeting() + self.bot.wait_until_guild_available.assert_awaited_once_with() + self.bot.get_channel.assert_called_once_with(constants.Channels.dev_log) + self.dev_log.send.assert_awaited_once() + + @patch("bot.exts.backend.logging.DEBUG_MODE", True) + async def test_debug_mode_true(self): + """Should not send anything to dev-log.""" + await self.cog.startup_greeting() + self.bot.wait_until_guild_available.assert_awaited_once_with() + self.bot.get_channel.assert_not_called() diff --git a/tests/bot/exts/filters/__init__.py b/tests/bot/exts/filters/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/bot/exts/filters/__init__.py diff --git a/tests/bot/exts/filters/test_antimalware.py b/tests/bot/exts/filters/test_antimalware.py new file mode 100644 index 000000000..3393c6cdc --- /dev/null +++ b/tests/bot/exts/filters/test_antimalware.py @@ -0,0 +1,187 @@ +import unittest +from unittest.mock import AsyncMock, Mock + +from discord import NotFound + +from bot.constants import Channels, STAFF_ROLES +from bot.exts.filters import antimalware +from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole + + +class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): + """Test the AntiMalware cog.""" + + def setUp(self): + """Sets up fresh objects for each test.""" + self.bot = MockBot() + self.bot.filter_list_cache = { + "FILE_FORMAT.True": { + ".first": {}, + ".second": {}, + ".third": {}, + } + } + self.cog = antimalware.AntiMalware(self.bot) + self.message = MockMessage() + self.message.webhook_id = None + self.message.author.bot = None + self.whitelist = [".first", ".second", ".third"] + + async def test_message_with_allowed_attachment(self): + """Messages with allowed extensions should not be deleted""" + attachment = MockAttachment(filename="python.first") + self.message.attachments = [attachment] + + await self.cog.on_message(self.message) + self.message.delete.assert_not_called() + + async def test_message_without_attachment(self): + """Messages without attachments should result in no action.""" + await self.cog.on_message(self.message) + self.message.delete.assert_not_called() + + async def test_direct_message_with_attachment(self): + """Direct messages should have no action taken.""" + attachment = MockAttachment(filename="python.disallowed") + self.message.attachments = [attachment] + self.message.guild = None + + await self.cog.on_message(self.message) + + self.message.delete.assert_not_called() + + async def test_webhook_message_with_illegal_extension(self): + """A webhook message containing an illegal extension should be ignored.""" + attachment = MockAttachment(filename="python.disallowed") + self.message.webhook_id = 697140105563078727 + self.message.attachments = [attachment] + + await self.cog.on_message(self.message) + + self.message.delete.assert_not_called() + + async def test_bot_message_with_illegal_extension(self): + """A bot message containing an illegal extension should be ignored.""" + attachment = MockAttachment(filename="python.disallowed") + self.message.author.bot = 409107086526644234 + self.message.attachments = [attachment] + + await self.cog.on_message(self.message) + + self.message.delete.assert_not_called() + + async def test_message_with_illegal_extension_gets_deleted(self): + """A message containing an illegal extension should send an embed.""" + attachment = MockAttachment(filename="python.disallowed") + self.message.attachments = [attachment] + + await self.cog.on_message(self.message) + + self.message.delete.assert_called_once() + + async def test_message_send_by_staff(self): + """A message send by a member of staff should be ignored.""" + staff_role = MockRole(id=STAFF_ROLES[0]) + self.message.author.roles.append(staff_role) + attachment = MockAttachment(filename="python.disallowed") + self.message.attachments = [attachment] + + await self.cog.on_message(self.message) + + self.message.delete.assert_not_called() + + async def test_python_file_redirect_embed_description(self): + """A message containing a .py file should result in an embed redirecting the user to our paste site""" + attachment = MockAttachment(filename="python.py") + self.message.attachments = [attachment] + self.message.channel.send = AsyncMock() + + await self.cog.on_message(self.message) + self.message.channel.send.assert_called_once() + args, kwargs = self.message.channel.send.call_args + embed = kwargs.pop("embed") + + self.assertEqual(embed.description, antimalware.PY_EMBED_DESCRIPTION) + + async def test_txt_file_redirect_embed_description(self): + """A message containing a .txt file should result in the correct embed.""" + attachment = MockAttachment(filename="python.txt") + self.message.attachments = [attachment] + self.message.channel.send = AsyncMock() + antimalware.TXT_EMBED_DESCRIPTION = Mock() + antimalware.TXT_EMBED_DESCRIPTION.format.return_value = "test" + + await self.cog.on_message(self.message) + self.message.channel.send.assert_called_once() + args, kwargs = self.message.channel.send.call_args + embed = kwargs.pop("embed") + cmd_channel = self.bot.get_channel(Channels.bot_commands) + + self.assertEqual(embed.description, antimalware.TXT_EMBED_DESCRIPTION.format.return_value) + antimalware.TXT_EMBED_DESCRIPTION.format.assert_called_with(cmd_channel_mention=cmd_channel.mention) + + async def test_other_disallowed_extension_embed_description(self): + """Test the description for a non .py/.txt disallowed extension.""" + attachment = MockAttachment(filename="python.disallowed") + self.message.attachments = [attachment] + self.message.channel.send = AsyncMock() + antimalware.DISALLOWED_EMBED_DESCRIPTION = Mock() + antimalware.DISALLOWED_EMBED_DESCRIPTION.format.return_value = "test" + + await self.cog.on_message(self.message) + self.message.channel.send.assert_called_once() + args, kwargs = self.message.channel.send.call_args + embed = kwargs.pop("embed") + meta_channel = self.bot.get_channel(Channels.meta) + + self.assertEqual(embed.description, antimalware.DISALLOWED_EMBED_DESCRIPTION.format.return_value) + antimalware.DISALLOWED_EMBED_DESCRIPTION.format.assert_called_with( + joined_whitelist=", ".join(self.whitelist), + blocked_extensions_str=".disallowed", + meta_channel_mention=meta_channel.mention + ) + + async def test_removing_deleted_message_logs(self): + """Removing an already deleted message logs the correct message""" + attachment = MockAttachment(filename="python.disallowed") + self.message.attachments = [attachment] + self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) + + with self.assertLogs(logger=antimalware.log, level="INFO"): + await self.cog.on_message(self.message) + self.message.delete.assert_called_once() + + async def test_message_with_illegal_attachment_logs(self): + """Deleting a message with an illegal attachment should result in a log.""" + attachment = MockAttachment(filename="python.disallowed") + self.message.attachments = [attachment] + + with self.assertLogs(logger=antimalware.log, level="INFO"): + await self.cog.on_message(self.message) + + async def test_get_disallowed_extensions(self): + """The return value should include all non-whitelisted extensions.""" + test_values = ( + ([], []), + (self.whitelist, []), + ([".first"], []), + ([".first", ".disallowed"], [".disallowed"]), + ([".disallowed"], [".disallowed"]), + ([".disallowed", ".illegal"], [".disallowed", ".illegal"]), + ) + + for extensions, expected_disallowed_extensions in test_values: + with self.subTest(extensions=extensions, expected_disallowed_extensions=expected_disallowed_extensions): + self.message.attachments = [MockAttachment(filename=f"filename{extension}") for extension in extensions] + disallowed_extensions = self.cog._get_disallowed_extensions(self.message) + self.assertCountEqual(disallowed_extensions, expected_disallowed_extensions) + + +class AntiMalwareSetupTests(unittest.TestCase): + """Tests setup of the `AntiMalware` cog.""" + + def test_setup(self): + """Setup of the extension should call add_cog.""" + bot = MockBot() + antimalware.setup(bot) + bot.add_cog.assert_called_once() diff --git a/tests/bot/cogs/test_antispam.py b/tests/bot/exts/filters/test_antispam.py index ce5472c71..6a0e4fded 100644 --- a/tests/bot/cogs/test_antispam.py +++ b/tests/bot/exts/filters/test_antispam.py @@ -1,6 +1,6 @@ import unittest -from bot.cogs import antispam +from bot.exts.filters import antispam class AntispamConfigurationValidationTests(unittest.TestCase): diff --git a/tests/bot/cogs/test_security.py b/tests/bot/exts/filters/test_security.py index 9d1a62f7e..c0c3baa42 100644 --- a/tests/bot/cogs/test_security.py +++ b/tests/bot/exts/filters/test_security.py @@ -3,7 +3,7 @@ from unittest.mock import MagicMock from discord.ext.commands import NoPrivateMessage -from bot.cogs import security +from bot.exts.filters import security from tests.helpers import MockBot, MockContext diff --git a/tests/bot/exts/filters/test_token_remover.py b/tests/bot/exts/filters/test_token_remover.py new file mode 100644 index 000000000..f99cc3370 --- /dev/null +++ b/tests/bot/exts/filters/test_token_remover.py @@ -0,0 +1,408 @@ +import unittest +from re import Match +from unittest import mock +from unittest.mock import MagicMock + +from discord import Colour, NotFound + +from bot import constants +from bot.exts.filters import token_remover +from bot.exts.filters.token_remover import Token, TokenRemover +from bot.exts.moderation.modlog import ModLog +from bot.utils.messages import format_user +from tests.helpers import MockBot, MockMessage, autospec + + +class TokenRemoverTests(unittest.IsolatedAsyncioTestCase): + """Tests the `TokenRemover` cog.""" + + def setUp(self): + """Adds the cog, a bot, and a message to the instance for usage in tests.""" + self.bot = MockBot() + self.cog = TokenRemover(bot=self.bot) + + self.msg = MockMessage(id=555, content="hello world") + self.msg.channel.mention = "#lemonade-stand" + self.msg.guild.get_member.return_value.bot = False + self.msg.guild.get_member.return_value.__str__.return_value = "Woody" + self.msg.author.__str__ = MagicMock(return_value=self.msg.author.name) + self.msg.author.avatar_url_as.return_value = "picture-lemon.png" + + def test_extract_user_id_valid(self): + """Should consider user IDs valid if they decode into an integer ID.""" + id_pairs = ( + ("NDcyMjY1OTQzMDYyNDEzMzMy", 472265943062413332), + ("NDc1MDczNjI5Mzk5NTQ3OTA0", 475073629399547904), + ("NDY3MjIzMjMwNjUwNzc3NjQx", 467223230650777641), + ) + + for token_id, user_id in id_pairs: + with self.subTest(token_id=token_id): + result = TokenRemover.extract_user_id(token_id) + self.assertEqual(result, user_id) + + def test_extract_user_id_invalid(self): + """Should consider non-digit and non-ASCII IDs invalid.""" + ids = ( + ("SGVsbG8gd29ybGQ", "non-digit ASCII"), + ("0J_RgNC40LLQtdGCINC80LjRgA", "cyrillic text"), + ("4pO14p6L4p6C4pG34p264pGl8J-EiOKSj-KCieKBsA", "Unicode digits"), + ("4oaA4oaB4oWh4oWi4Lyz4Lyq4Lyr4LG9", "Unicode numerals"), + ("8J2fjvCdn5nwnZ-k8J2fr_Cdn7rgravvvJngr6c", "Unicode decimals"), + ("{hello}[world]&(bye!)", "ASCII invalid Base64"), + ("Þíß-ï§-ňøẗ-våłìÐ", "Unicode invalid Base64"), + ) + + for user_id, msg in ids: + with self.subTest(msg=msg): + result = TokenRemover.extract_user_id(user_id) + self.assertIsNone(result) + + def test_is_valid_timestamp_valid(self): + """Should consider timestamps valid if they're greater than the Discord epoch.""" + timestamps = ( + "XsyRkw", + "Xrim9Q", + "XsyR-w", + "XsySD_", + "Dn9r_A", + ) + + for timestamp in timestamps: + with self.subTest(timestamp=timestamp): + result = TokenRemover.is_valid_timestamp(timestamp) + self.assertTrue(result) + + def test_is_valid_timestamp_invalid(self): + """Should consider timestamps invalid if they're before Discord epoch or can't be parsed.""" + timestamps = ( + ("B4Yffw", "DISCORD_EPOCH - TOKEN_EPOCH - 1"), + ("ew", "123"), + ("AoIKgA", "42076800"), + ("{hello}[world]&(bye!)", "ASCII invalid Base64"), + ("Þíß-ï§-ňøẗ-våłìÐ", "Unicode invalid Base64"), + ) + + for timestamp, msg in timestamps: + with self.subTest(msg=msg): + result = TokenRemover.is_valid_timestamp(timestamp) + self.assertFalse(result) + + def test_is_valid_hmac_valid(self): + """Should consider an HMAC valid if it has at least 3 unique characters.""" + valid_hmacs = ( + "VXmErH7j511turNpfURmb0rVNm8", + "Ysnu2wacjaKs7qnoo46S8Dm2us8", + "sJf6omBPORBPju3WJEIAcwW9Zds", + "s45jqDV_Iisn-symw0yDRrk_jf4", + ) + + for hmac in valid_hmacs: + with self.subTest(msg=hmac): + result = TokenRemover.is_maybe_valid_hmac(hmac) + self.assertTrue(result) + + def test_is_invalid_hmac_invalid(self): + """Should consider an HMAC invalid if has fewer than 3 unique characters.""" + invalid_hmacs = ( + ("xxxxxxxxxxxxxxxxxx", "Single character"), + ("XxXxXxXxXxXxXxXxXx", "Single character alternating case"), + ("ASFasfASFasfASFASsf", "Three characters alternating-case"), + ("asdasdasdasdasdasdasd", "Three characters one case"), + ) + + for hmac, msg in invalid_hmacs: + with self.subTest(msg=msg): + result = TokenRemover.is_maybe_valid_hmac(hmac) + self.assertFalse(result) + + def test_mod_log_property(self): + """The `mod_log` property should ask the bot to return the `ModLog` cog.""" + self.bot.get_cog.return_value = 'lemon' + self.assertEqual(self.cog.mod_log, self.bot.get_cog.return_value) + self.bot.get_cog.assert_called_once_with('ModLog') + + async def test_on_message_edit_uses_on_message(self): + """The edit listener should delegate handling of the message to the normal listener.""" + self.cog.on_message = mock.create_autospec(self.cog.on_message, spec_set=True) + + await self.cog.on_message_edit(MockMessage(), self.msg) + self.cog.on_message.assert_awaited_once_with(self.msg) + + @autospec(TokenRemover, "find_token_in_message", "take_action") + async def test_on_message_takes_action(self, find_token_in_message, take_action): + """Should take action if a valid token is found when a message is sent.""" + cog = TokenRemover(self.bot) + found_token = "foobar" + find_token_in_message.return_value = found_token + + await cog.on_message(self.msg) + + find_token_in_message.assert_called_once_with(self.msg) + take_action.assert_awaited_once_with(cog, self.msg, found_token) + + @autospec(TokenRemover, "find_token_in_message", "take_action") + async def test_on_message_skips_missing_token(self, find_token_in_message, take_action): + """Shouldn't take action if a valid token isn't found when a message is sent.""" + cog = TokenRemover(self.bot) + find_token_in_message.return_value = False + + await cog.on_message(self.msg) + + find_token_in_message.assert_called_once_with(self.msg) + take_action.assert_not_awaited() + + @autospec(TokenRemover, "find_token_in_message") + async def test_on_message_ignores_dms_bots(self, find_token_in_message): + """Shouldn't parse a message if it is a DM or authored by a bot.""" + cog = TokenRemover(self.bot) + dm_msg = MockMessage(guild=None) + bot_msg = MockMessage(author=MagicMock(bot=True)) + + for msg in (dm_msg, bot_msg): + await cog.on_message(msg) + find_token_in_message.assert_not_called() + + @autospec("bot.exts.filters.token_remover", "TOKEN_RE") + def test_find_token_no_matches(self, token_re): + """None should be returned if the regex matches no tokens in a message.""" + token_re.finditer.return_value = () + + return_value = TokenRemover.find_token_in_message(self.msg) + + self.assertIsNone(return_value) + token_re.finditer.assert_called_once_with(self.msg.content) + + @autospec(TokenRemover, "extract_user_id", "is_valid_timestamp", "is_maybe_valid_hmac") + @autospec("bot.exts.filters.token_remover", "Token") + @autospec("bot.exts.filters.token_remover", "TOKEN_RE") + def test_find_token_valid_match( + self, + token_re, + token_cls, + extract_user_id, + is_valid_timestamp, + is_maybe_valid_hmac, + ): + """The first match with a valid user ID, timestamp, and HMAC should be returned as a `Token`.""" + matches = [ + mock.create_autospec(Match, spec_set=True, instance=True), + mock.create_autospec(Match, spec_set=True, instance=True), + ] + tokens = [ + mock.create_autospec(Token, spec_set=True, instance=True), + mock.create_autospec(Token, spec_set=True, instance=True), + ] + + token_re.finditer.return_value = matches + token_cls.side_effect = tokens + extract_user_id.side_effect = (None, True) # The 1st match will be invalid, 2nd one valid. + is_valid_timestamp.return_value = True + is_maybe_valid_hmac.return_value = True + + return_value = TokenRemover.find_token_in_message(self.msg) + + self.assertEqual(tokens[1], return_value) + token_re.finditer.assert_called_once_with(self.msg.content) + + @autospec(TokenRemover, "extract_user_id", "is_valid_timestamp", "is_maybe_valid_hmac") + @autospec("bot.exts.filters.token_remover", "Token") + @autospec("bot.exts.filters.token_remover", "TOKEN_RE") + def test_find_token_invalid_matches( + self, + token_re, + token_cls, + extract_user_id, + is_valid_timestamp, + is_maybe_valid_hmac, + ): + """None should be returned if no matches have valid user IDs, HMACs, and timestamps.""" + token_re.finditer.return_value = [mock.create_autospec(Match, spec_set=True, instance=True)] + token_cls.return_value = mock.create_autospec(Token, spec_set=True, instance=True) + extract_user_id.return_value = None + is_valid_timestamp.return_value = False + is_maybe_valid_hmac.return_value = False + + return_value = TokenRemover.find_token_in_message(self.msg) + + self.assertIsNone(return_value) + token_re.finditer.assert_called_once_with(self.msg.content) + + def test_regex_invalid_tokens(self): + """Messages without anything looking like a token are not matched.""" + tokens = ( + "", + "lemon wins", + "..", + "x.y", + "x.y.", + ".y.z", + ".y.", + "..z", + "x..z", + " . . ", + "\n.\n.\n", + "hellö.world.bye", + "base64.nötbåse64.morebase64", + "19jd3J.dfkm3d.€víł§tüff", + ) + + for token in tokens: + with self.subTest(token=token): + results = token_remover.TOKEN_RE.findall(token) + self.assertEqual(len(results), 0) + + def test_regex_valid_tokens(self): + """Messages that look like tokens should be matched.""" + # Don't worry, these tokens have been invalidated. + tokens = ( + "NDcyMjY1OTQzMDYy_DEzMz-y.XsyRkw.VXmErH7j511turNpfURmb0rVNm8", + "NDcyMjY1OTQzMDYyNDEzMzMy.Xrim9Q.Ysnu2wacjaKs7qnoo46S8Dm2us8", + "NDc1MDczNjI5Mzk5NTQ3OTA0.XsyR-w.sJf6omBPORBPju3WJEIAcwW9Zds", + "NDY3MjIzMjMwNjUwNzc3NjQx.XsySD_.s45jqDV_Iisn-symw0yDRrk_jf4", + ) + + for token in tokens: + with self.subTest(token=token): + results = token_remover.TOKEN_RE.fullmatch(token) + self.assertIsNotNone(results, f"{token} was not matched by the regex") + + def test_regex_matches_multiple_valid(self): + """Should support multiple matches in the middle of a string.""" + token_1 = "NDY3MjIzMjMwNjUwNzc3NjQx.XsyWGg.uFNEQPCc4ePwGh7egG8UicQssz8" + token_2 = "NDcyMjY1OTQzMDYyNDEzMzMy.XsyWMw.l8XPnDqb0lp-EiQ2g_0xVFT1pyc" + message = f"garbage {token_1} hello {token_2} world" + + results = token_remover.TOKEN_RE.finditer(message) + results = [match[0] for match in results] + self.assertCountEqual((token_1, token_2), results) + + @autospec("bot.exts.filters.token_remover", "LOG_MESSAGE") + def test_format_log_message(self, log_message): + """Should correctly format the log message with info from the message and token.""" + token = Token("NDcyMjY1OTQzMDYyNDEzMzMy", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") + log_message.format.return_value = "Howdy" + + return_value = TokenRemover.format_log_message(self.msg, token) + + self.assertEqual(return_value, log_message.format.return_value) + log_message.format.assert_called_once_with( + author=format_user(self.msg.author), + channel=self.msg.channel.mention, + user_id=token.user_id, + timestamp=token.timestamp, + hmac="x" * len(token.hmac), + ) + + @autospec("bot.exts.filters.token_remover", "UNKNOWN_USER_LOG_MESSAGE") + def test_format_userid_log_message_unknown(self, unknown_user_log_message): + """Should correctly format the user ID portion when the actual user it belongs to is unknown.""" + token = Token("NDcyMjY1OTQzMDYyNDEzMzMy", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") + unknown_user_log_message.format.return_value = " Partner" + msg = MockMessage(id=555, content="hello world") + msg.guild.get_member.return_value = None + + return_value = TokenRemover.format_userid_log_message(msg, token) + + self.assertEqual(return_value, (unknown_user_log_message.format.return_value, False)) + unknown_user_log_message.format.assert_called_once_with(user_id=472265943062413332) + + @autospec("bot.exts.filters.token_remover", "KNOWN_USER_LOG_MESSAGE") + def test_format_userid_log_message_bot(self, known_user_log_message): + """Should correctly format the user ID portion when the ID belongs to a known bot.""" + token = Token("NDcyMjY1OTQzMDYyNDEzMzMy", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") + known_user_log_message.format.return_value = " Partner" + msg = MockMessage(id=555, content="hello world") + msg.guild.get_member.return_value.__str__.return_value = "Sam" + msg.guild.get_member.return_value.bot = True + + return_value = TokenRemover.format_userid_log_message(msg, token) + + self.assertEqual(return_value, (known_user_log_message.format.return_value, False)) + + known_user_log_message.format.assert_called_once_with( + user_id=472265943062413332, + user_name="Sam", + kind="BOT", + ) + + @autospec("bot.exts.filters.token_remover", "KNOWN_USER_LOG_MESSAGE") + def test_format_log_message_user_token_user(self, user_token_message): + """Should correctly format the user ID portion when the ID belongs to a known user.""" + token = Token("NDY3MjIzMjMwNjUwNzc3NjQx", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") + user_token_message.format.return_value = "Partner" + + return_value = TokenRemover.format_userid_log_message(self.msg, token) + + self.assertEqual(return_value, (user_token_message.format.return_value, True)) + user_token_message.format.assert_called_once_with( + user_id=467223230650777641, + user_name="Woody", + kind="USER", + ) + + @mock.patch.object(TokenRemover, "mod_log", new_callable=mock.PropertyMock) + @autospec("bot.exts.filters.token_remover", "log") + @autospec(TokenRemover, "format_log_message", "format_userid_log_message") + async def test_take_action(self, format_log_message, format_userid_log_message, logger, mod_log_property): + """Should delete the message and send a mod log.""" + cog = TokenRemover(self.bot) + mod_log = mock.create_autospec(ModLog, spec_set=True, instance=True) + token = mock.create_autospec(Token, spec_set=True, instance=True) + token.user_id = "no-id" + log_msg = "testing123" + userid_log_message = "userid-log-message" + + mod_log_property.return_value = mod_log + format_log_message.return_value = log_msg + format_userid_log_message.return_value = (userid_log_message, True) + + await cog.take_action(self.msg, token) + + self.msg.delete.assert_called_once_with() + self.msg.channel.send.assert_called_once_with( + token_remover.DELETION_MESSAGE_TEMPLATE.format(mention=self.msg.author.mention) + ) + + format_log_message.assert_called_once_with(self.msg, token) + format_userid_log_message.assert_called_once_with(self.msg, token) + logger.debug.assert_called_with(log_msg) + self.bot.stats.incr.assert_called_once_with("tokens.removed_tokens") + + mod_log.ignore.assert_called_once_with(constants.Event.message_delete, self.msg.id) + mod_log.send_log_message.assert_called_once_with( + icon_url=constants.Icons.token_removed, + colour=Colour(constants.Colours.soft_red), + title="Token removed!", + text=log_msg + "\n" + userid_log_message, + thumbnail=self.msg.author.avatar_url_as.return_value, + channel_id=constants.Channels.mod_alerts, + ping_everyone=True, + ) + + @mock.patch.object(TokenRemover, "mod_log", new_callable=mock.PropertyMock) + async def test_take_action_delete_failure(self, mod_log_property): + """Shouldn't send any messages if the token message can't be deleted.""" + cog = TokenRemover(self.bot) + mod_log_property.return_value = mock.create_autospec(ModLog, spec_set=True, instance=True) + self.msg.delete.side_effect = NotFound(MagicMock(), MagicMock()) + + token = mock.create_autospec(Token, spec_set=True, instance=True) + await cog.take_action(self.msg, token) + + self.msg.delete.assert_called_once_with() + self.msg.channel.send.assert_not_awaited() + + +class TokenRemoverExtensionTests(unittest.TestCase): + """Tests for the token_remover extension.""" + + @autospec("bot.exts.filters.token_remover", "TokenRemover") + def test_extension_setup(self, cog): + """The TokenRemover cog should be added.""" + bot = MockBot() + token_remover.setup(bot) + + cog.assert_called_once_with(bot) + bot.add_cog.assert_called_once() + self.assertTrue(isinstance(bot.add_cog.call_args.args[0], TokenRemover)) diff --git a/tests/bot/exts/info/__init__.py b/tests/bot/exts/info/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/bot/exts/info/__init__.py diff --git a/tests/bot/cogs/test_information.py b/tests/bot/exts/info/test_information.py index a3f80b1e5..36a35c8e2 100644 --- a/tests/bot/cogs/test_information.py +++ b/tests/bot/exts/info/test_information.py @@ -5,12 +5,11 @@ import unittest.mock import discord from bot import constants -from bot.cogs import information -from bot.decorators import InChannelCheckFailure +from bot.exts.info import information +from bot.utils.checks import InWhitelistCheckFailure from tests import helpers - -COG_PATH = "bot.cogs.information.Information" +COG_PATH = "bot.exts.info.information.Information" class InformationCogTests(unittest.IsolatedAsyncioTestCase): @@ -93,7 +92,7 @@ class InformationCogTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(admin_embed.title, "Admins info") self.assertEqual(admin_embed.colour, discord.Colour.red()) - @unittest.mock.patch('bot.cogs.information.time_since') + @unittest.mock.patch('bot.exts.info.information.time_since') async def test_server_info_command(self, time_since_patch): time_since_patch.return_value = '2 days ago' @@ -143,14 +142,18 @@ class InformationCogTests(unittest.IsolatedAsyncioTestCase): Voice region: {self.ctx.guild.region} Features: {', '.join(self.ctx.guild.features)} - **Counts** - Members: {self.ctx.guild.member_count:,} - Roles: {len(self.ctx.guild.roles)} + **Channel counts** Category channels: 1 Text channels: 1 Voice channels: 1 + Staff channels: 0 - **Members** + **Member counts** + Members: {self.ctx.guild.member_count:,} + Staff members: 0 + Roles: {len(self.ctx.guild.roles)} + + **Member statuses** {constants.Emojis.status_online} 2 {constants.Emojis.status_idle} 1 {constants.Emojis.status_dnd} 4 @@ -206,10 +209,10 @@ class UserInfractionHelperMethodTests(unittest.IsolatedAsyncioTestCase): with self.subTest(method=method, api_response=api_response, expected_lines=expected_lines): self.bot.api_client.get.return_value = api_response - expected_output = "\n".join(default_header + expected_lines) + expected_output = "\n".join(expected_lines) actual_output = await method(self.member) - self.assertEqual(expected_output, actual_output) + self.assertEqual((default_header, expected_output), actual_output) async def test_basic_user_infraction_counts_returns_correct_strings(self): """The method should correctly list both the total and active number of non-hidden infractions.""" @@ -240,7 +243,7 @@ class UserInfractionHelperMethodTests(unittest.IsolatedAsyncioTestCase): }, ) - header = ["**Infractions**"] + header = "Infractions" await self._method_subtests(self.cog.basic_user_infraction_counts, test_values, header) @@ -249,7 +252,7 @@ class UserInfractionHelperMethodTests(unittest.IsolatedAsyncioTestCase): test_values = ( { "api response": [], - "expected_lines": ["This user has never received an infraction."], + "expected_lines": ["No infractions"], }, # Shows non-hidden inactive infraction as expected { @@ -295,7 +298,7 @@ class UserInfractionHelperMethodTests(unittest.IsolatedAsyncioTestCase): }, ) - header = ["**Infractions**"] + header = "Infractions" await self._method_subtests(self.cog.expanded_user_infraction_counts, test_values, header) @@ -304,15 +307,15 @@ class UserInfractionHelperMethodTests(unittest.IsolatedAsyncioTestCase): test_values = ( { "api response": [], - "expected_lines": ["This user has never been nominated."], + "expected_lines": ["No nominations"], }, { "api response": [{'active': True}], - "expected_lines": ["This user is **currently** nominated (1 nomination in total)."], + "expected_lines": ["This user is **currently** nominated", "(1 nomination in total)"], }, { "api response": [{'active': True}, {'active': False}], - "expected_lines": ["This user is **currently** nominated (2 nominations in total)."], + "expected_lines": ["This user is **currently** nominated", "(2 nominations in total)"], }, { "api response": [{'active': False}], @@ -325,13 +328,13 @@ class UserInfractionHelperMethodTests(unittest.IsolatedAsyncioTestCase): ) - header = ["**Nominations**"] + header = "Nominations" await self._method_subtests(self.cog.user_nomination_counts, test_values, header) [email protected]("bot.cogs.information.time_since", new=unittest.mock.MagicMock(return_value="1 year ago")) [email protected]("bot.cogs.information.constants.MODERATION_CHANNELS", new=[50]) [email protected]("bot.exts.info.information.time_since", new=unittest.mock.MagicMock(return_value="1 year ago")) [email protected]("bot.exts.info.information.constants.MODERATION_CHANNELS", new=[50]) class UserEmbedTests(unittest.IsolatedAsyncioTestCase): """Tests for the creation of the `!user` embed.""" @@ -341,7 +344,10 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): self.bot.api_client.get = unittest.mock.AsyncMock() self.cog = information.Information(self.bot) - @unittest.mock.patch(f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value="")) + @unittest.mock.patch( + f"{COG_PATH}.basic_user_infraction_counts", + new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) + ) async def test_create_user_embed_uses_string_representation_of_user_in_title_if_nick_is_not_available(self): """The embed should use the string representation of the user if they don't have a nick.""" ctx = helpers.MockContext(channel=helpers.MockTextChannel(id=1)) @@ -353,7 +359,10 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(embed.title, "Mr. Hemlock") - @unittest.mock.patch(f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value="")) + @unittest.mock.patch( + f"{COG_PATH}.basic_user_infraction_counts", + new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) + ) async def test_create_user_embed_uses_nick_in_title_if_available(self): """The embed should use the nick if it's available.""" ctx = helpers.MockContext(channel=helpers.MockTextChannel(id=1)) @@ -365,7 +374,10 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(embed.title, "Cat lover (Mr. Hemlock)") - @unittest.mock.patch(f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value="")) + @unittest.mock.patch( + f"{COG_PATH}.basic_user_infraction_counts", + new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) + ) async def test_create_user_embed_ignores_everyone_role(self): """Created `!user` embeds should not contain mention of the @everyone-role.""" ctx = helpers.MockContext(channel=helpers.MockTextChannel(id=1)) @@ -377,8 +389,8 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): embed = await self.cog.create_user_embed(ctx, user) - self.assertIn("&Admins", embed.description) - self.assertNotIn("&Everyone", embed.description) + self.assertIn("&Admins", embed.fields[1].value) + self.assertNotIn("&Everyone", embed.fields[1].value) @unittest.mock.patch(f"{COG_PATH}.expanded_user_infraction_counts", new_callable=unittest.mock.AsyncMock) @unittest.mock.patch(f"{COG_PATH}.user_nomination_counts", new_callable=unittest.mock.AsyncMock) @@ -393,8 +405,8 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): moderators_role = helpers.MockRole(name='Moderators') moderators_role.colour = 100 - infraction_counts.return_value = "expanded infractions info" - nomination_counts.return_value = "nomination info" + infraction_counts.return_value = ("Infractions", "expanded infractions info") + nomination_counts.return_value = ("Nominations", "nomination info") user = helpers.MockMember(id=314, roles=[moderators_role], top_role=moderators_role) embed = await self.cog.create_user_embed(ctx, user) @@ -404,20 +416,19 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): self.assertEqual( textwrap.dedent(f""" - **User Information** Created: {"1 year ago"} Profile: {user.mention} ID: {user.id} + """).strip(), + embed.fields[0].value + ) - **Member Information** + self.assertEqual( + textwrap.dedent(f""" Joined: {"1 year ago"} Roles: &Moderators - - expanded infractions info - - nomination info """).strip(), - embed.description + embed.fields[1].value ) @unittest.mock.patch(f"{COG_PATH}.basic_user_infraction_counts", new_callable=unittest.mock.AsyncMock) @@ -428,7 +439,7 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): moderators_role = helpers.MockRole(name='Moderators') moderators_role.colour = 100 - infraction_counts.return_value = "basic infractions info" + infraction_counts.return_value = ("Infractions", "basic infractions info") user = helpers.MockMember(id=314, roles=[moderators_role], top_role=moderators_role) embed = await self.cog.create_user_embed(ctx, user) @@ -437,21 +448,30 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): self.assertEqual( textwrap.dedent(f""" - **User Information** Created: {"1 year ago"} Profile: {user.mention} ID: {user.id} + """).strip(), + embed.fields[0].value + ) - **Member Information** + self.assertEqual( + textwrap.dedent(f""" Joined: {"1 year ago"} Roles: &Moderators - - basic infractions info """).strip(), - embed.description + embed.fields[1].value ) - @unittest.mock.patch(f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value="")) + self.assertEqual( + "basic infractions info", + embed.fields[3].value + ) + + @unittest.mock.patch( + f"{COG_PATH}.basic_user_infraction_counts", + new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) + ) async def test_create_user_embed_uses_top_role_colour_when_user_has_roles(self): """The embed should be created with the colour of the top role, if a top role is available.""" ctx = helpers.MockContext() @@ -464,7 +484,10 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(embed.colour, discord.Colour(moderators_role.colour)) - @unittest.mock.patch(f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value="")) + @unittest.mock.patch( + f"{COG_PATH}.basic_user_infraction_counts", + new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) + ) async def test_create_user_embed_uses_blurple_colour_when_user_has_no_roles(self): """The embed should be created with a blurple colour if the user has no assigned roles.""" ctx = helpers.MockContext() @@ -474,7 +497,10 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(embed.colour, discord.Colour.blurple()) - @unittest.mock.patch(f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value="")) + @unittest.mock.patch( + f"{COG_PATH}.basic_user_infraction_counts", + new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) + ) async def test_create_user_embed_uses_png_format_of_user_avatar_as_thumbnail(self): """The embed thumbnail should be set to the user's avatar in `png` format.""" ctx = helpers.MockContext() @@ -483,11 +509,11 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase): user.avatar_url_as.return_value = "avatar url" embed = await self.cog.create_user_embed(ctx, user) - user.avatar_url_as.assert_called_once_with(format="png") + user.avatar_url_as.assert_called_once_with(static_format="png") self.assertEqual(embed.thumbnail.url, "avatar url") [email protected]("bot.cogs.information.constants") [email protected]("bot.exts.info.information.constants") class UserCommandTests(unittest.IsolatedAsyncioTestCase): """Tests for the `!user` command.""" @@ -504,10 +530,13 @@ class UserCommandTests(unittest.IsolatedAsyncioTestCase): self.moderator = helpers.MockMember(id=2, name="riffautae", roles=[self.moderator_role]) self.target = helpers.MockMember(id=3, name="__fluzz__") + # There's no way to mock the channel constant without deferring imports. The constant is + # used as a default value for a parameter, which gets defined upon import. + self.bot_command_channel = helpers.MockTextChannel(id=constants.Channels.bot_commands) + async def test_regular_member_cannot_target_another_member(self, constants): """A regular user should not be able to use `!user` targeting another user.""" constants.MODERATION_ROLES = [self.moderator_role.id] - ctx = helpers.MockContext(author=self.author) await self.cog.user_info(self.cog, ctx, self.target) @@ -518,46 +547,38 @@ class UserCommandTests(unittest.IsolatedAsyncioTestCase): """A regular user should not be able to use this command outside of bot-commands.""" constants.MODERATION_ROLES = [self.moderator_role.id] constants.STAFF_ROLES = [self.moderator_role.id] - constants.Channels.bot_commands = 50 - ctx = helpers.MockContext(author=self.author, channel=helpers.MockTextChannel(id=100)) msg = "Sorry, but you may only use this command within <#50>." - with self.assertRaises(InChannelCheckFailure, msg=msg): + with self.assertRaises(InWhitelistCheckFailure, msg=msg): await self.cog.user_info(self.cog, ctx) - @unittest.mock.patch("bot.cogs.information.Information.create_user_embed", new_callable=unittest.mock.AsyncMock) + @unittest.mock.patch("bot.exts.info.information.Information.create_user_embed") async def test_regular_user_may_use_command_in_bot_commands_channel(self, create_embed, constants): """A regular user should be allowed to use `!user` targeting themselves in bot-commands.""" constants.STAFF_ROLES = [self.moderator_role.id] - constants.Channels.bot_commands = 50 - - ctx = helpers.MockContext(author=self.author, channel=helpers.MockTextChannel(id=50)) + ctx = helpers.MockContext(author=self.author, channel=self.bot_command_channel) await self.cog.user_info(self.cog, ctx) create_embed.assert_called_once_with(ctx, self.author) ctx.send.assert_called_once() - @unittest.mock.patch("bot.cogs.information.Information.create_user_embed", new_callable=unittest.mock.AsyncMock) - async def test_regular_user_can_explicitly_target_themselves(self, create_embed, constants): + @unittest.mock.patch("bot.exts.info.information.Information.create_user_embed") + async def test_regular_user_can_explicitly_target_themselves(self, create_embed, _): """A user should target itself with `!user` when a `user` argument was not provided.""" constants.STAFF_ROLES = [self.moderator_role.id] - constants.Channels.bot_commands = 50 - - ctx = helpers.MockContext(author=self.author, channel=helpers.MockTextChannel(id=50)) + ctx = helpers.MockContext(author=self.author, channel=self.bot_command_channel) await self.cog.user_info(self.cog, ctx, self.author) create_embed.assert_called_once_with(ctx, self.author) ctx.send.assert_called_once() - @unittest.mock.patch("bot.cogs.information.Information.create_user_embed", new_callable=unittest.mock.AsyncMock) + @unittest.mock.patch("bot.exts.info.information.Information.create_user_embed") async def test_staff_members_can_bypass_channel_restriction(self, create_embed, constants): """Staff members should be able to bypass the bot-commands channel restriction.""" constants.STAFF_ROLES = [self.moderator_role.id] - constants.Channels.bot_commands = 50 - ctx = helpers.MockContext(author=self.moderator, channel=helpers.MockTextChannel(id=200)) await self.cog.user_info(self.cog, ctx) @@ -565,12 +586,11 @@ class UserCommandTests(unittest.IsolatedAsyncioTestCase): create_embed.assert_called_once_with(ctx, self.moderator) ctx.send.assert_called_once() - @unittest.mock.patch("bot.cogs.information.Information.create_user_embed", new_callable=unittest.mock.AsyncMock) + @unittest.mock.patch("bot.exts.info.information.Information.create_user_embed") async def test_moderators_can_target_another_member(self, create_embed, constants): """A moderator should be able to use `!user` targeting another user.""" constants.MODERATION_ROLES = [self.moderator_role.id] constants.STAFF_ROLES = [self.moderator_role.id] - ctx = helpers.MockContext(author=self.moderator, channel=helpers.MockTextChannel(id=50)) await self.cog.user_info(self.cog, ctx, self.target) diff --git a/tests/bot/exts/moderation/__init__.py b/tests/bot/exts/moderation/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/bot/exts/moderation/__init__.py diff --git a/tests/bot/exts/moderation/infraction/__init__.py b/tests/bot/exts/moderation/infraction/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/bot/exts/moderation/infraction/__init__.py diff --git a/tests/bot/exts/moderation/infraction/test_infractions.py b/tests/bot/exts/moderation/infraction/test_infractions.py new file mode 100644 index 000000000..be1b649e1 --- /dev/null +++ b/tests/bot/exts/moderation/infraction/test_infractions.py @@ -0,0 +1,55 @@ +import textwrap +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from bot.exts.moderation.infraction.infractions import Infractions +from tests.helpers import MockBot, MockContext, MockGuild, MockMember, MockRole + + +class TruncationTests(unittest.IsolatedAsyncioTestCase): + """Tests for ban and kick command reason truncation.""" + + def setUp(self): + self.bot = MockBot() + self.cog = Infractions(self.bot) + self.user = MockMember(id=1234, top_role=MockRole(id=3577, position=10)) + self.target = MockMember(id=1265, top_role=MockRole(id=9876, position=0)) + self.guild = MockGuild(id=4567) + self.ctx = MockContext(bot=self.bot, author=self.user, guild=self.guild) + + @patch("bot.exts.moderation.infraction._utils.get_active_infraction") + @patch("bot.exts.moderation.infraction._utils.post_infraction") + async def test_apply_ban_reason_truncation(self, post_infraction_mock, get_active_mock): + """Should truncate reason for `ctx.guild.ban`.""" + get_active_mock.return_value = None + post_infraction_mock.return_value = {"foo": "bar"} + + self.cog.apply_infraction = AsyncMock() + self.bot.get_cog.return_value = AsyncMock() + self.cog.mod_log.ignore = Mock() + self.ctx.guild.ban = Mock() + + await self.cog.apply_ban(self.ctx, self.target, "foo bar" * 3000) + self.ctx.guild.ban.assert_called_once_with( + self.target, + reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="..."), + delete_message_days=0 + ) + self.cog.apply_infraction.assert_awaited_once_with( + self.ctx, {"foo": "bar"}, self.target, self.ctx.guild.ban.return_value + ) + + @patch("bot.exts.moderation.infraction._utils.post_infraction") + async def test_apply_kick_reason_truncation(self, post_infraction_mock): + """Should truncate reason for `Member.kick`.""" + post_infraction_mock.return_value = {"foo": "bar"} + + self.cog.apply_infraction = AsyncMock() + self.cog.mod_log.ignore = Mock() + self.target.kick = Mock() + + await self.cog.apply_kick(self.ctx, self.target, "foo bar" * 3000) + self.target.kick.assert_called_once_with(reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="...")) + self.cog.apply_infraction.assert_awaited_once_with( + self.ctx, {"foo": "bar"}, self.target, self.target.kick.return_value + ) diff --git a/tests/bot/exts/moderation/infraction/test_utils.py b/tests/bot/exts/moderation/infraction/test_utils.py new file mode 100644 index 000000000..5b62463e0 --- /dev/null +++ b/tests/bot/exts/moderation/infraction/test_utils.py @@ -0,0 +1,359 @@ +import unittest +from collections import namedtuple +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock, call, patch + +from discord import Embed, Forbidden, HTTPException, NotFound + +from bot.api import ResponseCodeError +from bot.constants import Colours, Icons +from bot.exts.moderation.infraction import _utils as utils +from tests.helpers import MockBot, MockContext, MockMember, MockUser + + +class ModerationUtilsTests(unittest.IsolatedAsyncioTestCase): + """Tests Moderation utils.""" + + def setUp(self): + self.bot = MockBot() + self.member = MockMember(id=1234) + self.user = MockUser(id=1234) + self.ctx = MockContext(bot=self.bot, author=self.member) + + async def test_post_user(self): + """Should POST a new user and return the response if successful or otherwise send an error message.""" + user = MockUser(discriminator=5678, id=1234, name="Test user") + not_user = MagicMock(discriminator=3333, id=5678, name="Wrong user") + test_cases = [ + { + "user": user, + "post_result": "bar", + "raise_error": None, + "payload": { + "discriminator": 5678, + "id": self.user.id, + "in_guild": False, + "name": "Test user", + "roles": [] + } + }, + { + "user": self.member, + "post_result": "foo", + "raise_error": ResponseCodeError(MagicMock(status=400), "foo"), + "payload": { + "discriminator": 0, + "id": self.member.id, + "in_guild": False, + "name": "Name unknown", + "roles": [] + } + }, + { + "user": not_user, + "post_result": "bar", + "raise_error": None, + "payload": { + "discriminator": not_user.discriminator, + "id": not_user.id, + "in_guild": False, + "name": not_user.name, + "roles": [] + } + } + ] + + for case in test_cases: + user = case["user"] + post_result = case["post_result"] + raise_error = case["raise_error"] + payload = case["payload"] + + with self.subTest(user=user, post_result=post_result, raise_error=raise_error, payload=payload): + self.bot.api_client.post.reset_mock(side_effect=True) + self.ctx.bot.api_client.post.return_value = post_result + + self.ctx.bot.api_client.post.side_effect = raise_error + + result = await utils.post_user(self.ctx, user) + + if raise_error: + self.assertIsNone(result) + self.ctx.send.assert_awaited_once() + self.assertIn(str(raise_error.status), self.ctx.send.call_args[0][0]) + else: + self.assertEqual(result, post_result) + self.bot.api_client.post.assert_awaited_once_with("bot/users", json=payload) + + async def test_get_active_infraction(self): + """ + Should request the API for active infractions and return infraction if the user has one or `None` otherwise. + + A message should be sent to the context indicating a user already has an infraction, if that's the case. + """ + test_case = namedtuple("test_case", ["get_return_value", "expected_output", "infraction_nr", "send_msg"]) + test_cases = [ + test_case([], None, None, True), + test_case([{"id": 123987}], {"id": 123987}, "123987", False), + test_case([{"id": 123987}], {"id": 123987}, "123987", True) + ] + + for case in test_cases: + with self.subTest(return_value=case.get_return_value, expected=case.expected_output): + self.bot.api_client.get.reset_mock() + self.ctx.send.reset_mock() + + params = { + "active": "true", + "type": "ban", + "user__id": str(self.member.id) + } + + self.bot.api_client.get.return_value = case.get_return_value + + result = await utils.get_active_infraction(self.ctx, self.member, "ban", send_msg=case.send_msg) + self.assertEqual(result, case.expected_output) + self.bot.api_client.get.assert_awaited_once_with("bot/infractions", params=params) + + if case.send_msg and case.get_return_value: + self.ctx.send.assert_awaited_once() + sent_message = self.ctx.send.call_args[0][0] + self.assertIn(case.infraction_nr, sent_message) + self.assertIn("ban", sent_message) + else: + self.ctx.send.assert_not_awaited() + + @patch("bot.exts.moderation.infraction._utils.send_private_embed") + async def test_notify_infraction(self, send_private_embed_mock): + """ + Should send an embed of a certain format as a DM and return `True` if DM successful. + + Appealable infractions should have the appeal message in the embed's footer. + """ + test_cases = [ + { + "args": (self.user, "ban", "2020-02-26 09:20 (23 hours and 59 minutes)"), + "expected_output": Embed( + title=utils.INFRACTION_TITLE, + description=utils.INFRACTION_DESCRIPTION_TEMPLATE.format( + type="Ban", + expires="2020-02-26 09:20 (23 hours and 59 minutes)", + reason="No reason provided." + ), + colour=Colours.soft_red, + url=utils.RULES_URL + ).set_author( + name=utils.INFRACTION_AUTHOR_NAME, + url=utils.RULES_URL, + icon_url=Icons.token_removed + ).set_footer(text=utils.INFRACTION_APPEAL_FOOTER), + "send_result": True + }, + { + "args": (self.user, "warning", None, "Test reason."), + "expected_output": Embed( + title=utils.INFRACTION_TITLE, + description=utils.INFRACTION_DESCRIPTION_TEMPLATE.format( + type="Warning", + expires="N/A", + reason="Test reason." + ), + colour=Colours.soft_red, + url=utils.RULES_URL + ).set_author( + name=utils.INFRACTION_AUTHOR_NAME, + url=utils.RULES_URL, + icon_url=Icons.token_removed + ), + "send_result": False + }, + { + "args": (self.user, "note", None, None, Icons.defcon_denied), + "expected_output": Embed( + title=utils.INFRACTION_TITLE, + description=utils.INFRACTION_DESCRIPTION_TEMPLATE.format( + type="Note", + expires="N/A", + reason="No reason provided." + ), + colour=Colours.soft_red, + url=utils.RULES_URL + ).set_author( + name=utils.INFRACTION_AUTHOR_NAME, + url=utils.RULES_URL, + icon_url=Icons.defcon_denied + ), + "send_result": False + }, + { + "args": (self.user, "mute", "2020-02-26 09:20 (23 hours and 59 minutes)", "Test", Icons.defcon_denied), + "expected_output": Embed( + title=utils.INFRACTION_TITLE, + description=utils.INFRACTION_DESCRIPTION_TEMPLATE.format( + type="Mute", + expires="2020-02-26 09:20 (23 hours and 59 minutes)", + reason="Test" + ), + colour=Colours.soft_red, + url=utils.RULES_URL + ).set_author( + name=utils.INFRACTION_AUTHOR_NAME, + url=utils.RULES_URL, + icon_url=Icons.defcon_denied + ).set_footer(text=utils.INFRACTION_APPEAL_FOOTER), + "send_result": False + }, + { + "args": (self.user, "mute", None, "foo bar" * 4000, Icons.defcon_denied), + "expected_output": Embed( + title=utils.INFRACTION_TITLE, + description=utils.INFRACTION_DESCRIPTION_TEMPLATE.format( + type="Mute", + expires="N/A", + reason="foo bar" * 4000 + )[:2045] + "...", + colour=Colours.soft_red, + url=utils.RULES_URL + ).set_author( + name=utils.INFRACTION_AUTHOR_NAME, + url=utils.RULES_URL, + icon_url=Icons.defcon_denied + ).set_footer(text=utils.INFRACTION_APPEAL_FOOTER), + "send_result": True + } + ] + + for case in test_cases: + with self.subTest(args=case["args"], expected=case["expected_output"], send=case["send_result"]): + send_private_embed_mock.reset_mock() + + send_private_embed_mock.return_value = case["send_result"] + result = await utils.notify_infraction(*case["args"]) + + self.assertEqual(case["send_result"], result) + + embed = send_private_embed_mock.call_args[0][1] + + self.assertEqual(embed.to_dict(), case["expected_output"].to_dict()) + + send_private_embed_mock.assert_awaited_once_with(case["args"][0], embed) + + @patch("bot.exts.moderation.infraction._utils.send_private_embed") + async def test_notify_pardon(self, send_private_embed_mock): + """Should send an embed of a certain format as a DM and return `True` if DM successful.""" + test_case = namedtuple("test_case", ["args", "icon", "send_result"]) + test_cases = [ + test_case((self.user, "Test title", "Example content"), Icons.user_verified, True), + test_case((self.user, "Test title", "Example content", Icons.user_update), Icons.user_update, False) + ] + + for case in test_cases: + expected = Embed( + description="Example content", + colour=Colours.soft_green + ).set_author( + name="Test title", + icon_url=case.icon + ) + + with self.subTest(args=case.args, expected=expected): + send_private_embed_mock.reset_mock() + + send_private_embed_mock.return_value = case.send_result + + result = await utils.notify_pardon(*case.args) + self.assertEqual(case.send_result, result) + + embed = send_private_embed_mock.call_args[0][1] + self.assertEqual(embed.to_dict(), expected.to_dict()) + + send_private_embed_mock.assert_awaited_once_with(case.args[0], embed) + + async def test_send_private_embed(self): + """Should DM the user and return `True` on success or `False` on failure.""" + embed = Embed(title="Test", description="Test val") + + test_case = namedtuple("test_case", ["expected_output", "raised_exception"]) + test_cases = [ + test_case(True, None), + test_case(False, HTTPException(AsyncMock(), AsyncMock())), + test_case(False, Forbidden(AsyncMock(), AsyncMock())), + test_case(False, NotFound(AsyncMock(), AsyncMock())) + ] + + for case in test_cases: + with self.subTest(expected=case.expected_output, raised=case.raised_exception): + self.user.send.reset_mock(side_effect=True) + self.user.send.side_effect = case.raised_exception + + result = await utils.send_private_embed(self.user, embed) + + self.assertEqual(result, case.expected_output) + self.user.send.assert_awaited_once_with(embed=embed) + + +class TestPostInfraction(unittest.IsolatedAsyncioTestCase): + """Tests for the `post_infraction` function.""" + + def setUp(self): + self.bot = MockBot() + self.member = MockMember(id=1234) + self.user = MockUser(id=1234) + self.ctx = MockContext(bot=self.bot, author=self.member) + + async def test_normal_post_infraction(self): + """Should return response from POST request if there are no errors.""" + now = datetime.now() + payload = { + "actor": self.ctx.author.id, + "hidden": True, + "reason": "Test reason", + "type": "ban", + "user": self.member.id, + "active": False, + "expires_at": now.isoformat() + } + + self.ctx.bot.api_client.post.return_value = "foo" + actual = await utils.post_infraction(self.ctx, self.member, "ban", "Test reason", now, True, False) + + self.assertEqual(actual, "foo") + self.ctx.bot.api_client.post.assert_awaited_once_with("bot/infractions", json=payload) + + async def test_unknown_error_post_infraction(self): + """Should send an error message to chat when a non-400 error occurs.""" + self.ctx.bot.api_client.post.side_effect = ResponseCodeError(AsyncMock(), AsyncMock()) + self.ctx.bot.api_client.post.side_effect.status = 500 + + actual = await utils.post_infraction(self.ctx, self.user, "ban", "Test reason") + self.assertIsNone(actual) + + self.assertTrue("500" in self.ctx.send.call_args[0][0]) + + @patch("bot.exts.moderation.infraction._utils.post_user", return_value=None) + async def test_user_not_found_none_post_infraction(self, post_user_mock): + """Should abort and return `None` when a new user fails to be posted.""" + self.bot.api_client.post.side_effect = ResponseCodeError(MagicMock(status=400), {"user": "foo"}) + + actual = await utils.post_infraction(self.ctx, self.user, "mute", "Test reason") + self.assertIsNone(actual) + post_user_mock.assert_awaited_once_with(self.ctx, self.user) + + @patch("bot.exts.moderation.infraction._utils.post_user", return_value="bar") + async def test_first_fail_second_success_user_post_infraction(self, post_user_mock): + """Should post the user if they don't exist, POST infraction again, and return the response if successful.""" + payload = { + "actor": self.ctx.author.id, + "hidden": False, + "reason": "Test reason", + "type": "mute", + "user": self.user.id, + "active": True + } + + self.bot.api_client.post.side_effect = [ResponseCodeError(MagicMock(status=400), {"user": "foo"}), "foo"] + + actual = await utils.post_infraction(self.ctx, self.user, "mute", "Test reason") + self.assertEqual(actual, "foo") + self.bot.api_client.post.assert_has_awaits([call("bot/infractions", json=payload)] * 2) + post_user_mock.assert_awaited_once_with(self.ctx, self.user) diff --git a/tests/bot/exts/moderation/test_incidents.py b/tests/bot/exts/moderation/test_incidents.py new file mode 100644 index 000000000..cbf7f7bcf --- /dev/null +++ b/tests/bot/exts/moderation/test_incidents.py @@ -0,0 +1,770 @@ +import asyncio +import enum +import logging +import typing as t +import unittest +from unittest.mock import AsyncMock, MagicMock, call, patch + +import aiohttp +import discord + +from bot.constants import Colours +from bot.exts.moderation import incidents +from tests.helpers import ( + MockAsyncWebhook, + MockAttachment, + MockBot, + MockMember, + MockMessage, + MockReaction, + MockRole, + MockTextChannel, + MockUser, +) + + +class MockAsyncIterable: + """ + Helper for mocking asynchronous for loops. + + It does not appear that the `unittest` library currently provides anything that would + allow us to simply mock an async iterator, such as `discord.TextChannel.history`. + + We therefore write our own helper to wrap a regular synchronous iterable, and feed + its values via `__anext__` rather than `__next__`. + + This class was written for the purposes of testing the `Incidents` cog - it may not + be generic enough to be placed in the `tests.helpers` module. + """ + + def __init__(self, messages: t.Iterable): + """Take a sync iterable to be wrapped.""" + self.iter_messages = iter(messages) + + def __aiter__(self): + """Return `self` as we provide the `__anext__` method.""" + return self + + async def __anext__(self): + """ + Feed the next item, or raise `StopAsyncIteration`. + + Since we're wrapping a sync iterator, it will communicate that it has been depleted + by raising a `StopIteration`. The `async for` construct does not expect it, and we + therefore need to substitute it for the appropriate exception type. + """ + try: + return next(self.iter_messages) + except StopIteration: + raise StopAsyncIteration + + +class MockSignal(enum.Enum): + A = "A" + B = "B" + + +mock_404 = discord.NotFound( + response=MagicMock(aiohttp.ClientResponse), # Mock the erroneous response + message="Not found", +) + + +class TestDownloadFile(unittest.IsolatedAsyncioTestCase): + """Collection of tests for the `download_file` helper function.""" + + async def test_download_file_success(self): + """If `to_file` succeeds, function returns the acquired `discord.File`.""" + file = MagicMock(discord.File, filename="bigbadlemon.jpg") + attachment = MockAttachment(to_file=AsyncMock(return_value=file)) + + acquired_file = await incidents.download_file(attachment) + self.assertIs(file, acquired_file) + + async def test_download_file_404(self): + """If `to_file` encounters a 404, function handles the exception & returns None.""" + attachment = MockAttachment(to_file=AsyncMock(side_effect=mock_404)) + + acquired_file = await incidents.download_file(attachment) + self.assertIsNone(acquired_file) + + async def test_download_file_fail(self): + """If `to_file` fails on a non-404 error, function logs the exception & returns None.""" + arbitrary_error = discord.HTTPException(MagicMock(aiohttp.ClientResponse), "Arbitrary API error") + attachment = MockAttachment(to_file=AsyncMock(side_effect=arbitrary_error)) + + with self.assertLogs(logger=incidents.log, level=logging.ERROR): + acquired_file = await incidents.download_file(attachment) + + self.assertIsNone(acquired_file) + + +class TestMakeEmbed(unittest.IsolatedAsyncioTestCase): + """Collection of tests for the `make_embed` helper function.""" + + async def test_make_embed_actioned(self): + """Embed is coloured green and footer contains 'Actioned' when `outcome=Signal.ACTIONED`.""" + embed, file = await incidents.make_embed(MockMessage(), incidents.Signal.ACTIONED, MockMember()) + + self.assertEqual(embed.colour.value, Colours.soft_green) + self.assertIn("Actioned", embed.footer.text) + + async def test_make_embed_not_actioned(self): + """Embed is coloured red and footer contains 'Rejected' when `outcome=Signal.NOT_ACTIONED`.""" + embed, file = await incidents.make_embed(MockMessage(), incidents.Signal.NOT_ACTIONED, MockMember()) + + self.assertEqual(embed.colour.value, Colours.soft_red) + self.assertIn("Rejected", embed.footer.text) + + async def test_make_embed_content(self): + """Incident content appears as embed description.""" + incident = MockMessage(content="this is an incident") + embed, file = await incidents.make_embed(incident, incidents.Signal.ACTIONED, MockMember()) + + self.assertEqual(incident.content, embed.description) + + async def test_make_embed_with_attachment_succeeds(self): + """Incident's attachment is downloaded and displayed in the embed's image field.""" + file = MagicMock(discord.File, filename="bigbadjoe.jpg") + attachment = MockAttachment(filename="bigbadjoe.jpg") + incident = MockMessage(content="this is an incident", attachments=[attachment]) + + # Patch `download_file` to return our `file` + with patch("bot.exts.moderation.incidents.download_file", AsyncMock(return_value=file)): + embed, returned_file = await incidents.make_embed(incident, incidents.Signal.ACTIONED, MockMember()) + + self.assertIs(file, returned_file) + self.assertEqual("attachment://bigbadjoe.jpg", embed.image.url) + + async def test_make_embed_with_attachment_fails(self): + """Incident's attachment fails to download, proxy url is linked instead.""" + attachment = MockAttachment(proxy_url="discord.com/bigbadjoe.jpg") + incident = MockMessage(content="this is an incident", attachments=[attachment]) + + # Patch `download_file` to return None as if the download failed + with patch("bot.exts.moderation.incidents.download_file", AsyncMock(return_value=None)): + embed, returned_file = await incidents.make_embed(incident, incidents.Signal.ACTIONED, MockMember()) + + self.assertIsNone(returned_file) + + # The author name field is simply expected to have something in it, we do not assert the message + self.assertGreater(len(embed.author.name), 0) + self.assertEqual(embed.author.url, "discord.com/bigbadjoe.jpg") # However, it should link the exact url + + +@patch("bot.constants.Channels.incidents", 123) +class TestIsIncident(unittest.TestCase): + """ + Collection of tests for the `is_incident` helper function. + + In `setUp`, we will create a mock message which should qualify as an incident. Each + test case will then mutate this instance to make it **not** qualify, in various ways. + + Notice that we patch the #incidents channel id globally for this class. + """ + + def setUp(self) -> None: + """Prepare a mock message which should qualify as an incident.""" + self.incident = MockMessage( + channel=MockTextChannel(id=123), + content="this is an incident", + author=MockUser(bot=False), + pinned=False, + ) + + def test_is_incident_true(self): + """Message qualifies as an incident if unchanged.""" + self.assertTrue(incidents.is_incident(self.incident)) + + def check_false(self): + """Assert that `self.incident` does **not** qualify as an incident.""" + self.assertFalse(incidents.is_incident(self.incident)) + + def test_is_incident_false_channel(self): + """Message doesn't qualify if sent outside of #incidents.""" + self.incident.channel = MockTextChannel(id=456) + self.check_false() + + def test_is_incident_false_content(self): + """Message doesn't qualify if content begins with hash symbol.""" + self.incident.content = "# this is a comment message" + self.check_false() + + def test_is_incident_false_author(self): + """Message doesn't qualify if author is a bot.""" + self.incident.author = MockUser(bot=True) + self.check_false() + + def test_is_incident_false_pinned(self): + """Message doesn't qualify if it is pinned.""" + self.incident.pinned = True + self.check_false() + + +class TestOwnReactions(unittest.TestCase): + """Assertions for the `own_reactions` function.""" + + def test_own_reactions(self): + """Only bot's own emoji are extracted from the input incident.""" + reactions = ( + MockReaction(emoji="A", me=True), + MockReaction(emoji="B", me=True), + MockReaction(emoji="C", me=False), + ) + message = MockMessage(reactions=reactions) + self.assertSetEqual(incidents.own_reactions(message), {"A", "B"}) + + +@patch("bot.exts.moderation.incidents.ALL_SIGNALS", {"A", "B"}) +class TestHasSignals(unittest.TestCase): + """ + Assertions for the `has_signals` function. + + We patch `ALL_SIGNALS` globally. Each test function then patches `own_reactions` + as appropriate. + """ + + def test_has_signals_true(self): + """True when `own_reactions` returns all emoji in `ALL_SIGNALS`.""" + message = MockMessage() + own_reactions = MagicMock(return_value={"A", "B"}) + + with patch("bot.exts.moderation.incidents.own_reactions", own_reactions): + self.assertTrue(incidents.has_signals(message)) + + def test_has_signals_false(self): + """False when `own_reactions` does not return all emoji in `ALL_SIGNALS`.""" + message = MockMessage() + own_reactions = MagicMock(return_value={"A", "C"}) + + with patch("bot.exts.moderation.incidents.own_reactions", own_reactions): + self.assertFalse(incidents.has_signals(message)) + + +@patch("bot.exts.moderation.incidents.Signal", MockSignal) +class TestAddSignals(unittest.IsolatedAsyncioTestCase): + """ + Assertions for the `add_signals` coroutine. + + These are all fairly similar and could go into a single test function, but I found the + patching & sub-testing fairly awkward in that case and decided to split them up + to avoid unnecessary syntax noise. + """ + + def setUp(self): + """Prepare a mock incident message for tests to use.""" + self.incident = MockMessage() + + @patch("bot.exts.moderation.incidents.own_reactions", MagicMock(return_value=set())) + async def test_add_signals_missing(self): + """All emoji are added when none are present.""" + await incidents.add_signals(self.incident) + self.incident.add_reaction.assert_has_calls([call("A"), call("B")]) + + @patch("bot.exts.moderation.incidents.own_reactions", MagicMock(return_value={"A"})) + async def test_add_signals_partial(self): + """Only missing emoji are added when some are present.""" + await incidents.add_signals(self.incident) + self.incident.add_reaction.assert_has_calls([call("B")]) + + @patch("bot.exts.moderation.incidents.own_reactions", MagicMock(return_value={"A", "B"})) + async def test_add_signals_present(self): + """No emoji are added when all are present.""" + await incidents.add_signals(self.incident) + self.incident.add_reaction.assert_not_called() + + +class TestIncidents(unittest.IsolatedAsyncioTestCase): + """ + Tests for bound methods of the `Incidents` cog. + + Use this as a base class for `Incidents` tests - it will prepare a fresh instance + for each test function, but not make any assertions on its own. Tests can mutate + the instance as they wish. + """ + + def setUp(self): + """ + Prepare a fresh `Incidents` instance for each test. + + Note that this will not schedule `crawl_incidents` in the background, as everything + is being mocked. The `crawl_task` attribute will end up being None. + """ + self.cog_instance = incidents.Incidents(MockBot()) + + +@patch("asyncio.sleep", AsyncMock()) # Prevent the coro from sleeping to speed up the test +class TestCrawlIncidents(TestIncidents): + """ + Tests for the `Incidents.crawl_incidents` coroutine. + + Apart from `test_crawl_incidents_waits_until_cache_ready`, all tests in this class + will patch the return values of `is_incident` and `has_signal` and then observe + whether the `AsyncMock` for `add_signals` was awaited or not. + + The `add_signals` mock is added by each test separately to ensure it is clean (has not + been awaited by another test yet). The mock can be reset, but this appears to be the + cleaner way. + + For each test, we inject a mock channel with a history of 1 message only (see: `setUp`). + """ + + def setUp(self): + """For each test, ensure `bot.get_channel` returns a channel with 1 arbitrary message.""" + super().setUp() # First ensure we get `cog_instance` from parent + + incidents_history = MagicMock(return_value=MockAsyncIterable([MockMessage()])) + self.cog_instance.bot.get_channel = MagicMock(return_value=MockTextChannel(history=incidents_history)) + + async def test_crawl_incidents_waits_until_cache_ready(self): + """ + The coroutine will await the `wait_until_guild_available` event. + + Since this task is schedule in the `__init__`, it is critical that it waits for the + cache to be ready, so that it can safely get the #incidents channel. + """ + await self.cog_instance.crawl_incidents() + self.cog_instance.bot.wait_until_guild_available.assert_awaited() + + @patch("bot.exts.moderation.incidents.add_signals", AsyncMock()) + @patch("bot.exts.moderation.incidents.is_incident", MagicMock(return_value=False)) # Message doesn't qualify + @patch("bot.exts.moderation.incidents.has_signals", MagicMock(return_value=False)) + async def test_crawl_incidents_noop_if_is_not_incident(self): + """Signals are not added for a non-incident message.""" + await self.cog_instance.crawl_incidents() + incidents.add_signals.assert_not_awaited() + + @patch("bot.exts.moderation.incidents.add_signals", AsyncMock()) + @patch("bot.exts.moderation.incidents.is_incident", MagicMock(return_value=True)) # Message qualifies + @patch("bot.exts.moderation.incidents.has_signals", MagicMock(return_value=True)) # But already has signals + async def test_crawl_incidents_noop_if_message_already_has_signals(self): + """Signals are not added for messages which already have them.""" + await self.cog_instance.crawl_incidents() + incidents.add_signals.assert_not_awaited() + + @patch("bot.exts.moderation.incidents.add_signals", AsyncMock()) + @patch("bot.exts.moderation.incidents.is_incident", MagicMock(return_value=True)) # Message qualifies + @patch("bot.exts.moderation.incidents.has_signals", MagicMock(return_value=False)) # And doesn't have signals + async def test_crawl_incidents_add_signals_called(self): + """Message has signals added as it does not have them yet and qualifies as an incident.""" + await self.cog_instance.crawl_incidents() + incidents.add_signals.assert_awaited_once() + + +class TestArchive(TestIncidents): + """Tests for the `Incidents.archive` coroutine.""" + + async def test_archive_webhook_not_found(self): + """ + Method recovers and returns False when the webhook is not found. + + Implicitly, this also tests that the error is handled internally and doesn't + propagate out of the method, which is just as important. + """ + self.cog_instance.bot.fetch_webhook = AsyncMock(side_effect=mock_404) + self.assertFalse( + await self.cog_instance.archive(incident=MockMessage(), outcome=MagicMock(), actioned_by=MockMember()) + ) + + async def test_archive_relays_incident(self): + """ + If webhook is found, method relays `incident` properly. + + This test will assert that the fetched webhook's `send` method is fed the correct arguments, + and that the `archive` method returns True. + """ + webhook = MockAsyncWebhook() + self.cog_instance.bot.fetch_webhook = AsyncMock(return_value=webhook) # Patch in our webhook + + # Define our own `incident` to be archived + incident = MockMessage( + content="this is an incident", + author=MockUser(name="author_name", avatar_url="author_avatar"), + id=123, + ) + built_embed = MagicMock(discord.Embed, id=123) # We patch `make_embed` to return this + + with patch("bot.exts.moderation.incidents.make_embed", AsyncMock(return_value=(built_embed, None))): + archive_return = await self.cog_instance.archive(incident, MagicMock(value="A"), MockMember()) + + # Now we check that the webhook was given the correct args, and that `archive` returned True + webhook.send.assert_called_once_with( + embed=built_embed, + username="author_name", + avatar_url="author_avatar", + file=None, + ) + self.assertTrue(archive_return) + + async def test_archive_clyde_username(self): + """ + The archive webhook username is cleansed using `sub_clyde`. + + Discord will reject any webhook with "clyde" in the username field, as it impersonates + the official Clyde bot. Since we do not control what the username will be (the incident + author name is used), we must ensure the name is cleansed, otherwise the relay may fail. + + This test assumes the username is passed as a kwarg. If this test fails, please review + whether the passed argument is being retrieved correctly. + """ + webhook = MockAsyncWebhook() + self.cog_instance.bot.fetch_webhook = AsyncMock(return_value=webhook) + + message_from_clyde = MockMessage(author=MockUser(name="clyde the great")) + await self.cog_instance.archive(message_from_clyde, MagicMock(incidents.Signal), MockMember()) + + self.assertNotIn("clyde", webhook.send.call_args.kwargs["username"]) + + +class TestMakeConfirmationTask(TestIncidents): + """ + Tests for the `Incidents.make_confirmation_task` method. + + Writing tests for this method is difficult, as it mostly just delegates the provided + information elsewhere. There is very little internal logic. Whether our approach + works conceptually is difficult to prove using unit tests. + """ + + def test_make_confirmation_task_check(self): + """ + The internal check will recognize the passed incident. + + This is a little tricky - we first pass a message with a specific `id` in, and then + retrieve the built check from the `call_args` of the `wait_for` method. This relies + on the check being passed as a kwarg. + + Once the check is retrieved, we assert that it gives True for our incident's `id`, + and False for any other. + + If this function begins to fail, first check that `created_check` is being retrieved + correctly. It should be the function that is built locally in the tested method. + """ + self.cog_instance.make_confirmation_task(MockMessage(id=123)) + + self.cog_instance.bot.wait_for.assert_called_once() + created_check = self.cog_instance.bot.wait_for.call_args.kwargs["check"] + + # The `message_id` matches the `id` of our incident + self.assertTrue(created_check(payload=MagicMock(message_id=123))) + + # This `message_id` does not match + self.assertFalse(created_check(payload=MagicMock(message_id=0))) + + +@patch("bot.exts.moderation.incidents.ALLOWED_ROLES", {1, 2}) +@patch("bot.exts.moderation.incidents.Incidents.make_confirmation_task", AsyncMock()) # Generic awaitable +class TestProcessEvent(TestIncidents): + """Tests for the `Incidents.process_event` coroutine.""" + + async def test_process_event_bad_role(self): + """The reaction is removed when the author lacks all allowed roles.""" + incident = MockMessage() + member = MockMember(roles=[MockRole(id=0)]) # Must have role 1 or 2 + + await self.cog_instance.process_event("reaction", incident, member) + incident.remove_reaction.assert_called_once_with("reaction", member) + + async def test_process_event_bad_emoji(self): + """ + The reaction is removed when an invalid emoji is used. + + This requires that we pass in a `member` with valid roles, as we need the role check + to succeed. + """ + incident = MockMessage() + member = MockMember(roles=[MockRole(id=1)]) # Member has allowed role + + await self.cog_instance.process_event("invalid_signal", incident, member) + incident.remove_reaction.assert_called_once_with("invalid_signal", member) + + async def test_process_event_no_archive_on_investigating(self): + """Message is not archived on `Signal.INVESTIGATING`.""" + with patch("bot.exts.moderation.incidents.Incidents.archive", AsyncMock()) as mocked_archive: + await self.cog_instance.process_event( + reaction=incidents.Signal.INVESTIGATING.value, + incident=MockMessage(), + member=MockMember(roles=[MockRole(id=1)]), + ) + + mocked_archive.assert_not_called() + + async def test_process_event_no_delete_if_archive_fails(self): + """ + Original message is not deleted when `Incidents.archive` returns False. + + This is the way of signaling that the relay failed, and we should not remove the original, + as that would result in losing the incident record. + """ + incident = MockMessage() + + with patch("bot.exts.moderation.incidents.Incidents.archive", AsyncMock(return_value=False)): + await self.cog_instance.process_event( + reaction=incidents.Signal.ACTIONED.value, + incident=incident, + member=MockMember(roles=[MockRole(id=1)]) + ) + + incident.delete.assert_not_called() + + async def test_process_event_confirmation_task_is_awaited(self): + """Task given by `Incidents.make_confirmation_task` is awaited before method exits.""" + mock_task = AsyncMock() + + with patch("bot.exts.moderation.incidents.Incidents.make_confirmation_task", mock_task): + await self.cog_instance.process_event( + reaction=incidents.Signal.ACTIONED.value, + incident=MockMessage(), + member=MockMember(roles=[MockRole(id=1)]) + ) + + mock_task.assert_awaited() + + async def test_process_event_confirmation_task_timeout_is_handled(self): + """ + Confirmation task `asyncio.TimeoutError` is handled gracefully. + + We have `make_confirmation_task` return a mock with a side effect, and then catch the + exception should it propagate out of `process_event`. This is so that we can then manually + fail the test with a more informative message than just the plain traceback. + """ + mock_task = AsyncMock(side_effect=asyncio.TimeoutError()) + + try: + with patch("bot.exts.moderation.incidents.Incidents.make_confirmation_task", mock_task): + await self.cog_instance.process_event( + reaction=incidents.Signal.ACTIONED.value, + incident=MockMessage(), + member=MockMember(roles=[MockRole(id=1)]) + ) + except asyncio.TimeoutError: + self.fail("TimeoutError was not handled gracefully, and propagated out of `process_event`!") + + +class TestResolveMessage(TestIncidents): + """Tests for the `Incidents.resolve_message` coroutine.""" + + async def test_resolve_message_pass_message_id(self): + """Method will call `_get_message` with the passed `message_id`.""" + await self.cog_instance.resolve_message(123) + self.cog_instance.bot._connection._get_message.assert_called_once_with(123) + + async def test_resolve_message_in_cache(self): + """ + No API call is made if the queried message exists in the cache. + + We mock the `_get_message` return value regardless of input. Whether it finds the message + internally is considered d.py's responsibility, not ours. + """ + cached_message = MockMessage(id=123) + self.cog_instance.bot._connection._get_message = MagicMock(return_value=cached_message) + + return_value = await self.cog_instance.resolve_message(123) + + self.assertIs(return_value, cached_message) + self.cog_instance.bot.get_channel.assert_not_called() # The `fetch_message` line was never hit + + async def test_resolve_message_not_in_cache(self): + """ + The message is retrieved from the API if it isn't cached. + + This is desired behaviour for messages which exist, but were sent before the bot's + current session. + """ + self.cog_instance.bot._connection._get_message = MagicMock(return_value=None) # Cache returns None + + # API returns our message + uncached_message = MockMessage() + fetch_message = AsyncMock(return_value=uncached_message) + self.cog_instance.bot.get_channel = MagicMock(return_value=MockTextChannel(fetch_message=fetch_message)) + + retrieved_message = await self.cog_instance.resolve_message(123) + self.assertIs(retrieved_message, uncached_message) + + async def test_resolve_message_doesnt_exist(self): + """ + If the API returns a 404, the function handles it gracefully and returns None. + + This is an edge-case happening with racing events - event A will relay the message + to the archive and delete the original. Once event B acquires the `event_lock`, + it will not find the message in the cache, and will ask the API. + """ + self.cog_instance.bot._connection._get_message = MagicMock(return_value=None) # Cache returns None + + fetch_message = AsyncMock(side_effect=mock_404) + self.cog_instance.bot.get_channel = MagicMock(return_value=MockTextChannel(fetch_message=fetch_message)) + + self.assertIsNone(await self.cog_instance.resolve_message(123)) + + async def test_resolve_message_fetch_fails(self): + """ + Non-404 errors are handled, logged & None is returned. + + In contrast with a 404, this should make an error-level log. We assert that at least + one such log was made - we do not make any assertions about the log's message. + """ + self.cog_instance.bot._connection._get_message = MagicMock(return_value=None) # Cache returns None + + arbitrary_error = discord.HTTPException( + response=MagicMock(aiohttp.ClientResponse), + message="Arbitrary error", + ) + fetch_message = AsyncMock(side_effect=arbitrary_error) + self.cog_instance.bot.get_channel = MagicMock(return_value=MockTextChannel(fetch_message=fetch_message)) + + with self.assertLogs(logger=incidents.log, level=logging.ERROR): + self.assertIsNone(await self.cog_instance.resolve_message(123)) + + +@patch("bot.constants.Channels.incidents", 123) +class TestOnRawReactionAdd(TestIncidents): + """ + Tests for the `Incidents.on_raw_reaction_add` listener. + + Writing tests for this listener comes with additional complexity due to the listener + awaiting the `crawl_task` task. See `asyncSetUp` for further details, which attempts + to make unit testing this function possible. + """ + + def setUp(self): + """ + Prepare & assign `payload` attribute. + + This attribute represents an *ideal* payload which will not be rejected by the + listener. As each test will receive a fresh instance, it can be mutated to + observe how the listener's behaviour changes with different attributes on + the passed payload. + """ + super().setUp() # Ensure `cog_instance` is assigned + + self.payload = MagicMock( + discord.RawReactionActionEvent, + channel_id=123, # Patched at class level + message_id=456, + member=MockMember(bot=False), + emoji="reaction", + ) + + async def asyncSetUp(self): # noqa: N802 + """ + Prepare an empty task and assign it as `crawl_task`. + + It appears that the `unittest` framework does not provide anything for mocking + asyncio tasks. An `AsyncMock` instance can be called and then awaited, however, + it does not provide the `done` method or any other parts of the `asyncio.Task` + interface. + + Although we do not need to make any assertions about the task itself while + testing the listener, the code will still await it and call the `done` method, + and so we must inject something that will not fail on either action. + + Note that this is done in an `asyncSetUp`, which runs after `setUp`. + The justification is that creating an actual task requires the event + loop to be ready, which is not the case in the `setUp`. + """ + mock_task = asyncio.create_task(AsyncMock()()) # Mock async func, then a coro + self.cog_instance.crawl_task = mock_task + + async def test_on_raw_reaction_add_wrong_channel(self): + """ + Events outside of #incidents will be ignored. + + We check this by asserting that `resolve_message` was never queried. + """ + self.payload.channel_id = 0 + self.cog_instance.resolve_message = AsyncMock() + + await self.cog_instance.on_raw_reaction_add(self.payload) + self.cog_instance.resolve_message.assert_not_called() + + async def test_on_raw_reaction_add_user_is_bot(self): + """ + Events dispatched by bot accounts will be ignored. + + We check this by asserting that `resolve_message` was never queried. + """ + self.payload.member = MockMember(bot=True) + self.cog_instance.resolve_message = AsyncMock() + + await self.cog_instance.on_raw_reaction_add(self.payload) + self.cog_instance.resolve_message.assert_not_called() + + async def test_on_raw_reaction_add_message_doesnt_exist(self): + """ + Listener gracefully handles the case where `resolve_message` gives None. + + We check this by asserting that `process_event` was never called. + """ + self.cog_instance.process_event = AsyncMock() + self.cog_instance.resolve_message = AsyncMock(return_value=None) + + await self.cog_instance.on_raw_reaction_add(self.payload) + self.cog_instance.process_event.assert_not_called() + + async def test_on_raw_reaction_add_message_is_not_an_incident(self): + """ + The event won't be processed if the related message is not an incident. + + This is an edge-case that can happen if someone manually leaves a reaction + on a pinned message, or a comment. + + We check this by asserting that `process_event` was never called. + """ + self.cog_instance.process_event = AsyncMock() + self.cog_instance.resolve_message = AsyncMock(return_value=MockMessage()) + + with patch("bot.exts.moderation.incidents.is_incident", MagicMock(return_value=False)): + await self.cog_instance.on_raw_reaction_add(self.payload) + + self.cog_instance.process_event.assert_not_called() + + async def test_on_raw_reaction_add_valid_event_is_processed(self): + """ + If the reaction event is valid, it is passed to `process_event`. + + This is the case when everything goes right: + * The reaction was placed in #incidents, and not by a bot + * The message was found successfully + * The message qualifies as an incident + + Additionally, we check that all arguments were passed as expected. + """ + incident = MockMessage(id=1) + + self.cog_instance.process_event = AsyncMock() + self.cog_instance.resolve_message = AsyncMock(return_value=incident) + + with patch("bot.exts.moderation.incidents.is_incident", MagicMock(return_value=True)): + await self.cog_instance.on_raw_reaction_add(self.payload) + + self.cog_instance.process_event.assert_called_with( + "reaction", # Defined in `self.payload` + incident, + self.payload.member, + ) + + +class TestOnMessage(TestIncidents): + """ + Tests for the `Incidents.on_message` listener. + + Notice the decorators mocking the `is_incident` return value. The `is_incidents` + function is tested in `TestIsIncident` - here we do not worry about it. + """ + + @patch("bot.exts.moderation.incidents.is_incident", MagicMock(return_value=True)) + async def test_on_message_incident(self): + """Messages qualifying as incidents are passed to `add_signals`.""" + incident = MockMessage() + + with patch("bot.exts.moderation.incidents.add_signals", AsyncMock()) as mock_add_signals: + await self.cog_instance.on_message(incident) + + mock_add_signals.assert_called_once_with(incident) + + @patch("bot.exts.moderation.incidents.is_incident", MagicMock(return_value=False)) + async def test_on_message_non_incident(self): + """Messages not qualifying as incidents are ignored.""" + with patch("bot.exts.moderation.incidents.add_signals", AsyncMock()) as mock_add_signals: + await self.cog_instance.on_message(MockMessage()) + + mock_add_signals.assert_not_called() diff --git a/tests/bot/exts/moderation/test_modlog.py b/tests/bot/exts/moderation/test_modlog.py new file mode 100644 index 000000000..f8f142484 --- /dev/null +++ b/tests/bot/exts/moderation/test_modlog.py @@ -0,0 +1,29 @@ +import unittest + +import discord + +from bot.exts.moderation.modlog import ModLog +from tests.helpers import MockBot, MockTextChannel + + +class ModLogTests(unittest.IsolatedAsyncioTestCase): + """Tests for moderation logs.""" + + def setUp(self): + self.bot = MockBot() + self.cog = ModLog(self.bot) + self.channel = MockTextChannel() + + async def test_log_entry_description_truncation(self): + """Test that embed description for ModLog entry is truncated.""" + self.bot.get_channel.return_value = self.channel + await self.cog.send_log_message( + icon_url="foo", + colour=discord.Colour.blue(), + title="bar", + text="foo bar" * 3000 + ) + embed = self.channel.send.call_args[1]["embed"] + self.assertEqual( + embed.description, ("foo bar" * 3000)[:2045] + "..." + ) diff --git a/tests/bot/cogs/moderation/test_silence.py b/tests/bot/exts/moderation/test_silence.py index 52b7d47f1..3c2d52ae0 100644 --- a/tests/bot/cogs/moderation/test_silence.py +++ b/tests/bot/exts/moderation/test_silence.py @@ -4,8 +4,8 @@ from unittest.mock import MagicMock, Mock from discord import PermissionOverwrite -from bot.cogs.moderation.silence import Silence, SilenceNotifier from bot.constants import Channels, Emojis, Guild, Roles +from bot.exts.moderation.silence import Silence, SilenceNotifier from tests.helpers import MockBot, MockContext, MockTextChannel @@ -99,7 +99,7 @@ class SilenceTests(unittest.IsolatedAsyncioTestCase): self.bot.get_channel.called_once_with(Channels.mod_alerts) self.bot.get_channel.called_once_with(Channels.mod_log) - @mock.patch("bot.cogs.moderation.silence.SilenceNotifier") + @mock.patch("bot.exts.moderation.silence.SilenceNotifier") async def test_instance_vars_got_notifier(self, notifier): """Notifier was started with channel.""" mod_log = MockTextChannel() @@ -127,10 +127,20 @@ class SilenceTests(unittest.IsolatedAsyncioTestCase): self.ctx.reset_mock() async def test_unsilence_sent_correct_discord_message(self): - """Proper reply after a successful unsilence.""" - with mock.patch.object(self.cog, "_unsilence", return_value=True): - await self.cog.unsilence(self.cog, self.ctx) - self.ctx.send.assert_called_once_with(f"{Emojis.check_mark} unsilenced current channel.") + """Check if proper message was sent when unsilencing channel.""" + test_cases = ( + (True, f"{Emojis.check_mark} unsilenced current channel."), + (False, f"{Emojis.cross_mark} current channel was not silenced.") + ) + for _unsilence_patch_return, result_message in test_cases: + with self.subTest( + starting_silenced_state=_unsilence_patch_return, + result_message=result_message + ): + with mock.patch.object(self.cog, "_unsilence", return_value=_unsilence_patch_return): + await self.cog.unsilence(self.cog, self.ctx) + self.ctx.send.assert_called_once_with(result_message) + self.ctx.reset_mock() async def test_silence_private_for_false(self): """Permissions are not set and `False` is returned in an already silenced channel.""" @@ -228,7 +238,7 @@ class SilenceTests(unittest.IsolatedAsyncioTestCase): del mock_permissions_dict['send_messages'] self.assertDictEqual(mock_permissions_dict, new_permissions) - @mock.patch("bot.cogs.moderation.silence.asyncio") + @mock.patch("bot.exts.moderation.silence.asyncio") @mock.patch.object(Silence, "_mod_alerts_channel", create=True) def test_cog_unload_starts_task(self, alert_channel, asyncio_mock): """Task for sending an alert was created with present `muted_channels`.""" @@ -237,15 +247,17 @@ class SilenceTests(unittest.IsolatedAsyncioTestCase): alert_channel.send.assert_called_once_with(f"<@&{Roles.moderators}> channels left silenced on cog unload: ") asyncio_mock.create_task.assert_called_once_with(alert_channel.send()) - @mock.patch("bot.cogs.moderation.silence.asyncio") + @mock.patch("bot.exts.moderation.silence.asyncio") def test_cog_unload_skips_task_start(self, asyncio_mock): """No task created with no channels.""" self.cog.cog_unload() asyncio_mock.create_task.assert_not_called() - @mock.patch("bot.cogs.moderation.silence.with_role_check") - @mock.patch("bot.cogs.moderation.silence.MODERATION_ROLES", new=(1, 2, 3)) - def test_cog_check(self, role_check): + @mock.patch("discord.ext.commands.has_any_role") + @mock.patch("bot.exts.moderation.silence.MODERATION_ROLES", new=(1, 2, 3)) + async def test_cog_check(self, role_check): """Role check is called with `MODERATION_ROLES`""" - self.cog.cog_check(self.ctx) - role_check.assert_called_once_with(self.ctx, *(1, 2, 3)) + role_check.return_value.predicate = mock.AsyncMock() + await self.cog.cog_check(self.ctx) + role_check.assert_called_once_with(*(1, 2, 3)) + role_check.return_value.predicate.assert_awaited_once_with(self.ctx) diff --git a/tests/bot/exts/moderation/test_slowmode.py b/tests/bot/exts/moderation/test_slowmode.py new file mode 100644 index 000000000..dad751e0d --- /dev/null +++ b/tests/bot/exts/moderation/test_slowmode.py @@ -0,0 +1,113 @@ +import unittest +from unittest import mock + +from dateutil.relativedelta import relativedelta + +from bot.constants import Emojis +from bot.exts.moderation.slowmode import Slowmode +from tests.helpers import MockBot, MockContext, MockTextChannel + + +class SlowmodeTests(unittest.IsolatedAsyncioTestCase): + + def setUp(self) -> None: + self.bot = MockBot() + self.cog = Slowmode(self.bot) + self.ctx = MockContext() + + async def test_get_slowmode_no_channel(self) -> None: + """Get slowmode without a given channel.""" + self.ctx.channel = MockTextChannel(name='python-general', slowmode_delay=5) + + await self.cog.get_slowmode(self.cog, self.ctx, None) + self.ctx.send.assert_called_once_with("The slowmode delay for #python-general is 5 seconds.") + + async def test_get_slowmode_with_channel(self) -> None: + """Get slowmode with a given channel.""" + text_channel = MockTextChannel(name='python-language', slowmode_delay=2) + + await self.cog.get_slowmode(self.cog, self.ctx, text_channel) + self.ctx.send.assert_called_once_with('The slowmode delay for #python-language is 2 seconds.') + + async def test_set_slowmode_no_channel(self) -> None: + """Set slowmode without a given channel.""" + test_cases = ( + ('helpers', 23, True, f'{Emojis.check_mark} The slowmode delay for #helpers is now 23 seconds.'), + ('mods', 76526, False, f'{Emojis.cross_mark} The slowmode delay must be between 0 and 6 hours.'), + ('admins', 97, True, f'{Emojis.check_mark} The slowmode delay for #admins is now 1 minute and 37 seconds.') + ) + + for channel_name, seconds, edited, result_msg in test_cases: + with self.subTest( + channel_mention=channel_name, + seconds=seconds, + edited=edited, + result_msg=result_msg + ): + self.ctx.channel = MockTextChannel(name=channel_name) + + await self.cog.set_slowmode(self.cog, self.ctx, None, relativedelta(seconds=seconds)) + + if edited: + self.ctx.channel.edit.assert_awaited_once_with(slowmode_delay=float(seconds)) + else: + self.ctx.channel.edit.assert_not_called() + + self.ctx.send.assert_called_once_with(result_msg) + + self.ctx.reset_mock() + + async def test_set_slowmode_with_channel(self) -> None: + """Set slowmode with a given channel.""" + test_cases = ( + ('bot-commands', 12, True, f'{Emojis.check_mark} The slowmode delay for #bot-commands is now 12 seconds.'), + ('mod-spam', 21, True, f'{Emojis.check_mark} The slowmode delay for #mod-spam is now 21 seconds.'), + ('admin-spam', 4323598, False, f'{Emojis.cross_mark} The slowmode delay must be between 0 and 6 hours.') + ) + + for channel_name, seconds, edited, result_msg in test_cases: + with self.subTest( + channel_mention=channel_name, + seconds=seconds, + edited=edited, + result_msg=result_msg + ): + text_channel = MockTextChannel(name=channel_name) + + await self.cog.set_slowmode(self.cog, self.ctx, text_channel, relativedelta(seconds=seconds)) + + if edited: + text_channel.edit.assert_awaited_once_with(slowmode_delay=float(seconds)) + else: + text_channel.edit.assert_not_called() + + self.ctx.send.assert_called_once_with(result_msg) + + self.ctx.reset_mock() + + async def test_reset_slowmode_no_channel(self) -> None: + """Reset slowmode without a given channel.""" + self.ctx.channel = MockTextChannel(name='careers', slowmode_delay=6) + + await self.cog.reset_slowmode(self.cog, self.ctx, None) + self.ctx.send.assert_called_once_with( + f'{Emojis.check_mark} The slowmode delay for #careers has been reset to 0 seconds.' + ) + + async def test_reset_slowmode_with_channel(self) -> None: + """Reset slowmode with a given channel.""" + text_channel = MockTextChannel(name='meta', slowmode_delay=1) + + await self.cog.reset_slowmode(self.cog, self.ctx, text_channel) + self.ctx.send.assert_called_once_with( + f'{Emojis.check_mark} The slowmode delay for #meta has been reset to 0 seconds.' + ) + + @mock.patch("bot.exts.moderation.slowmode.has_any_role") + @mock.patch("bot.exts.moderation.slowmode.MODERATION_ROLES", new=(1, 2, 3)) + async def test_cog_check(self, role_check): + """Role check is called with `MODERATION_ROLES`""" + role_check.return_value.predicate = mock.AsyncMock() + await self.cog.cog_check(self.ctx) + role_check.assert_called_once_with(*(1, 2, 3)) + role_check.return_value.predicate.assert_awaited_once_with(self.ctx) diff --git a/tests/bot/cogs/test_cogs.py b/tests/bot/exts/test_cogs.py index 39f6492cb..f8e120262 100644 --- a/tests/bot/cogs/test_cogs.py +++ b/tests/bot/exts/test_cogs.py @@ -10,7 +10,7 @@ from unittest import mock from discord.ext import commands -from bot import cogs +from bot import exts class CommandNameTests(unittest.TestCase): @@ -29,13 +29,14 @@ class CommandNameTests(unittest.TestCase): @staticmethod def walk_modules() -> t.Iterator[ModuleType]: - """Yield imported modules from the bot.cogs subpackage.""" + """Yield imported modules from the bot.exts subpackage.""" def on_error(name: str) -> t.NoReturn: - raise ImportError(name=name) + raise ImportError(name=name) # pragma: no cover # The mock prevents asyncio.get_event_loop() from being called. with mock.patch("discord.ext.tasks.loop"): - for module in pkgutil.walk_packages(cogs.__path__, "bot.cogs.", onerror=on_error): + prefix = f"{exts.__name__}." + for module in pkgutil.walk_packages(exts.__path__, prefix, onerror=on_error): if not module.ispkg: yield importlib.import_module(module.name) @@ -53,6 +54,7 @@ class CommandNameTests(unittest.TestCase): """Return a list of all qualified names, including aliases, for the `command`.""" names = [f"{command.full_parent_name} {alias}".strip() for alias in command.aliases] names.append(command.qualified_name) + names += getattr(command, "root_aliases", []) return names @@ -71,7 +73,7 @@ class CommandNameTests(unittest.TestCase): for name in self.get_qualified_names(cmd): with self.subTest(cmd=func_name, name=name): - if name in all_names: + if name in all_names: # pragma: no cover conflicts = ", ".join(all_names.get(name, "")) self.fail( f"Name '{name}' of the command {func_name} conflicts with {conflicts}." diff --git a/tests/bot/exts/utils/__init__.py b/tests/bot/exts/utils/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/bot/exts/utils/__init__.py diff --git a/tests/bot/exts/utils/test_jams.py b/tests/bot/exts/utils/test_jams.py new file mode 100644 index 000000000..45e7b5b51 --- /dev/null +++ b/tests/bot/exts/utils/test_jams.py @@ -0,0 +1,173 @@ +import unittest +from unittest.mock import AsyncMock, MagicMock, create_autospec + +from discord import CategoryChannel + +from bot.constants import Roles +from bot.exts.utils import jams +from tests.helpers import MockBot, MockContext, MockGuild, MockMember, MockRole, MockTextChannel + + +def get_mock_category(channel_count: int, name: str) -> CategoryChannel: + """Return a mocked code jam category.""" + category = create_autospec(CategoryChannel, spec_set=True, instance=True) + category.name = name + category.channels = [MockTextChannel() for _ in range(channel_count)] + + return category + + +class JamCreateTeamTests(unittest.IsolatedAsyncioTestCase): + """Tests for `createteam` command.""" + + def setUp(self): + self.bot = MockBot() + self.admin_role = MockRole(name="Admins", id=Roles.admins) + self.command_user = MockMember([self.admin_role]) + self.guild = MockGuild([self.admin_role]) + self.ctx = MockContext(bot=self.bot, author=self.command_user, guild=self.guild) + self.cog = jams.CodeJams(self.bot) + + async def test_too_small_amount_of_team_members_passed(self): + """Should `ctx.send` and exit early when too small amount of members.""" + for case in (1, 2): + with self.subTest(amount_of_members=case): + self.cog.create_channels = AsyncMock() + self.cog.add_roles = AsyncMock() + + self.ctx.reset_mock() + members = (MockMember() for _ in range(case)) + await self.cog.createteam(self.cog, self.ctx, "foo", members) + + self.ctx.send.assert_awaited_once() + self.cog.create_channels.assert_not_awaited() + self.cog.add_roles.assert_not_awaited() + + async def test_duplicate_members_provided(self): + """Should `ctx.send` and exit early because duplicate members provided and total there is only 1 member.""" + self.cog.create_channels = AsyncMock() + self.cog.add_roles = AsyncMock() + + member = MockMember() + await self.cog.createteam(self.cog, self.ctx, "foo", (member for _ in range(5))) + + self.ctx.send.assert_awaited_once() + self.cog.create_channels.assert_not_awaited() + self.cog.add_roles.assert_not_awaited() + + async def test_result_sending(self): + """Should call `ctx.send` when everything goes right.""" + self.cog.create_channels = AsyncMock() + self.cog.add_roles = AsyncMock() + + members = [MockMember() for _ in range(5)] + await self.cog.createteam(self.cog, self.ctx, "foo", members) + + self.cog.create_channels.assert_awaited_once() + self.cog.add_roles.assert_awaited_once() + self.ctx.send.assert_awaited_once() + + async def test_category_doesnt_exist(self): + """Should create a new code jam category.""" + subtests = ( + [], + [get_mock_category(jams.MAX_CHANNELS - 1, jams.CATEGORY_NAME)], + [get_mock_category(jams.MAX_CHANNELS - 2, "other")], + ) + + for categories in subtests: + self.guild.reset_mock() + self.guild.categories = categories + + with self.subTest(categories=categories): + actual_category = await self.cog.get_category(self.guild) + + self.guild.create_category_channel.assert_awaited_once() + category_overwrites = self.guild.create_category_channel.call_args[1]["overwrites"] + + self.assertFalse(category_overwrites[self.guild.default_role].read_messages) + self.assertTrue(category_overwrites[self.guild.me].read_messages) + self.assertEqual(self.guild.create_category_channel.return_value, actual_category) + + async def test_category_channel_exist(self): + """Should not try to create category channel.""" + expected_category = get_mock_category(jams.MAX_CHANNELS - 2, jams.CATEGORY_NAME) + self.guild.categories = [ + get_mock_category(jams.MAX_CHANNELS - 2, "other"), + expected_category, + get_mock_category(0, jams.CATEGORY_NAME), + ] + + actual_category = await self.cog.get_category(self.guild) + self.assertEqual(expected_category, actual_category) + + async def test_channel_overwrites(self): + """Should have correct permission overwrites for users and roles.""" + leader = MockMember() + members = [leader] + [MockMember() for _ in range(4)] + overwrites = self.cog.get_overwrites(members, self.guild) + + # Leader permission overwrites + self.assertTrue(overwrites[leader].manage_messages) + self.assertTrue(overwrites[leader].read_messages) + self.assertTrue(overwrites[leader].manage_webhooks) + self.assertTrue(overwrites[leader].connect) + + # Other members permission overwrites + for member in members[1:]: + self.assertTrue(overwrites[member].read_messages) + self.assertTrue(overwrites[member].connect) + + # Everyone and verified role overwrite + self.assertFalse(overwrites[self.guild.default_role].read_messages) + self.assertFalse(overwrites[self.guild.default_role].connect) + self.assertFalse(overwrites[self.guild.get_role(Roles.verified)].read_messages) + self.assertFalse(overwrites[self.guild.get_role(Roles.verified)].connect) + + async def test_team_channels_creation(self): + """Should create new voice and text channel for team.""" + members = [MockMember() for _ in range(5)] + + self.cog.get_overwrites = MagicMock() + self.cog.get_category = AsyncMock() + self.ctx.guild.create_text_channel.return_value = MockTextChannel(mention="foobar-channel") + actual = await self.cog.create_channels(self.guild, "my-team", members) + + self.assertEqual("foobar-channel", actual) + self.cog.get_overwrites.assert_called_once_with(members, self.guild) + self.cog.get_category.assert_awaited_once_with(self.guild) + + self.guild.create_text_channel.assert_awaited_once_with( + "my-team", + overwrites=self.cog.get_overwrites.return_value, + category=self.cog.get_category.return_value + ) + self.guild.create_voice_channel.assert_awaited_once_with( + "My Team", + overwrites=self.cog.get_overwrites.return_value, + category=self.cog.get_category.return_value + ) + + async def test_jam_roles_adding(self): + """Should add team leader role to leader and jam role to every team member.""" + leader_role = MockRole(name="Team Leader") + jam_role = MockRole(name="Jammer") + self.guild.get_role.side_effect = [leader_role, jam_role] + + leader = MockMember() + members = [leader] + [MockMember() for _ in range(4)] + await self.cog.add_roles(self.guild, members) + + leader.add_roles.assert_any_await(leader_role) + for member in members: + member.add_roles.assert_any_await(jam_role) + + +class CodeJamSetup(unittest.TestCase): + """Test for `setup` function of `CodeJam` cog.""" + + def test_setup(self): + """Should call `bot.add_cog`.""" + bot = MockBot() + jams.setup(bot) + bot.add_cog.assert_called_once() diff --git a/tests/bot/cogs/test_snekbox.py b/tests/bot/exts/utils/test_snekbox.py index bcb3550f8..6601fad2c 100644 --- a/tests/bot/cogs/test_snekbox.py +++ b/tests/bot/exts/utils/test_snekbox.py @@ -1,13 +1,12 @@ import asyncio -import logging import unittest from unittest.mock import AsyncMock, MagicMock, Mock, call, create_autospec, patch from discord.ext import commands from bot import constants -from bot.cogs import snekbox -from bot.cogs.snekbox import Snekbox +from bot.exts.utils import snekbox +from bot.exts.utils.snekbox import Snekbox from tests.helpers import MockBot, MockContext, MockMessage, MockReaction, MockUser @@ -21,7 +20,10 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): """Post the eval code to the URLs.snekbox_eval_api endpoint.""" resp = MagicMock() resp.json = AsyncMock(return_value="return") - self.bot.http_session.post().__aenter__.return_value = resp + + context_manager = MagicMock() + context_manager.__aenter__.return_value = resp + self.bot.http_session.post.return_value = context_manager self.assertEqual(await self.cog.post_eval("import random"), "return") self.bot.http_session.post.assert_called_with( @@ -36,37 +38,14 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): result = await self.cog.upload_output("-" * (snekbox.MAX_PASTE_LEN + 1)) self.assertEqual(result, "too long to upload") - async def test_upload_output(self): + @patch("bot.exts.utils.snekbox.send_to_paste_service") + async def test_upload_output(self, mock_paste_util): """Upload the eval output to the URLs.paste_service.format(key="documents") endpoint.""" - key = "MarkDiamond" - resp = MagicMock() - resp.json = AsyncMock(return_value={"key": key}) - self.bot.http_session.post().__aenter__.return_value = resp - - self.assertEqual( - await self.cog.upload_output("My awesome output"), - constants.URLs.paste_service.format(key=key) - ) - self.bot.http_session.post.assert_called_with( - constants.URLs.paste_service.format(key="documents"), - data="My awesome output", - raise_for_status=True + await self.cog.upload_output("Test output.") + mock_paste_util.assert_called_once_with( + self.bot.http_session, "Test output.", extension="txt" ) - async def test_upload_output_gracefully_fallback_if_exception_during_request(self): - """Output upload gracefully fallback if the upload fail.""" - resp = MagicMock() - resp.json = AsyncMock(side_effect=Exception) - self.bot.http_session.post().__aenter__.return_value = resp - - log = logging.getLogger("bot.cogs.snekbox") - with self.assertLogs(logger=log, level='ERROR'): - await self.cog.upload_output('My awesome output!') - - async def test_upload_output_gracefully_fallback_if_no_key_in_response(self): - """Output upload gracefully fallback if there is no key entry in the response body.""" - self.assertEqual((await self.cog.upload_output('My awesome output!')), None) - def test_prepare_input(self): cases = ( ('print("Hello world!")', 'print("Hello world!")', 'non-formatted'), @@ -90,14 +69,14 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): actual = self.cog.get_results_message({'stdout': stdout, 'returncode': returncode}) self.assertEqual(actual, expected) - @patch('bot.cogs.snekbox.Signals', side_effect=ValueError) + @patch('bot.exts.utils.snekbox.Signals', side_effect=ValueError) def test_get_results_message_invalid_signal(self, mock_signals: Mock): self.assertEqual( self.cog.get_results_message({'stdout': '', 'returncode': 127}), ('Your eval job has completed with return code 127', '') ) - @patch('bot.cogs.snekbox.Signals') + @patch('bot.exts.utils.snekbox.Signals') def test_get_results_message_valid_signal(self, mock_signals: Mock): mock_signals.return_value.name = 'SIGTEST' self.assertEqual( @@ -138,12 +117,12 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): ('<!@', ("<!@\u200B", None), r'Convert <!@ to <!@\u200B'), ( '\u202E\u202E\u202E', - ('Code block escape attempt detected; will not output result', None), + ('Code block escape attempt detected; will not output result', 'https://testificate.com/'), 'Detect RIGHT-TO-LEFT OVERRIDE' ), ( '\u200B\u200B\u200B', - ('Code block escape attempt detected; will not output result', None), + ('Code block escape attempt detected; will not output result', 'https://testificate.com/'), 'Detect ZERO WIDTH SPACE' ), ('long\nbeard', ('001 | long\n002 | beard', None), 'Two line output'), @@ -208,10 +187,9 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): async def test_eval_command_call_help(self): """Test if the eval command call the help command if no code is provided.""" - ctx = MockContext() - ctx.invoke = AsyncMock() + ctx = MockContext(command="sentinel") await self.cog.eval_command(self.cog, ctx=ctx, code='') - ctx.send_help.assert_called_once_with("eval") + ctx.send_help.assert_called_once_with(ctx.command) async def test_send_eval(self): """Test the send_eval function.""" @@ -225,9 +203,13 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): self.cog.get_status_emoji = MagicMock(return_value=':yay!:') self.cog.format_output = AsyncMock(return_value=('[No output]', None)) + mocked_filter_cog = MagicMock() + mocked_filter_cog.filter_eval = AsyncMock(return_value=False) + self.bot.get_cog.return_value = mocked_filter_cog + await self.cog.send_eval(ctx, 'MyAwesomeCode') ctx.send.assert_called_once_with( - '@LemonLemonishBeard#0042 :yay!: Return code 0.\n\n```py\n[No output]\n```' + '@LemonLemonishBeard#0042 :yay!: Return code 0.\n\n```\n[No output]\n```' ) self.cog.post_eval.assert_called_once_with('MyAwesomeCode') self.cog.get_status_emoji.assert_called_once_with({'stdout': '', 'returncode': 0}) @@ -246,10 +228,14 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): self.cog.get_status_emoji = MagicMock(return_value=':yay!:') self.cog.format_output = AsyncMock(return_value=('Way too long beard', 'lookatmybeard.com')) + mocked_filter_cog = MagicMock() + mocked_filter_cog.filter_eval = AsyncMock(return_value=False) + self.bot.get_cog.return_value = mocked_filter_cog + await self.cog.send_eval(ctx, 'MyAwesomeCode') ctx.send.assert_called_once_with( '@LemonLemonishBeard#0042 :yay!: Return code 0.' - '\n\n```py\nWay too long beard\n```\nFull output: lookatmybeard.com' + '\n\n```\nWay too long beard\n```\nFull output: lookatmybeard.com' ) self.cog.post_eval.assert_called_once_with('MyAwesomeCode') self.cog.get_status_emoji.assert_called_once_with({'stdout': 'Way too long beard', 'returncode': 0}) @@ -267,16 +253,20 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): self.cog.get_status_emoji = MagicMock(return_value=':nope!:') self.cog.format_output = AsyncMock() # This function isn't called + mocked_filter_cog = MagicMock() + mocked_filter_cog.filter_eval = AsyncMock(return_value=False) + self.bot.get_cog.return_value = mocked_filter_cog + await self.cog.send_eval(ctx, 'MyAwesomeCode') ctx.send.assert_called_once_with( - '@LemonLemonishBeard#0042 :nope!: Return code 127.\n\n```py\nBeard got stuck in the eval\n```' + '@LemonLemonishBeard#0042 :nope!: Return code 127.\n\n```\nBeard got stuck in the eval\n```' ) self.cog.post_eval.assert_called_once_with('MyAwesomeCode') self.cog.get_status_emoji.assert_called_once_with({'stdout': 'ERROR', 'returncode': 127}) self.cog.get_results_message.assert_called_once_with({'stdout': 'ERROR', 'returncode': 127}) self.cog.format_output.assert_not_called() - @patch("bot.cogs.snekbox.partial") + @patch("bot.exts.utils.snekbox.partial") async def test_continue_eval_does_continue(self, partial_mock): """Test that the continue_eval function does continue if required conditions are met.""" ctx = MockContext(message=MockMessage(add_reaction=AsyncMock(), clear_reactions=AsyncMock())) @@ -291,7 +281,11 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(actual, expected) self.bot.wait_for.assert_has_awaits( ( - call('message_edit', check=partial_mock(snekbox.predicate_eval_message_edit, ctx), timeout=10), + call( + 'message_edit', + check=partial_mock(snekbox.predicate_eval_message_edit, ctx), + timeout=snekbox.REEVAL_TIMEOUT, + ), call('reaction_add', check=partial_mock(snekbox.predicate_eval_emoji_reaction, ctx), timeout=10) ) ) diff --git a/tests/bot/test_constants.py b/tests/bot/test_constants.py index dae7c066c..f10d6fbe8 100644 --- a/tests/bot/test_constants.py +++ b/tests/bot/test_constants.py @@ -1,14 +1,40 @@ import inspect +import typing import unittest from bot import constants +def is_annotation_instance(value: typing.Any, annotation: typing.Any) -> bool: + """ + Return True if `value` is an instance of the type represented by `annotation`. + + This doesn't account for things like Unions or checking for homogenous types in collections. + """ + origin = typing.get_origin(annotation) + + # This is done in case a bare e.g. `typing.List` is used. + # In such case, for the assertion to pass, the type needs to be normalised to e.g. `list`. + # `get_origin()` does this normalisation for us. + type_ = annotation if origin is None else origin + + return isinstance(value, type_) + + +def is_any_instance(value: typing.Any, types: typing.Collection) -> bool: + """Return True if `value` is an instance of any type in `types`.""" + for type_ in types: + if is_annotation_instance(value, type_): + return True + + return False + + class ConstantsTests(unittest.TestCase): """Tests for our constants.""" def test_section_configuration_matches_type_specification(self): - """The section annotations should match the actual types of the sections.""" + """"The section annotations should match the actual types of the sections.""" sections = ( cls @@ -17,10 +43,15 @@ class ConstantsTests(unittest.TestCase): ) for section in sections: for name, annotation in section.__annotations__.items(): - with self.subTest(section=section, name=name, annotation=annotation): + with self.subTest(section=section.__name__, name=name, annotation=annotation): value = getattr(section, name) + origin = typing.get_origin(annotation) + annotation_args = typing.get_args(annotation) + failure_msg = f"{value} is not an instance of {annotation}" - if getattr(annotation, '_name', None) in ('Dict', 'List'): - self.skipTest("Cannot validate containers yet.") - - self.assertIsInstance(value, annotation) + if origin is typing.Union: + is_instance = is_any_instance(value, annotation_args) + self.assertTrue(is_instance, failure_msg) + else: + is_instance = is_annotation_instance(value, annotation) + self.assertTrue(is_instance, failure_msg) diff --git a/tests/bot/test_converters.py b/tests/bot/test_converters.py index ca8cb6825..c42111f3f 100644 --- a/tests/bot/test_converters.py +++ b/tests/bot/test_converters.py @@ -1,5 +1,5 @@ -import asyncio import datetime +import re import unittest from unittest.mock import MagicMock, patch @@ -16,7 +16,7 @@ from bot.converters import ( ) -class ConverterTests(unittest.TestCase): +class ConverterTests(unittest.IsolatedAsyncioTestCase): """Tests our custom argument converters.""" @classmethod @@ -26,7 +26,7 @@ class ConverterTests(unittest.TestCase): cls.fixed_utc_now = datetime.datetime.fromisoformat('2019-01-01T00:00:00') - def test_tag_content_converter_for_valid(self): + async def test_tag_content_converter_for_valid(self): """TagContentConverter should return correct values for valid input.""" test_values = ( ('hello', 'hello'), @@ -35,10 +35,10 @@ class ConverterTests(unittest.TestCase): for content, expected_conversion in test_values: with self.subTest(content=content, expected_conversion=expected_conversion): - conversion = asyncio.run(TagContentConverter.convert(self.context, content)) + conversion = await TagContentConverter.convert(self.context, content) self.assertEqual(conversion, expected_conversion) - def test_tag_content_converter_for_invalid(self): + async def test_tag_content_converter_for_invalid(self): """TagContentConverter should raise the proper exception for invalid input.""" test_values = ( ('', "Tag contents should not be empty, or filled with whitespace."), @@ -47,10 +47,10 @@ class ConverterTests(unittest.TestCase): for value, exception_message in test_values: with self.subTest(tag_content=value, exception_message=exception_message): - with self.assertRaises(BadArgument, msg=exception_message): - asyncio.run(TagContentConverter.convert(self.context, value)) + with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): + await TagContentConverter.convert(self.context, value) - def test_tag_name_converter_for_valid(self): + async def test_tag_name_converter_for_valid(self): """TagNameConverter should return the correct values for valid tag names.""" test_values = ( ('tracebacks', 'tracebacks'), @@ -60,10 +60,10 @@ class ConverterTests(unittest.TestCase): for name, expected_conversion in test_values: with self.subTest(name=name, expected_conversion=expected_conversion): - conversion = asyncio.run(TagNameConverter.convert(self.context, name)) + conversion = await TagNameConverter.convert(self.context, name) self.assertEqual(conversion, expected_conversion) - def test_tag_name_converter_for_invalid(self): + async def test_tag_name_converter_for_invalid(self): """TagNameConverter should raise the correct exception for invalid tag names.""" test_values = ( ('👋', "Don't be ridiculous, you can't use that character!"), @@ -75,29 +75,29 @@ class ConverterTests(unittest.TestCase): for invalid_name, exception_message in test_values: with self.subTest(invalid_name=invalid_name, exception_message=exception_message): - with self.assertRaises(BadArgument, msg=exception_message): - asyncio.run(TagNameConverter.convert(self.context, invalid_name)) + with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): + await TagNameConverter.convert(self.context, invalid_name) - def test_valid_python_identifier_for_valid(self): + async def test_valid_python_identifier_for_valid(self): """ValidPythonIdentifier returns valid identifiers unchanged.""" test_values = ('foo', 'lemon') for name in test_values: with self.subTest(identifier=name): - conversion = asyncio.run(ValidPythonIdentifier.convert(self.context, name)) + conversion = await ValidPythonIdentifier.convert(self.context, name) self.assertEqual(name, conversion) - def test_valid_python_identifier_for_invalid(self): + async def test_valid_python_identifier_for_invalid(self): """ValidPythonIdentifier raises the proper exception for invalid identifiers.""" test_values = ('nested.stuff', '#####') for name in test_values: with self.subTest(identifier=name): exception_message = f'`{name}` is not a valid Python identifier' - with self.assertRaises(BadArgument, msg=exception_message): - asyncio.run(ValidPythonIdentifier.convert(self.context, name)) + with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): + await ValidPythonIdentifier.convert(self.context, name) - def test_duration_converter_for_valid(self): + async def test_duration_converter_for_valid(self): """Duration returns the correct `datetime` for valid duration strings.""" test_values = ( # Simple duration strings @@ -159,35 +159,35 @@ class ConverterTests(unittest.TestCase): mock_datetime.utcnow.return_value = self.fixed_utc_now with self.subTest(duration=duration, duration_dict=duration_dict): - converted_datetime = asyncio.run(converter.convert(self.context, duration)) + converted_datetime = await converter.convert(self.context, duration) self.assertEqual(converted_datetime, expected_datetime) - def test_duration_converter_for_invalid(self): + async def test_duration_converter_for_invalid(self): """Duration raises the right exception for invalid duration strings.""" test_values = ( # Units in wrong order - ('1d1w'), - ('1s1y'), + '1d1w', + '1s1y', # Duplicated units - ('1 year 2 years'), - ('1 M 10 minutes'), + '1 year 2 years', + '1 M 10 minutes', # Unknown substrings - ('1MVes'), - ('1y3breads'), + '1MVes', + '1y3breads', # Missing amount - ('ym'), + 'ym', # Incorrect whitespace - (" 1y"), - ("1S "), - ("1y 1m"), + " 1y", + "1S ", + "1y 1m", # Garbage - ('Guido van Rossum'), - ('lemon lemon lemon lemon lemon lemon lemon'), + 'Guido van Rossum', + 'lemon lemon lemon lemon lemon lemon lemon', ) converter = Duration() @@ -195,10 +195,21 @@ class ConverterTests(unittest.TestCase): for invalid_duration in test_values: with self.subTest(invalid_duration=invalid_duration): exception_message = f'`{invalid_duration}` is not a valid duration string.' - with self.assertRaises(BadArgument, msg=exception_message): - asyncio.run(converter.convert(self.context, invalid_duration)) + with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): + await converter.convert(self.context, invalid_duration) - def test_isodatetime_converter_for_valid(self): + @patch("bot.converters.datetime") + async def test_duration_converter_out_of_range(self, mock_datetime): + """Duration converter should raise BadArgument if datetime raises a ValueError.""" + mock_datetime.__add__.side_effect = ValueError + mock_datetime.utcnow.return_value = mock_datetime + + duration = f"{datetime.MAXYEAR}y" + exception_message = f"`{duration}` results in a datetime outside the supported range." + with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): + await Duration().convert(self.context, duration) + + async def test_isodatetime_converter_for_valid(self): """ISODateTime converter returns correct datetime for valid datetime string.""" test_values = ( # `YYYY-mm-ddTHH:MM:SSZ` | `YYYY-mm-dd HH:MM:SSZ` @@ -243,37 +254,37 @@ class ConverterTests(unittest.TestCase): for datetime_string, expected_dt in test_values: with self.subTest(datetime_string=datetime_string, expected_dt=expected_dt): - converted_dt = asyncio.run(converter.convert(self.context, datetime_string)) + converted_dt = await converter.convert(self.context, datetime_string) self.assertIsNone(converted_dt.tzinfo) self.assertEqual(converted_dt, expected_dt) - def test_isodatetime_converter_for_invalid(self): + async def test_isodatetime_converter_for_invalid(self): """ISODateTime converter raises the correct exception for invalid datetime strings.""" test_values = ( # Make sure it doesn't interfere with the Duration converter - ('1Y'), - ('1d'), - ('1H'), + '1Y', + '1d', + '1H', # Check if it fails when only providing the optional time part - ('10:10:10'), - ('10:00'), + '10:10:10', + '10:00', # Invalid date format - ('19-01-01'), + '19-01-01', # Other non-valid strings - ('fisk the tag master'), + 'fisk the tag master', ) converter = ISODateTime() for datetime_string in test_values: with self.subTest(datetime_string=datetime_string): exception_message = f"`{datetime_string}` is not a valid ISO-8601 datetime string" - with self.assertRaises(BadArgument, msg=exception_message): - asyncio.run(converter.convert(self.context, datetime_string)) + with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): + await converter.convert(self.context, datetime_string) - def test_hush_duration_converter_for_valid(self): + async def test_hush_duration_converter_for_valid(self): """HushDurationConverter returns correct value for minutes duration or `"forever"` strings.""" test_values = ( ("0", 0), @@ -286,10 +297,10 @@ class ConverterTests(unittest.TestCase): converter = HushDurationConverter() for minutes_string, expected_minutes in test_values: with self.subTest(minutes_string=minutes_string, expected_minutes=expected_minutes): - converted = asyncio.run(converter.convert(self.context, minutes_string)) + converted = await converter.convert(self.context, minutes_string) self.assertEqual(expected_minutes, converted) - def test_hush_duration_converter_for_invalid(self): + async def test_hush_duration_converter_for_invalid(self): """HushDurationConverter raises correct exception for invalid minutes duration strings.""" test_values = ( ("16", "Duration must be at most 15 minutes."), @@ -299,5 +310,5 @@ class ConverterTests(unittest.TestCase): converter = HushDurationConverter() for invalid_minutes_string, exception_message in test_values: with self.subTest(invalid_minutes_string=invalid_minutes_string, exception_message=exception_message): - with self.assertRaisesRegex(BadArgument, exception_message): - asyncio.run(converter.convert(self.context, invalid_minutes_string)) + with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): + await converter.convert(self.context, invalid_minutes_string) diff --git a/tests/bot/test_decorators.py b/tests/bot/test_decorators.py new file mode 100644 index 000000000..3d450caa0 --- /dev/null +++ b/tests/bot/test_decorators.py @@ -0,0 +1,147 @@ +import collections +import unittest +import unittest.mock + +from bot import constants +from bot.decorators import in_whitelist +from bot.utils.checks import InWhitelistCheckFailure +from tests import helpers + +InWhitelistTestCase = collections.namedtuple("WhitelistedContextTestCase", ("kwargs", "ctx", "description")) + + +class InWhitelistTests(unittest.TestCase): + """Tests for the `in_whitelist` check.""" + + @classmethod + def setUpClass(cls): + """Set up helpers that only need to be defined once.""" + cls.bot_commands = helpers.MockTextChannel(id=123456789, category_id=123456) + cls.help_channel = helpers.MockTextChannel(id=987654321, category_id=987654) + cls.non_whitelisted_channel = helpers.MockTextChannel(id=666666) + cls.dm_channel = helpers.MockDMChannel() + + cls.non_staff_member = helpers.MockMember() + cls.staff_role = helpers.MockRole(id=121212) + cls.staff_member = helpers.MockMember(roles=(cls.staff_role,)) + + cls.channels = (cls.bot_commands.id,) + cls.categories = (cls.help_channel.category_id,) + cls.roles = (cls.staff_role.id,) + + def test_predicate_returns_true_for_whitelisted_context(self): + """The predicate should return `True` if a whitelisted context was passed to it.""" + test_cases = ( + InWhitelistTestCase( + kwargs={"channels": self.channels}, + ctx=helpers.MockContext(channel=self.bot_commands, author=self.non_staff_member), + description="In whitelisted channels by members without whitelisted roles", + ), + InWhitelistTestCase( + kwargs={"redirect": self.bot_commands.id}, + ctx=helpers.MockContext(channel=self.bot_commands, author=self.non_staff_member), + description="`redirect` should be implicitly added to `channels`", + ), + InWhitelistTestCase( + kwargs={"categories": self.categories}, + ctx=helpers.MockContext(channel=self.help_channel, author=self.non_staff_member), + description="Whitelisted category without whitelisted role", + ), + InWhitelistTestCase( + kwargs={"roles": self.roles}, + ctx=helpers.MockContext(channel=self.non_whitelisted_channel, author=self.staff_member), + description="Whitelisted role outside of whitelisted channel/category" + ), + InWhitelistTestCase( + kwargs={ + "channels": self.channels, + "categories": self.categories, + "roles": self.roles, + "redirect": self.bot_commands, + }, + ctx=helpers.MockContext(channel=self.help_channel, author=self.staff_member), + description="Case with all whitelist kwargs used", + ), + ) + + for test_case in test_cases: + # patch `commands.check` with a no-op lambda that just returns the predicate passed to it + # so we can test the predicate that was generated from the specified kwargs. + with unittest.mock.patch("bot.decorators.commands.check", new=lambda predicate: predicate): + predicate = in_whitelist(**test_case.kwargs) + + with self.subTest(test_description=test_case.description): + self.assertTrue(predicate(test_case.ctx)) + + def test_predicate_raises_exception_for_non_whitelisted_context(self): + """The predicate should raise `InWhitelistCheckFailure` for a non-whitelisted context.""" + test_cases = ( + # Failing check with explicit `redirect` + InWhitelistTestCase( + kwargs={ + "categories": self.categories, + "channels": self.channels, + "roles": self.roles, + "redirect": self.bot_commands.id, + }, + ctx=helpers.MockContext(channel=self.non_whitelisted_channel, author=self.non_staff_member), + description="Failing check with an explicit redirect channel", + ), + + # Failing check with implicit `redirect` + InWhitelistTestCase( + kwargs={ + "categories": self.categories, + "channels": self.channels, + "roles": self.roles, + }, + ctx=helpers.MockContext(channel=self.non_whitelisted_channel, author=self.non_staff_member), + description="Failing check with an implicit redirect channel", + ), + + # Failing check without `redirect` + InWhitelistTestCase( + kwargs={ + "categories": self.categories, + "channels": self.channels, + "roles": self.roles, + "redirect": None, + }, + ctx=helpers.MockContext(channel=self.non_whitelisted_channel, author=self.non_staff_member), + description="Failing check without a redirect channel", + ), + + # Command issued in DM channel + InWhitelistTestCase( + kwargs={ + "categories": self.categories, + "channels": self.channels, + "roles": self.roles, + "redirect": None, + }, + ctx=helpers.MockContext(channel=self.dm_channel, author=self.dm_channel.me), + description="Commands issued in DM channel should be rejected", + ), + ) + + for test_case in test_cases: + if "redirect" not in test_case.kwargs or test_case.kwargs["redirect"] is not None: + # There are two cases in which we have a redirect channel: + # 1. No redirect channel was passed; the default value of `bot_commands` is used + # 2. An explicit `redirect` is set that is "not None" + redirect_channel = test_case.kwargs.get("redirect", constants.Channels.bot_commands) + redirect_message = f" here. Please use the <#{redirect_channel}> channel instead" + else: + # If an explicit `None` was passed for `redirect`, there is no redirect channel + redirect_message = "" + + exception_message = f"You are not allowed to use that command{redirect_message}." + + # patch `commands.check` with a no-op lambda that just returns the predicate passed to it + # so we can test the predicate that was generated from the specified kwargs. + with unittest.mock.patch("bot.decorators.commands.check", new=lambda predicate: predicate): + predicate = in_whitelist(**test_case.kwargs) + + with self.subTest(test_description=test_case.description): + with self.assertRaisesRegex(InWhitelistCheckFailure, exception_message): + predicate(test_case.ctx) diff --git a/tests/bot/test_pagination.py b/tests/bot/test_pagination.py index 0a734b505..630f2516d 100644 --- a/tests/bot/test_pagination.py +++ b/tests/bot/test_pagination.py @@ -8,29 +8,39 @@ class LinePaginatorTests(TestCase): def setUp(self): """Create a paginator for the test method.""" - self.paginator = pagination.LinePaginator(prefix='', suffix='', max_size=30) - - def test_add_line_raises_on_too_long_lines(self): - """`add_line` should raise a `RuntimeError` for too long lines.""" - message = f"Line exceeds maximum page size {self.paginator.max_size - 2}" - with self.assertRaises(RuntimeError, msg=message): - self.paginator.add_line('x' * self.paginator.max_size) + self.paginator = pagination.LinePaginator(prefix='', suffix='', max_size=30, + scale_to_size=50) def test_add_line_works_on_small_lines(self): """`add_line` should allow small lines to be added.""" self.paginator.add_line('x' * (self.paginator.max_size - 3)) - - -class ImagePaginatorTests(TestCase): - """Tests functionality of the `ImagePaginator`.""" - - def setUp(self): - """Create a paginator for the test method.""" - self.paginator = pagination.ImagePaginator() - - def test_add_image_appends_image(self): - """`add_image` appends the image to the image list.""" - image = 'lemon' - self.paginator.add_image(image) - - assert self.paginator.images == [image] + # Note that the page isn't added to _pages until it's full. + self.assertEqual(len(self.paginator._pages), 0) + + def test_add_line_works_on_long_lines(self): + """After additional lines after `max_size` is exceeded should go on the next page.""" + self.paginator.add_line('x' * self.paginator.max_size) + self.assertEqual(len(self.paginator._pages), 0) + + # Any additional lines should start a new page after `max_size` is exceeded. + self.paginator.add_line('x') + self.assertEqual(len(self.paginator._pages), 1) + + def test_add_line_continuation(self): + """When `scale_to_size` is exceeded, remaining words should be split onto the next page.""" + self.paginator.add_line('zyz ' * (self.paginator.scale_to_size//4 + 1)) + self.assertEqual(len(self.paginator._pages), 1) + + def test_add_line_no_continuation(self): + """If adding a new line to an existing page would exceed `max_size`, it should start a new + page rather than using continuation. + """ + self.paginator.add_line('z' * (self.paginator.max_size - 3)) + self.paginator.add_line('z') + self.assertEqual(len(self.paginator._pages), 1) + + def test_add_line_truncates_very_long_words(self): + """`add_line` should truncate if a single long word exceeds `scale_to_size`.""" + self.paginator.add_line('x' * (self.paginator.scale_to_size + 1)) + # Note: item at index 1 is the truncated line, index 0 is prefix + self.assertEqual(self.paginator._current_page[1], 'x' * self.paginator.scale_to_size) diff --git a/tests/bot/utils/test_checks.py b/tests/bot/utils/test_checks.py index 9610771e5..883465e0b 100644 --- a/tests/bot/utils/test_checks.py +++ b/tests/bot/utils/test_checks.py @@ -1,51 +1,93 @@ import unittest +from unittest.mock import MagicMock + +from discord import DMChannel from bot.utils import checks +from bot.utils.checks import InWhitelistCheckFailure from tests.helpers import MockContext, MockRole -class ChecksTests(unittest.TestCase): +class ChecksTests(unittest.IsolatedAsyncioTestCase): """Tests the check functions defined in `bot.checks`.""" def setUp(self): self.ctx = MockContext() - def test_with_role_check_without_guild(self): - """`with_role_check` returns `False` if `Context.guild` is None.""" - self.ctx.guild = None - self.assertFalse(checks.with_role_check(self.ctx)) + async def test_has_any_role_check_without_guild(self): + """`has_any_role_check` returns `False` for non-guild channels.""" + self.ctx.channel = MagicMock(DMChannel) + self.assertFalse(await checks.has_any_role_check(self.ctx)) - def test_with_role_check_without_required_roles(self): - """`with_role_check` returns `False` if `Context.author` lacks the required role.""" + async def test_has_any_role_check_without_required_roles(self): + """`has_any_role_check` returns `False` if `Context.author` lacks the required role.""" self.ctx.author.roles = [] - self.assertFalse(checks.with_role_check(self.ctx)) + self.assertFalse(await checks.has_any_role_check(self.ctx)) - def test_with_role_check_with_guild_and_required_role(self): - """`with_role_check` returns `True` if `Context.author` has the required role.""" + async def test_has_any_role_check_with_guild_and_required_role(self): + """`has_any_role_check` returns `True` if `Context.author` has the required role.""" self.ctx.author.roles.append(MockRole(id=10)) - self.assertTrue(checks.with_role_check(self.ctx, 10)) + self.assertTrue(await checks.has_any_role_check(self.ctx, 10)) - def test_without_role_check_without_guild(self): - """`without_role_check` should return `False` when `Context.guild` is None.""" - self.ctx.guild = None - self.assertFalse(checks.without_role_check(self.ctx)) + async def test_has_no_roles_check_without_guild(self): + """`has_no_roles_check` should return `False` when `Context.guild` is None.""" + self.ctx.channel = MagicMock(DMChannel) + self.assertFalse(await checks.has_no_roles_check(self.ctx)) - def test_without_role_check_returns_false_with_unwanted_role(self): - """`without_role_check` returns `False` if `Context.author` has unwanted role.""" + async def test_has_no_roles_check_returns_false_with_unwanted_role(self): + """`has_no_roles_check` returns `False` if `Context.author` has unwanted role.""" role_id = 42 self.ctx.author.roles.append(MockRole(id=role_id)) - self.assertFalse(checks.without_role_check(self.ctx, role_id)) + self.assertFalse(await checks.has_no_roles_check(self.ctx, role_id)) - def test_without_role_check_returns_true_without_unwanted_role(self): - """`without_role_check` returns `True` if `Context.author` does not have unwanted role.""" + async def test_has_no_roles_check_returns_true_without_unwanted_role(self): + """`has_no_roles_check` returns `True` if `Context.author` does not have unwanted role.""" role_id = 42 self.ctx.author.roles.append(MockRole(id=role_id)) - self.assertTrue(checks.without_role_check(self.ctx, role_id + 10)) + self.assertTrue(await checks.has_no_roles_check(self.ctx, role_id + 10)) + + def test_in_whitelist_check_correct_channel(self): + """`in_whitelist_check` returns `True` if `Context.channel.id` is in the channel list.""" + channel_id = 3 + self.ctx.channel.id = channel_id + self.assertTrue(checks.in_whitelist_check(self.ctx, [channel_id])) + + def test_in_whitelist_check_incorrect_channel(self): + """`in_whitelist_check` raises InWhitelistCheckFailure if there's no channel match.""" + self.ctx.channel.id = 3 + with self.assertRaises(InWhitelistCheckFailure): + checks.in_whitelist_check(self.ctx, [4]) + + def test_in_whitelist_check_correct_category(self): + """`in_whitelist_check` returns `True` if `Context.channel.category_id` is in the category list.""" + category_id = 3 + self.ctx.channel.category_id = category_id + self.assertTrue(checks.in_whitelist_check(self.ctx, categories=[category_id])) + + def test_in_whitelist_check_incorrect_category(self): + """`in_whitelist_check` raises InWhitelistCheckFailure if there's no category match.""" + self.ctx.channel.category_id = 3 + with self.assertRaises(InWhitelistCheckFailure): + checks.in_whitelist_check(self.ctx, categories=[4]) + + def test_in_whitelist_check_correct_role(self): + """`in_whitelist_check` returns `True` if any of the `Context.author.roles` are in the roles list.""" + self.ctx.author.roles = (MagicMock(id=1), MagicMock(id=2)) + self.assertTrue(checks.in_whitelist_check(self.ctx, roles=[2, 6])) + + def test_in_whitelist_check_incorrect_role(self): + """`in_whitelist_check` raises InWhitelistCheckFailure if there's no role match.""" + self.ctx.author.roles = (MagicMock(id=1), MagicMock(id=2)) + with self.assertRaises(InWhitelistCheckFailure): + checks.in_whitelist_check(self.ctx, roles=[4]) - def test_in_channel_check_for_correct_channel(self): - self.ctx.channel.id = 42 - self.assertTrue(checks.in_channel_check(self.ctx, *[42])) + def test_in_whitelist_check_fail_silently(self): + """`in_whitelist_check` test no exception raised if `fail_silently` is `True`""" + self.assertFalse(checks.in_whitelist_check(self.ctx, roles=[2, 6], fail_silently=True)) - def test_in_channel_check_for_incorrect_channel(self): - self.ctx.channel.id = 42 + 10 - self.assertFalse(checks.in_channel_check(self.ctx, *[42])) + def test_in_whitelist_check_complex(self): + """`in_whitelist_check` test with multiple parameters""" + self.ctx.author.roles = (MagicMock(id=1), MagicMock(id=2)) + self.ctx.channel.category_id = 3 + self.ctx.channel.id = 5 + self.assertTrue(checks.in_whitelist_check(self.ctx, channels=[1], categories=[8], roles=[2])) diff --git a/tests/bot/utils/test_messages.py b/tests/bot/utils/test_messages.py new file mode 100644 index 000000000..9c22c9751 --- /dev/null +++ b/tests/bot/utils/test_messages.py @@ -0,0 +1,27 @@ +import unittest + +from bot.utils import messages + + +class TestMessages(unittest.TestCase): + """Tests for functions in the `bot.utils.messages` module.""" + + def test_sub_clyde(self): + """Uppercase E's and lowercase e's are substituted with their cyrillic counterparts.""" + sub_e = "\u0435" + sub_E = "\u0415" # noqa: N806: Uppercase E in variable name + + test_cases = ( + (None, None), + ("", ""), + ("clyde", f"clyd{sub_e}"), + ("CLYDE", f"CLYD{sub_E}"), + ("cLyDe", f"cLyD{sub_e}"), + ("BIGclyde", f"BIGclyd{sub_e}"), + ("small clydeus the unholy", f"small clyd{sub_e}us the unholy"), + ("BIGCLYDE, babyclyde", f"BIGCLYD{sub_E}, babyclyd{sub_e}"), + ) + + for username_in, username_out in test_cases: + with self.subTest(input=username_in, expected_output=username_out): + self.assertEqual(messages.sub_clyde(username_in), username_out) diff --git a/tests/bot/utils/test_services.py b/tests/bot/utils/test_services.py new file mode 100644 index 000000000..5e0855704 --- /dev/null +++ b/tests/bot/utils/test_services.py @@ -0,0 +1,74 @@ +import logging +import unittest +from unittest.mock import AsyncMock, MagicMock, Mock, patch + +from aiohttp import ClientConnectorError + +from bot.utils.services import FAILED_REQUEST_ATTEMPTS, send_to_paste_service + + +class PasteTests(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self.http_session = MagicMock() + + @patch("bot.utils.services.URLs.paste_service", "https://paste_service.com/{key}") + async def test_url_and_sent_contents(self): + """Correct url was used and post was called with expected data.""" + response = MagicMock( + json=AsyncMock(return_value={"key": ""}) + ) + self.http_session.post().__aenter__.return_value = response + self.http_session.post.reset_mock() + await send_to_paste_service(self.http_session, "Content") + self.http_session.post.assert_called_once_with("https://paste_service.com/documents", data="Content") + + @patch("bot.utils.services.URLs.paste_service", "https://paste_service.com/{key}") + async def test_paste_returns_correct_url_on_success(self): + """Url with specified extension is returned on successful requests.""" + key = "paste_key" + test_cases = ( + (f"https://paste_service.com/{key}.txt", "txt"), + (f"https://paste_service.com/{key}.py", "py"), + (f"https://paste_service.com/{key}", ""), + ) + response = MagicMock( + json=AsyncMock(return_value={"key": key}) + ) + self.http_session.post().__aenter__.return_value = response + + for expected_output, extension in test_cases: + with self.subTest(msg=f"Send contents with extension {repr(extension)}"): + self.assertEqual( + await send_to_paste_service(self.http_session, "", extension=extension), + expected_output + ) + + async def test_request_repeated_on_json_errors(self): + """Json with error message and invalid json are handled as errors and requests repeated.""" + test_cases = ({"message": "error"}, {"unexpected_key": None}, {}) + self.http_session.post().__aenter__.return_value = response = MagicMock() + self.http_session.post.reset_mock() + + for error_json in test_cases: + with self.subTest(error_json=error_json): + response.json = AsyncMock(return_value=error_json) + result = await send_to_paste_service(self.http_session, "") + self.assertEqual(self.http_session.post.call_count, FAILED_REQUEST_ATTEMPTS) + self.assertIsNone(result) + + self.http_session.post.reset_mock() + + async def test_request_repeated_on_connection_errors(self): + """Requests are repeated in the case of connection errors.""" + self.http_session.post = MagicMock(side_effect=ClientConnectorError(Mock(), Mock())) + result = await send_to_paste_service(self.http_session, "") + self.assertEqual(self.http_session.post.call_count, FAILED_REQUEST_ATTEMPTS) + self.assertIsNone(result) + + async def test_general_error_handled_and_request_repeated(self): + """All `Exception`s are handled, logged and request repeated.""" + self.http_session.post = MagicMock(side_effect=Exception) + result = await send_to_paste_service(self.http_session, "") + self.assertEqual(self.http_session.post.call_count, FAILED_REQUEST_ATTEMPTS) + self.assertLogs("bot.utils", logging.ERROR) + self.assertIsNone(result) diff --git a/tests/helpers.py b/tests/helpers.py index 8e13f0f28..e47fdf28f 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -4,12 +4,15 @@ import collections import itertools import logging import unittest.mock -from typing import Iterable, Optional +from asyncio import AbstractEventLoop +from typing import Callable, Iterable, Optional import discord +from aiohttp import ClientSession from discord.ext.commands import Context from bot.api import APIClient +from bot.async_stats import AsyncStatsClient from bot.bot import Bot @@ -23,6 +26,24 @@ for logger in logging.Logger.manager.loggerDict.values(): logger.setLevel(logging.CRITICAL) +def autospec(target, *attributes: str, **kwargs) -> Callable: + """Patch multiple `attributes` of a `target` with autospecced mocks and `spec_set` as True.""" + # Caller's kwargs should take priority and overwrite the defaults. + kwargs = {'spec_set': True, 'autospec': True, **kwargs} + + # Import the target if it's a string. + # This is to support both object and string targets like patch.multiple. + if type(target) is str: + target = unittest.mock._importer(target) + + def decorator(func): + for attribute in attributes: + patcher = unittest.mock.patch.object(target, attribute, **kwargs) + func = patcher(func) + return func + return decorator + + class HashableMixin(discord.mixins.EqualityComparable): """ Mixin that provides similar hashing and equality functionality as discord.py's `Hashable` mixin. @@ -205,6 +226,10 @@ class MockRole(CustomMockMixin, unittest.mock.Mock, ColourMixin, HashableMixin): """Simplified position-based comparisons similar to those of `discord.Role`.""" return self.position < other.position + def __ge__(self, other): + """Simplified position-based comparisons similar to those of `discord.Role`.""" + return self.position >= other.position + # Create a Member instance to get a realistic Mock of `discord.Member` member_data = {'user': 'lemon', 'roles': [1]} @@ -264,10 +289,16 @@ class MockAPIClient(CustomMockMixin, unittest.mock.MagicMock): spec_set = APIClient -# Create a Bot instance to get a realistic MagicMock of `discord.ext.commands.Bot` -bot_instance = Bot(command_prefix=unittest.mock.MagicMock()) -bot_instance.http_session = None -bot_instance.api_client = None +def _get_mock_loop() -> unittest.mock.Mock: + """Return a mocked asyncio.AbstractEventLoop.""" + loop = unittest.mock.create_autospec(spec=AbstractEventLoop, spec_set=True) + + # Since calling `create_task` on our MockBot does not actually schedule the coroutine object + # as a task in the asyncio loop, this `side_effect` calls `close()` on the coroutine object + # to prevent "has not been awaited"-warnings. + loop.create_task.side_effect = lambda coroutine: coroutine.close() + + return loop class MockBot(CustomMockMixin, unittest.mock.MagicMock): @@ -277,17 +308,20 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock): Instances of this class will follow the specifications of `discord.ext.commands.Bot` instances. For more information, see the `MockGuild` docstring. """ - spec_set = bot_instance - additional_spec_asyncs = ("wait_for",) + spec_set = Bot( + command_prefix=unittest.mock.MagicMock(), + loop=_get_mock_loop(), + redis_session=unittest.mock.MagicMock(), + ) + additional_spec_asyncs = ("wait_for", "redis_ready") def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - self.api_client = MockAPIClient() - # Since calling `create_task` on our MockBot does not actually schedule the coroutine object - # as a task in the asyncio loop, this `side_effect` calls `close()` on the coroutine object - # to prevent "has not been awaited"-warnings. - self.loop.create_task.side_effect = lambda coroutine: coroutine.close() + self.loop = _get_mock_loop() + self.api_client = MockAPIClient(loop=self.loop) + self.http_session = unittest.mock.create_autospec(spec=ClientSession, spec_set=True) + self.stats = unittest.mock.create_autospec(spec=AsyncStatsClient, spec_set=True) # Create a TextChannel instance to get a realistic MagicMock of `discord.TextChannel` @@ -315,7 +349,7 @@ class MockTextChannel(CustomMockMixin, unittest.mock.Mock, HashableMixin): """ spec_set = channel_instance - def __init__(self, name: str = 'channel', channel_id: int = 1, **kwargs) -> None: + def __init__(self, **kwargs) -> None: default_kwargs = {'id': next(self.discord_id), 'name': 'channel', 'guild': MockGuild()} super().__init__(**collections.ChainMap(kwargs, default_kwargs)) @@ -323,6 +357,27 @@ class MockTextChannel(CustomMockMixin, unittest.mock.Mock, HashableMixin): self.mention = f"#{self.name}" +# Create data for the DMChannel instance +state = unittest.mock.MagicMock() +me = unittest.mock.MagicMock() +dm_channel_data = {"id": 1, "recipients": [unittest.mock.MagicMock()]} +dm_channel_instance = discord.DMChannel(me=me, state=state, data=dm_channel_data) + + +class MockDMChannel(CustomMockMixin, unittest.mock.Mock, HashableMixin): + """ + A MagicMock subclass to mock TextChannel objects. + + Instances of this class will follow the specifications of `discord.TextChannel` instances. For + more information, see the `MockGuild` docstring. + """ + spec_set = dm_channel_instance + + def __init__(self, **kwargs) -> None: + default_kwargs = {'id': next(self.discord_id), 'recipient': MockUser(), "me": MockUser()} + super().__init__(**collections.ChainMap(kwargs, default_kwargs)) + + # Create a Message instance to get a realistic MagicMock of `discord.Message` message_data = { 'id': 1, |