diff options
| author | 2020-09-16 23:41:04 +0800 | |
|---|---|---|
| committer | 2020-09-16 23:41:04 +0800 | |
| commit | 32b6460f96aba7e5d6d0d4d2d16877962c9575f2 (patch) | |
| tree | 174afd706eaeb643ba45d946265c0e6616187ba1 | |
| parent | Change tests to work with the new file layout. (diff) | |
| parent | Verification: update & improve docstrings (diff) | |
Merge branch 'master' into truncate-internal-eval
81 files changed, 4960 insertions, 1923 deletions
| @@ -12,7 +12,7 @@ beautifulsoup4 = "~=4.9"  colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"}  coloredlogs = "~=14.0"  deepdiff = "~=4.0" -discord.py = "~=1.3.2" +discord.py = "~=1.4.0"  fakeredis = "~=1.4"  feedparser = "~=5.2"  fuzzywuzzy = "~=0.17" @@ -28,7 +28,7 @@ statsd = "~=3.3"  [dev-packages]  coverage = "~=5.0" -flake8 = "~=3.7" +flake8 = "~=3.8"  flake8-annotations = "~=2.0"  flake8-bugbear = "~=20.1"  flake8-docstrings = "~=1.4" diff --git a/Pipfile.lock b/Pipfile.lock index 0e591710c..50ddd478c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@  {      "_meta": {          "hash": { -            "sha256": "0297accc3d614d3da8080b89d56ef7fe489c28a0ada8102df396a604af7ee330" +            "sha256": "1905fd7eb15074ddbf04f2177b6cdd65edc4c74cb5fcbf4e6ca08ef649ba8a3c"          },          "pipfile-spec": 6,          "requires": { @@ -60,10 +60,11 @@          },          "aiormq": {              "hashes": [ -                "sha256:41a9d4eb17db805f30ed172f3f609fe0c2b16657fb15b1b67df19d251dd93c0d", -                "sha256:7c19477a9450824cb79f9949fd238f4148e2c0dca67756a2868863c387209f04" +                "sha256:106695a836f19c1af6c46b58e8aac80e00f86c5b3287a3c6483a1ee369cc95c9", +                "sha256:9f6dbf6155fe2b7a3d24bf68de97fb812db0fac0a54e96bc1af14ea95078ba7f"              ], -            "version": "==3.2.2" +            "markers": "python_version >= '3.6'", +            "version": "==3.2.3"          },          "alabaster": {              "hashes": [ @@ -77,6 +78,7 @@                  "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",                  "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"              ], +            "markers": "python_full_version >= '3.5.3'",              "version": "==3.0.1"          },          "attrs": { @@ -84,6 +86,7 @@                  "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",                  "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"              ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",              "version": "==19.3.0"          },          "babel": { @@ -91,6 +94,7 @@                  "sha256:1aac2ae2d0d8ea368fa90906567f5c08463d98ade155c0c4bfedd6a0f7160e38",                  "sha256:d670ea0b10f8b723672d3a6abeb87b565b244da220d76b4dba1b66269ec152d4"              ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",              "version": "==2.8.0"          },          "beautifulsoup4": { @@ -104,43 +108,43 @@          },          "certifi": {              "hashes": [ -                "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304", -                "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519" +                "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", +                "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"              ], -            "version": "==2020.4.5.1" +            "version": "==2020.6.20"          },          "cffi": {              "hashes": [ -                "sha256:001bf3242a1bb04d985d63e138230802c6c8d4db3668fb545fb5005ddf5bb5ff", -                "sha256:00789914be39dffba161cfc5be31b55775de5ba2235fe49aa28c148236c4e06b", -                "sha256:028a579fc9aed3af38f4892bdcc7390508adabc30c6af4a6e4f611b0c680e6ac", -                "sha256:14491a910663bf9f13ddf2bc8f60562d6bc5315c1f09c704937ef17293fb85b0", -                "sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384", -                "sha256:2089ed025da3919d2e75a4d963d008330c96751127dd6f73c8dc0c65041b4c26", -                "sha256:2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6", -                "sha256:337d448e5a725bba2d8293c48d9353fc68d0e9e4088d62a9571def317797522b", -                "sha256:399aed636c7d3749bbed55bc907c3288cb43c65c4389964ad5ff849b6370603e", -                "sha256:3b911c2dbd4f423b4c4fcca138cadde747abdb20d196c4a48708b8a2d32b16dd", -                "sha256:3d311bcc4a41408cf5854f06ef2c5cab88f9fded37a3b95936c9879c1640d4c2", -                "sha256:62ae9af2d069ea2698bf536dcfe1e4eed9090211dbaafeeedf5cb6c41b352f66", -                "sha256:66e41db66b47d0d8672d8ed2708ba91b2f2524ece3dee48b5dfb36be8c2f21dc", -                "sha256:675686925a9fb403edba0114db74e741d8181683dcf216be697d208857e04ca8", -                "sha256:7e63cbcf2429a8dbfe48dcc2322d5f2220b77b2e17b7ba023d6166d84655da55", -                "sha256:8a6c688fefb4e1cd56feb6c511984a6c4f7ec7d2a1ff31a10254f3c817054ae4", -                "sha256:8c0ffc886aea5df6a1762d0019e9cb05f825d0eec1f520c51be9d198701daee5", -                "sha256:95cd16d3dee553f882540c1ffe331d085c9e629499ceadfbda4d4fde635f4b7d", -                "sha256:99f748a7e71ff382613b4e1acc0ac83bf7ad167fb3802e35e90d9763daba4d78", -                "sha256:b8c78301cefcf5fd914aad35d3c04c2b21ce8629b5e4f4e45ae6812e461910fa", -                "sha256:c420917b188a5582a56d8b93bdd8e0f6eca08c84ff623a4c16e809152cd35793", -                "sha256:c43866529f2f06fe0edc6246eb4faa34f03fe88b64a0a9a942561c8e22f4b71f", -                "sha256:cab50b8c2250b46fe738c77dbd25ce017d5e6fb35d3407606e7a4180656a5a6a", -                "sha256:cef128cb4d5e0b3493f058f10ce32365972c554572ff821e175dbc6f8ff6924f", -                "sha256:cf16e3cf6c0a5fdd9bc10c21687e19d29ad1fe863372b5543deaec1039581a30", -                "sha256:e56c744aa6ff427a607763346e4170629caf7e48ead6921745986db3692f987f", -                "sha256:e577934fc5f8779c554639376beeaa5657d54349096ef24abe8c74c5d9c117c3", -                "sha256:f2b0fa0c01d8a0c7483afd9f31d7ecf2d71760ca24499c8697aeb5ca37dc090c" -            ], -            "version": "==1.14.0" +                "sha256:267adcf6e68d77ba154334a3e4fc921b8e63cbb38ca00d33d40655d4228502bc", +                "sha256:26f33e8f6a70c255767e3c3f957ccafc7f1f706b966e110b855bfe944511f1f9", +                "sha256:3cd2c044517f38d1b577f05927fb9729d3396f1d44d0c659a445599e79519792", +                "sha256:4a03416915b82b81af5502459a8a9dd62a3c299b295dcdf470877cb948d655f2", +                "sha256:4ce1e995aeecf7cc32380bc11598bfdfa017d592259d5da00fc7ded11e61d022", +                "sha256:4f53e4128c81ca3212ff4cf097c797ab44646a40b42ec02a891155cd7a2ba4d8", +                "sha256:4fa72a52a906425416f41738728268072d5acfd48cbe7796af07a923236bcf96", +                "sha256:66dd45eb9530e3dde8f7c009f84568bc7cac489b93d04ac86e3111fb46e470c2", +                "sha256:6923d077d9ae9e8bacbdb1c07ae78405a9306c8fd1af13bfa06ca891095eb995", +                "sha256:833401b15de1bb92791d7b6fb353d4af60dc688eaa521bd97203dcd2d124a7c1", +                "sha256:8416ed88ddc057bab0526d4e4e9f3660f614ac2394b5e019a628cdfff3733849", +                "sha256:892daa86384994fdf4856cb43c93f40cbe80f7f95bb5da94971b39c7f54b3a9c", +                "sha256:98be759efdb5e5fa161e46d404f4e0ce388e72fbf7d9baf010aff16689e22abe", +                "sha256:a6d28e7f14ecf3b2ad67c4f106841218c8ab12a0683b1528534a6c87d2307af3", +                "sha256:b1d6ebc891607e71fd9da71688fcf332a6630b7f5b7f5549e6e631821c0e5d90", +                "sha256:b2a2b0d276a136146e012154baefaea2758ef1f56ae9f4e01c612b0831e0bd2f", +                "sha256:b87dfa9f10a470eee7f24234a37d1d5f51e5f5fa9eeffda7c282e2b8f5162eb1", +                "sha256:bac0d6f7728a9cc3c1e06d4fcbac12aaa70e9379b3025b27ec1226f0e2d404cf", +                "sha256:c991112622baee0ae4d55c008380c32ecfd0ad417bcd0417ba432e6ba7328caa", +                "sha256:cda422d54ee7905bfc53ee6915ab68fe7b230cacf581110df4272ee10462aadc", +                "sha256:d3148b6ba3923c5850ea197a91a42683f946dba7e8eb82dfa211ab7e708de939", +                "sha256:d6033b4ffa34ef70f0b8086fd4c3df4bf801fee485a8a7d4519399818351aa8e", +                "sha256:ddff0b2bd7edcc8c82d1adde6dbbf5e60d57ce985402541cd2985c27f7bec2a0", +                "sha256:e23cb7f1d8e0f93addf0cae3c5b6f00324cccb4a7949ee558d7b6ca973ab8ae9", +                "sha256:effd2ba52cee4ceff1a77f20d2a9f9bf8d50353c854a282b8760ac15b9833168", +                "sha256:f90c2267101010de42f7273c94a1f026e56cbc043f9330acd8a80e64300aba33", +                "sha256:f960375e9823ae6a07072ff7f8a85954e5a6434f97869f50d0e41649a1c8144f", +                "sha256:fcf32bf76dc25e30ed793145a57426064520890d7c02866eb93d3e4abe516948" +            ], +            "version": "==1.14.1"          },          "chardet": {              "hashes": [ @@ -154,7 +158,6 @@                  "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff",                  "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"              ], -            "index": "pypi",              "markers": "sys_platform == 'win32'",              "version": "==0.4.3"          }, @@ -180,29 +183,32 @@                  "sha256:ff6653655e342e7721dfb3f10421345fd852c2a33f2cca912b1c39b3778a9429"              ],              "index": "pypi", +            "py": "~=1.4.0",              "version": "==1.0.1"          },          "discord.py": {              "hashes": [ -                "sha256:406871b06d86c3dc49fba63238519f28628dac946fef8a0e22988ff58ec05580", -                "sha256:ad00e34c72d2faa8db2157b651d05f3c415d7d05078e7e41dc9e8dc240051beb" +                "sha256:2b1846bfa382b54f4eace8e437a9f59f185388c5b08749ac0e1bbd98e05bfde5", +                "sha256:f3db9531fccc391f51de65cfa46133106a9ba12ff2927aca6c14bffd3b7f17b5"              ], -            "version": "==1.3.3" +            "markers": "python_full_version >= '3.5.3'", +            "version": "==1.4.0"          },          "docutils": {              "hashes": [                  "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af",                  "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"              ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",              "version": "==0.16"          },          "fakeredis": {              "hashes": [ -                "sha256:4d170886865a91dbc8b7f8cbd4e5d488f4c5f2f25dfae127f001617bbe9e8f97", -                "sha256:647b2593d349d9d4e566c8dadb2e4c71ba35be5bdc4f1f7ac2d565a12a965053" +                "sha256:790c85ad0f3b2967aba1f51767021bc59760fcb612159584be018ea7384f7fd2", +                "sha256:fdfe06f277092d022c271fcaefdc1f0c8d9bfa8cb15374cae41d66a20bd96d2b"              ],              "index": "pypi", -            "version": "==1.4.1" +            "version": "==1.4.2"          },          "feedparser": {              "hashes": [ @@ -223,68 +229,78 @@          },          "hiredis": {              "hashes": [ -                "sha256:01b577f84c20ecc9c07fc4c184231b08e3c3942de096fa99978e053de231c423", -                "sha256:01ff0900134166961c9e339df77c33b72f7edc5cb41739f0babcd9faa345926e", -                "sha256:03ed34a13316d0c34213c4fd46e0fa3a5299073f4d4f08e93fed8c2108b399b3", -                "sha256:040436e91df5143aff9e0debb49530d0b17a6bd52200ce568621c31ef581b10d", -                "sha256:091eb38fbf968d1c5b703e412bbbd25f43a7967d8400842cee33a5a07b33c27b", -                "sha256:102f9b9dc6ed57feb3a7c9bdf7e71cb7c278fe8df1edfcfe896bc3e0c2be9447", -                "sha256:2b4b392c7e3082860c8371fab3ae762139090f9115819e12d9f56060f9ede05d", -                "sha256:2c9cc0b986397b833073f466e6b9e9c70d1d4dc2c2c1b3e9cae3a23102ff296c", -                "sha256:2fa65a9df683bca72073cd77709ddeb289ea2b114d3775d225fbbcc5faf808c5", -                "sha256:38437a681f17c975fd22349e72c29bc643f8e7eb2d6dc5df419eac59afa4d7ce", -                "sha256:3b3428fa3cf1ee178807b52c9bee8950ab94cd4eaa9bfae8c1bbae3c49501d34", -                "sha256:3dd8c2fae7f5494978facb0e93297dd627b1a3f536f3b070cf0a7d9157a07dcb", -                "sha256:4414a96c212e732723b5c3d7c04d386ebbb2ec359e1de646322cbc3f875cbd0d", -                "sha256:48c627581ad4ef60adbac980981407939acf13a0e18f093502c7b542223c4f19", -                "sha256:4a60e71625a2d78d8ab84dfb2fa2cfd9458c964b6e6c04fea76d9ade153fb371", -                "sha256:585ace09f434e43d8a8dbeb366865b1a044d7c06319b3c7372a0a00e63b860f4", -                "sha256:74b364b3f06c9cf0a53f7df611045bc9437ed972a283fa1f0b12537236d23ddc", -                "sha256:75c65c3850e89e9daa68d1b9bedd5806f177d60aa5a7b0953b4829481cfc1f72", -                "sha256:7f052de8bf744730a9120dbdc67bfeb7605a01f69fb8e7ba5c475af33c24e145", -                "sha256:8113a7d5e87ecf57cd4ae263cc9e429adb9a3e59f5a7768da5d3312a8d0a051a", -                "sha256:84857ce239eb8ed191ac78e77ff65d52902f00f30f4ee83bf80eb71da73b70e6", -                "sha256:8644a48ddc4a40b3e3a6b9443f396c2ee353afb2d45656c4fc68d04a82e8e3f7", -                "sha256:936aa565e673536e8a211e43ec43197406f24cd1f290138bd143765079c8ba00", -                "sha256:9afeb88c67bbc663b9f27385c496da056d06ad87f55df6e393e1516cfecb0461", -                "sha256:9d62cc7880110e4f83b0a51d218f465d3095e2751fbddd34e553dbd106a929ff", -                "sha256:a1fadd062fc8d647ff39220c57ea2b48c99bb73f18223828ec97f88fc27e7898", -                "sha256:a7754a783b1e5d6f627c19d099b178059c62f782ab62b4d8ba165b9fbc2ee34c", -                "sha256:aa59dd63bb3f736de4fc2d080114429d5d369dfb3265f771778e8349d67a97a4", -                "sha256:ae2ee0992f8de249715435942137843a93db204dd7db1e7cc9bdc5a8436443e8", -                "sha256:b36842d7cf32929d568f37ec5b3173b72b2ec6572dec4d6be6ce774762215aee", -                "sha256:bcbf9379c553b5facc6c04c1e5569b44b38ff16bcbf354676287698d61ee0c92", -                "sha256:cbccbda6f1c62ab460449d9c85fdf24d0d32a6bf45176581151e53cc26a5d910", -                "sha256:d0caf98dfb8af395d6732bd16561c0a2458851bea522e39f12f04802dbf6f502", -                "sha256:d6456afeddba036def1a36d8a2758eca53202308d83db20ab5d0b66590919627", -                "sha256:dbaef9a21a4f10bc281684ee4124f169e62bb533c2a92b55f8c06f64f9af7b8f", -                "sha256:dce84916c09aaece006272b37234ae84a8ed13abb3a4d341a23933b8701abfb5", -                "sha256:eb8c9c8b9869539d58d60ff4a28373a22514d40495911451343971cb4835b7a9", -                "sha256:efc98b14ee3a8595e40b1425e8d42f5fd26f11a7b215a81ef9259068931754f4", -                "sha256:fa2dc05b87d97acc1c6ae63f3e0f39eae5246565232484b08db6bf2dc1580678", -                "sha256:fe7d6ce9f6a5fbe24f09d95ea93e9c7271abc4e1565da511e1449b107b4d7848" -            ], -            "version": "==1.0.1" +                "sha256:06a039208f83744a702279b894c8cf24c14fd63c59cd917dcde168b79eef0680", +                "sha256:0a909bf501459062aa1552be1461456518f367379fdc9fdb1f2ca5e4a1fdd7c0", +                "sha256:18402d9e54fb278cb9a8c638df6f1550aca36a009d47ecf5aa263a38600f35b0", +                "sha256:1e4cbbc3858ec7e680006e5ca590d89a5e083235988f26a004acf7244389ac01", +                "sha256:23344e3c2177baf6975fbfa361ed92eb7d36d08f454636e5054b3faa7c2aff8a", +                "sha256:289b31885b4996ce04cadfd5fc03d034dce8e2a8234479f7c9e23b9e245db06b", +                "sha256:2c1c570ae7bf1bab304f29427e2475fe1856814312c4a1cf1cd0ee133f07a3c6", +                "sha256:2c227c0ed371771ffda256034427320870e8ea2e4fd0c0a618c766e7c49aad73", +                "sha256:3bb9b63d319402cead8bbd9dd55dca3b667d2997e9a0d8a1f9b6cc274db4baee", +                "sha256:3ef2183de67b59930d2db8b8e8d4d58e00a50fcc5e92f4f678f6eed7a1c72d55", +                "sha256:43b8ed3dbfd9171e44c554cb4acf4ee4505caa84c5e341858b50ea27dd2b6e12", +                "sha256:47bcf3c5e6c1e87ceb86cdda2ee983fa0fe56a999e6185099b3c93a223f2fa9b", +                "sha256:5263db1e2e1e8ae30500cdd75a979ff99dcc184201e6b4b820d0de74834d2323", +                "sha256:5b1451727f02e7acbdf6aae4e06d75f66ee82966ff9114550381c3271a90f56c", +                "sha256:6996883a8a6ff9117cbb3d6f5b0dcbbae6fb9e31e1a3e4e2f95e0214d9a1c655", +                "sha256:6c96f64a54f030366657a54bb90b3093afc9c16c8e0dfa29fc0d6dbe169103a5", +                "sha256:7332d5c3e35154cd234fd79573736ddcf7a0ade7a986db35b6196b9171493e75", +                "sha256:7885b6f32c4a898e825bb7f56f36a02781ac4a951c63e4169f0afcf9c8c30dfb", +                "sha256:7b0f63f10a166583ab744a58baad04e0f52cfea1ac27bfa1b0c21a48d1003c23", +                "sha256:819f95d4eba3f9e484dd115ab7ab72845cf766b84286a00d4ecf76d33f1edca1", +                "sha256:8968eeaa4d37a38f8ca1f9dbe53526b69628edc9c42229a5b2f56d98bb828c1f", +                "sha256:89ebf69cb19a33d625db72d2ac589d26e936b8f7628531269accf4a3196e7872", +                "sha256:8daecd778c1da45b8bd54fd41ffcd471a86beed3d8e57a43acf7a8d63bba4058", +                "sha256:955ba8ea73cf3ed8bd2f963b4cb9f8f0dcb27becd2f4b3dd536fd24c45533454", +                "sha256:964f18a59f5a64c0170f684c417f4fe3e695a536612e13074c4dd5d1c6d7c882", +                "sha256:969843fbdfbf56cdb71da6f0bdf50f9985b8b8aeb630102945306cf10a9c6af2", +                "sha256:996021ef33e0f50b97ff2d6b5f422a0fe5577de21a8873b58a779a5ddd1c3132", +                "sha256:9e9c9078a7ce07e6fce366bd818be89365a35d2e4b163268f0ca9ba7e13bb2f6", +                "sha256:a04901757cb0fb0f5602ac11dda48f5510f94372144d06c2563ba56c480b467c", +                "sha256:a7bf1492429f18d205f3a818da3ff1f242f60aa59006e53dee00b4ef592a3363", +                "sha256:aa0af2deb166a5e26e0d554b824605e660039b161e37ed4f01b8d04beec184f3", +                "sha256:abfb15a6a7822f0fae681785cb38860e7a2cb1616a708d53df557b3d76c5bfd4", +                "sha256:b253fe4df2afea4dfa6b1fa8c5fef212aff8bcaaeb4207e81eed05cb5e4a7919", +                "sha256:b27f082f47d23cffc4cf1388b84fdc45c4ef6015f906cd7e0d988d9e35d36349", +                "sha256:b33aea449e7f46738811fbc6f0b3177c6777a572207412bbbf6f525ffed001ae", +                "sha256:b44f9421c4505c548435244d74037618f452844c5d3c67719d8a55e2613549da", +                "sha256:bcc371151d1512201d0214c36c0c150b1dc64f19c2b1a8c9cb1d7c7c15ebd93f", +                "sha256:c2851deeabd96d3f6283e9c6b26e0bfed4de2dc6fb15edf913e78b79fc5909ed", +                "sha256:cdfd501c7ac5b198c15df800a3a34c38345f5182e5f80770caf362bccca65628", +                "sha256:d2c0caffa47606d6d7c8af94ba42547bd2a441f06c74fd90a1ffe328524a6c64", +                "sha256:dcb2db95e629962db5a355047fb8aefb012df6c8ae608930d391619dbd96fd86", +                "sha256:e0eeb9c112fec2031927a1745788a181d0eecbacbed941fc5c4f7bc3f7b273bf", +                "sha256:e154891263306200260d7f3051982774d7b9ef35af3509d5adbbe539afd2610c", +                "sha256:e2e023a42dcbab8ed31f97c2bcdb980b7fbe0ada34037d87ba9d799664b58ded", +                "sha256:e64be68255234bb489a574c4f2f8df7029c98c81ec4d160d6cd836e7f0679390", +                "sha256:e82d6b930e02e80e5109b678c663a9ed210680ded81c1abaf54635d88d1da298" +            ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", +            "version": "==1.1.0"          },          "humanfriendly": {              "hashes": [                  "sha256:bf52ec91244819c780341a3438d5d7b09f431d3f113a475147ac9b7b167a3d12",                  "sha256:e78960b31198511f45fd455534ae7645a6207d33e512d2e842c766d15d9c8080"              ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",              "version": "==8.2"          },          "idna": {              "hashes": [ -                "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", -                "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" +                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", +                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"              ], -            "version": "==2.9" +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", +            "version": "==2.10"          },          "imagesize": {              "hashes": [                  "sha256:6965f19a6a2039c7d48bca7dba2473069ff854c36ae6f19d2cde309d998228a1",                  "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1"              ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",              "version": "==1.2.0"          },          "jinja2": { @@ -292,40 +308,45 @@                  "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0",                  "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035"              ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",              "version": "==2.11.2"          },          "lxml": {              "hashes": [ -                "sha256:06748c7192eab0f48e3d35a7adae609a329c6257495d5e53878003660dc0fec6", -                "sha256:0790ddca3f825dd914978c94c2545dbea5f56f008b050e835403714babe62a5f", -                "sha256:1aa7a6197c1cdd65d974f3e4953764eee3d9c7b67e3966616b41fab7f8f516b7", -                "sha256:22c6d34fdb0e65d5f782a4d1a1edb52e0a8365858dafb1c08cb1d16546cf0786", -                "sha256:2754d4406438c83144f9ffd3628bbe2dcc6d62b20dbc5c1ec4bc4385e5d44b42", -                "sha256:27ee0faf8077c7c1a589573b1450743011117f1aa1a91d5ae776bbc5ca6070f2", -                "sha256:2b02c106709466a93ed424454ce4c970791c486d5fcdf52b0d822a7e29789626", -                "sha256:2d1ddce96cf15f1254a68dba6935e6e0f1fe39247de631c115e84dd404a6f031", -                "sha256:4f282737d187ae723b2633856085c31ae5d4d432968b7f3f478a48a54835f5c4", -                "sha256:51bb4edeb36d24ec97eb3e6a6007be128b720114f9a875d6b370317d62ac80b9", -                "sha256:7eee37c1b9815e6505847aa5e68f192e8a1b730c5c7ead39ff317fde9ce29448", -                "sha256:7fd88cb91a470b383aafad554c3fe1ccf6dfb2456ff0e84b95335d582a799804", -                "sha256:9144ce36ca0824b29ebc2e02ca186e54040ebb224292072250467190fb613b96", -                "sha256:925baf6ff1ef2c45169f548cc85204433e061360bfa7d01e1be7ae38bef73194", -                "sha256:a636346c6c0e1092ffc202d97ec1843a75937d8c98aaf6771348ad6422e44bb0", -                "sha256:a87dbee7ad9dce3aaefada2081843caf08a44a8f52e03e0a4cc5819f8398f2f4", -                "sha256:a9e3b8011388e7e373565daa5e92f6c9cb844790dc18e43073212bb3e76f7007", -                "sha256:afb53edf1046599991fb4a7d03e601ab5f5422a5435c47ee6ba91ec3b61416a6", -                "sha256:b26719890c79a1dae7d53acac5f089d66fd8cc68a81f4e4bd355e45470dc25e1", -                "sha256:b7462cdab6fffcda853338e1741ce99706cdf880d921b5a769202ea7b94e8528", -                "sha256:b77975465234ff49fdad871c08aa747aae06f5e5be62866595057c43f8d2f62c", -                "sha256:c47a8a5d00060122ca5908909478abce7bbf62d812e3fc35c6c802df8fb01fe7", -                "sha256:c79e5debbe092e3c93ca4aee44c9a7631bdd407b2871cb541b979fd350bbbc29", -                "sha256:d8d40e0121ca1606aa9e78c28a3a7d88a05c06b3ca61630242cded87d8ce55fa", -                "sha256:ee2be8b8f72a2772e72ab926a3bccebf47bb727bda41ae070dc91d1fb759b726", -                "sha256:f95d28193c3863132b1f55c1056036bf580b5a488d908f7d22a04ace8935a3a9", -                "sha256:fadd2a63a2bfd7fb604508e553d1cf68eca250b2fbdbd81213b5f6f2fbf23529" -            ], -            "index": "pypi", -            "version": "==4.5.1" +                "sha256:05a444b207901a68a6526948c7cc8f9fe6d6f24c70781488e32fd74ff5996e3f", +                "sha256:08fc93257dcfe9542c0a6883a25ba4971d78297f63d7a5a26ffa34861ca78730", +                "sha256:107781b213cf7201ec3806555657ccda67b1fccc4261fb889ef7fc56976db81f", +                "sha256:121b665b04083a1e85ff1f5243d4a93aa1aaba281bc12ea334d5a187278ceaf1", +                "sha256:1fa21263c3aba2b76fd7c45713d4428dbcc7644d73dcf0650e9d344e433741b3", +                "sha256:2b30aa2bcff8e958cd85d907d5109820b01ac511eae5b460803430a7404e34d7", +                "sha256:4b4a111bcf4b9c948e020fd207f915c24a6de3f1adc7682a2d92660eb4e84f1a", +                "sha256:5591c4164755778e29e69b86e425880f852464a21c7bb53c7ea453bbe2633bbe", +                "sha256:59daa84aef650b11bccd18f99f64bfe44b9f14a08a28259959d33676554065a1", +                "sha256:5a9c8d11aa2c8f8b6043d845927a51eb9102eb558e3f936df494e96393f5fd3e", +                "sha256:5dd20538a60c4cc9a077d3b715bb42307239fcd25ef1ca7286775f95e9e9a46d", +                "sha256:74f48ec98430e06c1fa8949b49ebdd8d27ceb9df8d3d1c92e1fdc2773f003f20", +                "sha256:786aad2aa20de3dbff21aab86b2fb6a7be68064cbbc0219bde414d3a30aa47ae", +                "sha256:7ad7906e098ccd30d8f7068030a0b16668ab8aa5cda6fcd5146d8d20cbaa71b5", +                "sha256:80a38b188d20c0524fe8959c8ce770a8fdf0e617c6912d23fc97c68301bb9aba", +                "sha256:8f0ec6b9b3832e0bd1d57af41f9238ea7709bbd7271f639024f2fc9d3bb01293", +                "sha256:92282c83547a9add85ad658143c76a64a8d339028926d7dc1998ca029c88ea6a", +                "sha256:94150231f1e90c9595ccc80d7d2006c61f90a5995db82bccbca7944fd457f0f6", +                "sha256:9dc9006dcc47e00a8a6a029eb035c8f696ad38e40a27d073a003d7d1443f5d88", +                "sha256:a76979f728dd845655026ab991df25d26379a1a8fc1e9e68e25c7eda43004bed", +                "sha256:aa8eba3db3d8761db161003e2d0586608092e217151d7458206e243be5a43843", +                "sha256:bea760a63ce9bba566c23f726d72b3c0250e2fa2569909e2d83cda1534c79443", +                "sha256:c3f511a3c58676147c277eff0224c061dd5a6a8e1373572ac817ac6324f1b1e0", +                "sha256:c9d317efde4bafbc1561509bfa8a23c5cab66c44d49ab5b63ff690f5159b2304", +                "sha256:cc411ad324a4486b142c41d9b2b6a722c534096963688d879ea6fa8a35028258", +                "sha256:cdc13a1682b2a6241080745b1953719e7fe0850b40a5c71ca574f090a1391df6", +                "sha256:cfd7c5dd3c35c19cec59c63df9571c67c6d6e5c92e0fe63517920e97f61106d1", +                "sha256:e1cacf4796b20865789083252186ce9dc6cc59eca0c2e79cca332bdff24ac481", +                "sha256:e70d4e467e243455492f5de463b72151cc400710ac03a0678206a5f27e79ddef", +                "sha256:ecc930ae559ea8a43377e8b60ca6f8d61ac532fc57efb915d899de4a67928efd", +                "sha256:f161af26f596131b63b236372e4ce40f3167c1b5b5d459b29d2514bd8c9dc9ee" +            ], +            "index": "pypi", +            "version": "==4.5.2"          },          "markdownify": {              "hashes": [ @@ -370,15 +391,16 @@                  "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7",                  "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"              ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",              "version": "==1.1.1"          },          "more-itertools": {              "hashes": [ -                "sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be", -                "sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982" +                "sha256:68c70cc7167bdf5c7c9d8f6954a7837089c6a36bf565383919bb595efb8a17e5", +                "sha256:b78134b2063dd214000685165d81c154522c3ee0a1c0d4d113c80361c234c5a2"              ],              "index": "pypi", -            "version": "==8.3.0" +            "version": "==8.4.0"          },          "multidict": {              "hashes": [ @@ -400,19 +422,22 @@                  "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255",                  "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d"              ], +            "markers": "python_version >= '3.5'",              "version": "==4.7.6"          },          "ordered-set": {              "hashes": [ -                "sha256:a31008c57f9c9776b12eb8841b1f61d1e4d70dfbbe8875ccfa2403c54af3d51b" +                "sha256:ba93b2df055bca202116ec44b9bead3df33ea63a7d5827ff8e16738b97f33a95"              ], -            "version": "==4.0.1" +            "markers": "python_version >= '3.5'", +            "version": "==4.0.2"          },          "packaging": {              "hashes": [                  "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8",                  "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"              ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",              "version": "==20.4"          },          "pamqp": { @@ -461,6 +486,7 @@                  "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",                  "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"              ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",              "version": "==2.20"          },          "pygments": { @@ -468,6 +494,7 @@                  "sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44",                  "sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324"              ], +            "markers": "python_version >= '3.5'",              "version": "==2.6.1"          },          "pyparsing": { @@ -475,6 +502,7 @@                  "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",                  "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"              ], +            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",              "version": "==2.4.7"          },          "python-dateutil": { @@ -511,32 +539,34 @@          },          "redis": {              "hashes": [ -                "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", -                "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" +                "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2", +                "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24"              ], -            "version": "==3.5.2" +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", +            "version": "==3.5.3"          },          "requests": {              "hashes": [ -                "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", -                "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6" +                "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", +                "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"              ],              "index": "pypi", -            "version": "==2.23.0" +            "version": "==2.24.0"          },          "sentry-sdk": {              "hashes": [ -                "sha256:0e5e947d0f7a969314aa23669a94a9712be5a688ff069ff7b9fc36c66adc160c", -                "sha256:799a8bf76b012e3030a881be00e97bc0b922ce35dde699c6537122b751d80e2c" +                "sha256:21b17d6aa064c0fb703a7c00f77cf6c9c497cf2f83345c28892980a5e742d116", +                "sha256:4fc97114c77d005467b9b1a29f042e2bc01923cb683b0ef0bbda46e79fa12532"              ],              "index": "pypi", -            "version": "==0.14.4" +            "version": "==0.16.3"          },          "six": {              "hashes": [                  "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",                  "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"              ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",              "version": "==1.15.0"          },          "snowballstemmer": { @@ -548,16 +578,17 @@          },          "sortedcontainers": {              "hashes": [ -                "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", -                "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" +                "sha256:4e73a757831fc3ca4de2859c422564239a31d8213d09a2a666e375807034d2ba", +                "sha256:c633ebde8580f241f274c1f8994a665c0e54a17724fecd0cae2f079e09c36d3f"              ], -            "version": "==2.1.0" +            "version": "==2.2.2"          },          "soupsieve": {              "hashes": [                  "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",                  "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"              ], +            "markers": "python_version >= '3.5'",              "version": "==2.0.1"          },          "sphinx": { @@ -573,6 +604,7 @@                  "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a",                  "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58"              ], +            "markers": "python_version >= '3.5'",              "version": "==1.0.2"          },          "sphinxcontrib-devhelp": { @@ -580,6 +612,7 @@                  "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e",                  "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4"              ], +            "markers": "python_version >= '3.5'",              "version": "==1.0.2"          },          "sphinxcontrib-htmlhelp": { @@ -587,6 +620,7 @@                  "sha256:3c0bc24a2c41e340ac37c85ced6dafc879ab485c095b1d65d2461ac2f7cca86f",                  "sha256:e8f5bb7e31b2dbb25b9cc435c8ab7a79787ebf7f906155729338f3156d93659b"              ], +            "markers": "python_version >= '3.5'",              "version": "==1.0.3"          },          "sphinxcontrib-jsmath": { @@ -594,6 +628,7 @@                  "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178",                  "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"              ], +            "markers": "python_version >= '3.5'",              "version": "==1.0.1"          },          "sphinxcontrib-qthelp": { @@ -601,6 +636,7 @@                  "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72",                  "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6"              ], +            "markers": "python_version >= '3.5'",              "version": "==1.0.3"          },          "sphinxcontrib-serializinghtml": { @@ -608,6 +644,7 @@                  "sha256:eaa0eccc86e982a9b939b2b82d12cc5d013385ba5eadcc7e4fed23f4405f77bc",                  "sha256:f242a81d423f59617a8e5cf16f5d4d74e28ee9a66f9e5b637a18082991db5a9a"              ], +            "markers": "python_version >= '3.5'",              "version": "==1.1.4"          },          "statsd": { @@ -620,59 +657,34 @@          },          "urllib3": {              "hashes": [ -                "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", -                "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" -            ], -            "version": "==1.25.9" -        }, -        "websockets": { -            "hashes": [ -                "sha256:0e4fb4de42701340bd2353bb2eee45314651caa6ccee80dbd5f5d5978888fed5", -                "sha256:1d3f1bf059d04a4e0eb4985a887d49195e15ebabc42364f4eb564b1d065793f5", -                "sha256:20891f0dddade307ffddf593c733a3fdb6b83e6f9eef85908113e628fa5a8308", -                "sha256:295359a2cc78736737dd88c343cd0747546b2174b5e1adc223824bcaf3e164cb", -                "sha256:2db62a9142e88535038a6bcfea70ef9447696ea77891aebb730a333a51ed559a", -                "sha256:3762791ab8b38948f0c4d281c8b2ddfa99b7e510e46bd8dfa942a5fff621068c", -                "sha256:3db87421956f1b0779a7564915875ba774295cc86e81bc671631379371af1170", -                "sha256:3ef56fcc7b1ff90de46ccd5a687bbd13a3180132268c4254fc0fa44ecf4fc422", -                "sha256:4f9f7d28ce1d8f1295717c2c25b732c2bc0645db3215cf757551c392177d7cb8", -                "sha256:5c01fd846263a75bc8a2b9542606927cfad57e7282965d96b93c387622487485", -                "sha256:5c65d2da8c6bce0fca2528f69f44b2f977e06954c8512a952222cea50dad430f", -                "sha256:751a556205d8245ff94aeef23546a1113b1dd4f6e4d102ded66c39b99c2ce6c8", -                "sha256:7ff46d441db78241f4c6c27b3868c9ae71473fe03341340d2dfdbe8d79310acc", -                "sha256:965889d9f0e2a75edd81a07592d0ced54daa5b0785f57dc429c378edbcffe779", -                "sha256:9b248ba3dd8a03b1a10b19efe7d4f7fa41d158fdaa95e2cf65af5a7b95a4f989", -                "sha256:9bef37ee224e104a413f0780e29adb3e514a5b698aabe0d969a6ba426b8435d1", -                "sha256:c1ec8db4fac31850286b7cd3b9c0e1b944204668b8eb721674916d4e28744092", -                "sha256:c8a116feafdb1f84607cb3b14aa1418424ae71fee131642fc568d21423b51824", -                "sha256:ce85b06a10fc65e6143518b96d3dca27b081a740bae261c2fb20375801a9d56d", -                "sha256:d705f8aeecdf3262379644e4b55107a3b55860eb812b673b28d0fbc347a60c55", -                "sha256:e898a0863421650f0bebac8ba40840fc02258ef4714cb7e1fd76b6a6354bda36", -                "sha256:f8a7bff6e8664afc4e6c28b983845c5bc14965030e3fb98789734d416af77c4b" -            ], -            "version": "==8.1" +                "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a", +                "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461" +            ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", +            "version": "==1.25.10"          },          "yarl": {              "hashes": [ -                "sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce", -                "sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6", -                "sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce", -                "sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae", -                "sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d", -                "sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f", -                "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b", -                "sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b", -                "sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb", -                "sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462", -                "sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea", -                "sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70", -                "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1", -                "sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a", -                "sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b", -                "sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080", -                "sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2" -            ], -            "version": "==1.4.2" +                "sha256:040b237f58ff7d800e6e0fd89c8439b841f777dd99b4a9cca04d6935564b9409", +                "sha256:17668ec6722b1b7a3a05cc0167659f6c95b436d25a36c2d52db0eca7d3f72593", +                "sha256:3a584b28086bc93c888a6c2aa5c92ed1ae20932f078c46509a66dce9ea5533f2", +                "sha256:4439be27e4eee76c7632c2427ca5e73703151b22cae23e64adb243a9c2f565d8", +                "sha256:48e918b05850fffb070a496d2b5f97fc31d15d94ca33d3d08a4f86e26d4e7c5d", +                "sha256:9102b59e8337f9874638fcfc9ac3734a0cfadb100e47d55c20d0dc6087fb4692", +                "sha256:9b930776c0ae0c691776f4d2891ebc5362af86f152dd0da463a6614074cb1b02", +                "sha256:b3b9ad80f8b68519cc3372a6ca85ae02cc5a8807723ac366b53c0f089db19e4a", +                "sha256:bc2f976c0e918659f723401c4f834deb8a8e7798a71be4382e024bcc3f7e23a8", +                "sha256:c22c75b5f394f3d47105045ea551e08a3e804dc7e01b37800ca35b58f856c3d6", +                "sha256:c52ce2883dc193824989a9b97a76ca86ecd1fa7955b14f87bf367a61b6232511", +                "sha256:ce584af5de8830d8701b8979b18fcf450cef9a382b1a3c8ef189bedc408faf1e", +                "sha256:da456eeec17fa8aa4594d9a9f27c0b1060b6a75f2419fe0c00609587b2695f4a", +                "sha256:db6db0f45d2c63ddb1a9d18d1b9b22f308e52c83638c26b422d520a815c4b3fb", +                "sha256:df89642981b94e7db5596818499c4b2219028f2a528c9c37cc1de45bf2fd3a3f", +                "sha256:f18d68f2be6bf0e89f1521af2b1bb46e66ab0018faafa81d70f358153170a317", +                "sha256:f379b7f83f23fe12823085cd6b906edc49df969eb99757f58ff382349a3303c6" +            ], +            "markers": "python_version >= '3.5'", +            "version": "==1.5.1"          }      },      "develop": { @@ -688,57 +700,63 @@                  "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",                  "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"              ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",              "version": "==19.3.0"          },          "cfgv": {              "hashes": [ -                "sha256:1ccf53320421aeeb915275a196e23b3b8ae87dea8ac6698b1638001d4a486d53", -                "sha256:c8e8f552ffcc6194f4e18dd4f68d9aef0c0d58ae7e7be8c82bee3c5e9edfa513" +                "sha256:32e43d604bbe7896fe7c248a9c2276447dbef840feb28fe20494f62af110211d", +                "sha256:cf22deb93d4bcf92f345a5c3cd39d3d41d6340adc60c78bbbd6588c384fda6a1"              ], -            "version": "==3.1.0" +            "markers": "python_full_version >= '3.6.1'", +            "version": "==3.2.0"          },          "coverage": {              "hashes": [ -                "sha256:00f1d23f4336efc3b311ed0d807feb45098fc86dee1ca13b3d6768cdab187c8a", -                "sha256:01333e1bd22c59713ba8a79f088b3955946e293114479bbfc2e37d522be03355", -                "sha256:0cb4be7e784dcdc050fc58ef05b71aa8e89b7e6636b99967fadbdba694cf2b65", -                "sha256:0e61d9803d5851849c24f78227939c701ced6704f337cad0a91e0972c51c1ee7", -                "sha256:1601e480b9b99697a570cea7ef749e88123c04b92d84cedaa01e117436b4a0a9", -                "sha256:2742c7515b9eb368718cd091bad1a1b44135cc72468c731302b3d641895b83d1", -                "sha256:2d27a3f742c98e5c6b461ee6ef7287400a1956c11421eb574d843d9ec1f772f0", -                "sha256:402e1744733df483b93abbf209283898e9f0d67470707e3c7516d84f48524f55", -                "sha256:5c542d1e62eece33c306d66fe0a5c4f7f7b3c08fecc46ead86d7916684b36d6c", -                "sha256:5f2294dbf7875b991c381e3d5af2bcc3494d836affa52b809c91697449d0eda6", -                "sha256:6402bd2fdedabbdb63a316308142597534ea8e1895f4e7d8bf7476c5e8751fef", -                "sha256:66460ab1599d3cf894bb6baee8c684788819b71a5dc1e8fa2ecc152e5d752019", -                "sha256:782caea581a6e9ff75eccda79287daefd1d2631cc09d642b6ee2d6da21fc0a4e", -                "sha256:79a3cfd6346ce6c13145731d39db47b7a7b859c0272f02cdb89a3bdcbae233a0", -                "sha256:7a5bdad4edec57b5fb8dae7d3ee58622d626fd3a0be0dfceda162a7035885ecf", -                "sha256:8fa0cbc7ecad630e5b0f4f35b0f6ad419246b02bc750de7ac66db92667996d24", -                "sha256:a027ef0492ede1e03a8054e3c37b8def89a1e3c471482e9f046906ba4f2aafd2", -                "sha256:a3f3654d5734a3ece152636aad89f58afc9213c6520062db3978239db122f03c", -                "sha256:a82b92b04a23d3c8a581fc049228bafde988abacba397d57ce95fe95e0338ab4", -                "sha256:acf3763ed01af8410fc36afea23707d4ea58ba7e86a8ee915dfb9ceff9ef69d0", -                "sha256:adeb4c5b608574a3d647011af36f7586811a2c1197c861aedb548dd2453b41cd", -                "sha256:b83835506dfc185a319031cf853fa4bb1b3974b1f913f5bb1a0f3d98bdcded04", -                "sha256:bb28a7245de68bf29f6fb199545d072d1036a1917dca17a1e75bbb919e14ee8e", -                "sha256:bf9cb9a9fd8891e7efd2d44deb24b86d647394b9705b744ff6f8261e6f29a730", -                "sha256:c317eaf5ff46a34305b202e73404f55f7389ef834b8dbf4da09b9b9b37f76dd2", -                "sha256:dbe8c6ae7534b5b024296464f387d57c13caa942f6d8e6e0346f27e509f0f768", -                "sha256:de807ae933cfb7f0c7d9d981a053772452217df2bf38e7e6267c9cbf9545a796", -                "sha256:dead2ddede4c7ba6cb3a721870f5141c97dc7d85a079edb4bd8d88c3ad5b20c7", -                "sha256:dec5202bfe6f672d4511086e125db035a52b00f1648d6407cc8e526912c0353a", -                "sha256:e1ea316102ea1e1770724db01998d1603ed921c54a86a2efcb03428d5417e489", -                "sha256:f90bfc4ad18450c80b024036eaf91e4a246ae287701aaa88eaebebf150868052" -            ], -            "index": "pypi", -            "version": "==5.1" +                "sha256:098a703d913be6fbd146a8c50cc76513d726b022d170e5e98dc56d958fd592fb", +                "sha256:16042dc7f8e632e0dcd5206a5095ebd18cb1d005f4c89694f7f8aafd96dd43a3", +                "sha256:1adb6be0dcef0cf9434619d3b892772fdb48e793300f9d762e480e043bd8e716", +                "sha256:27ca5a2bc04d68f0776f2cdcb8bbd508bbe430a7bf9c02315cd05fb1d86d0034", +                "sha256:28f42dc5172ebdc32622a2c3f7ead1b836cdbf253569ae5673f499e35db0bac3", +                "sha256:2fcc8b58953d74d199a1a4d633df8146f0ac36c4e720b4a1997e9b6327af43a8", +                "sha256:304fbe451698373dc6653772c72c5d5e883a4aadaf20343592a7abb2e643dae0", +                "sha256:30bc103587e0d3df9e52cd9da1dd915265a22fad0b72afe54daf840c984b564f", +                "sha256:40f70f81be4d34f8d491e55936904db5c527b0711b2a46513641a5729783c2e4", +                "sha256:4186fc95c9febeab5681bc3248553d5ec8c2999b8424d4fc3a39c9cba5796962", +                "sha256:46794c815e56f1431c66d81943fa90721bb858375fb36e5903697d5eef88627d", +                "sha256:4869ab1c1ed33953bb2433ce7b894a28d724b7aa76c19b11e2878034a4e4680b", +                "sha256:4f6428b55d2916a69f8d6453e48a505c07b2245653b0aa9f0dee38785939f5e4", +                "sha256:52f185ffd3291196dc1aae506b42e178a592b0b60a8610b108e6ad892cfc1bb3", +                "sha256:538f2fd5eb64366f37c97fdb3077d665fa946d2b6d95447622292f38407f9258", +                "sha256:64c4f340338c68c463f1b56e3f2f0423f7b17ba6c3febae80b81f0e093077f59", +                "sha256:675192fca634f0df69af3493a48224f211f8db4e84452b08d5fcebb9167adb01", +                "sha256:700997b77cfab016533b3e7dbc03b71d33ee4df1d79f2463a318ca0263fc29dd", +                "sha256:8505e614c983834239f865da2dd336dcf9d72776b951d5dfa5ac36b987726e1b", +                "sha256:962c44070c281d86398aeb8f64e1bf37816a4dfc6f4c0f114756b14fc575621d", +                "sha256:9e536783a5acee79a9b308be97d3952b662748c4037b6a24cbb339dc7ed8eb89", +                "sha256:9ea749fd447ce7fb1ac71f7616371f04054d969d412d37611716721931e36efd", +                "sha256:a34cb28e0747ea15e82d13e14de606747e9e484fb28d63c999483f5d5188e89b", +                "sha256:a3ee9c793ffefe2944d3a2bd928a0e436cd0ac2d9e3723152d6fd5398838ce7d", +                "sha256:aab75d99f3f2874733946a7648ce87a50019eb90baef931698f96b76b6769a46", +                "sha256:b1ed2bdb27b4c9fc87058a1cb751c4df8752002143ed393899edb82b131e0546", +                "sha256:b360d8fd88d2bad01cb953d81fd2edd4be539df7bfec41e8753fe9f4456a5082", +                "sha256:b8f58c7db64d8f27078cbf2a4391af6aa4e4767cc08b37555c4ae064b8558d9b", +                "sha256:c1bbb628ed5192124889b51204de27c575b3ffc05a5a91307e7640eff1d48da4", +                "sha256:c2ff24df02a125b7b346c4c9078c8936da06964cc2d276292c357d64378158f8", +                "sha256:c890728a93fffd0407d7d37c1e6083ff3f9f211c83b4316fae3778417eab9811", +                "sha256:c96472b8ca5dc135fb0aa62f79b033f02aa434fb03a8b190600a5ae4102df1fd", +                "sha256:ce7866f29d3025b5b34c2e944e66ebef0d92e4a4f2463f7266daa03a1332a651", +                "sha256:e26c993bd4b220429d4ec8c1468eca445a4064a61c74ca08da7429af9bc53bb0" +            ], +            "index": "pypi", +            "version": "==5.2.1"          },          "distlib": {              "hashes": [ -                "sha256:2e166e231a26b36d6dfe35a48c4464346620f8645ed0ace01ee31822b288de21" +                "sha256:8c09de2c67b3e7deef7184574fc060ab8a793e7adbb183d942c389c8b13c52fb", +                "sha256:edf6116872c863e1aa9d5bb7cb5e05a022c519a4594dc703843343a9ddd9bff1"              ], -            "version": "==0.3.0" +            "version": "==0.3.1"          },          "filelock": {              "hashes": [ @@ -749,19 +767,19 @@          },          "flake8": {              "hashes": [ -                "sha256:c69ac1668e434d37a2d2880b3ca9aafd54b3a10a3ac1ab101d22f29e29cf8634", -                "sha256:ccaa799ef9893cebe69fdfefed76865aeaefbb94cb8545617b2298786a4de9a5" +                "sha256:15e351d19611c887e482fb960eae4d44845013cc142d42896e9862f775d8cf5c", +                "sha256:f04b9fcbac03b0a3e58c0ab3a0ecc462e023a9faf046d57794184028123aa208"              ],              "index": "pypi", -            "version": "==3.8.2" +            "version": "==3.8.3"          },          "flake8-annotations": {              "hashes": [ -                "sha256:9091d920406a7ff10e401e0dd1baa396d1d7d2e3d101a9beecf815f5894ad554", -                "sha256:f59fdceb8c8f380a20aed20e1ba8a57bde05935958166c52be2249f113f7ab75" +                "sha256:7816a5d8f65ffdf37b8e21e5b17e0fd1e492aa92638573276de066e889a22b26", +                "sha256:8d18db74a750dd97f40b483cc3ef80d07d03f687525bad8fd83365dcd3bfd414"              ],              "index": "pypi", -            "version": "==2.1.0" +            "version": "==2.3.0"          },          "flake8-bugbear": {              "hashes": [ @@ -819,10 +837,11 @@          },          "identify": {              "hashes": [ -                "sha256:0f3c3aac62b51b86fea6ff52fe8ff9e06f57f10411502443809064d23e16f1c2", -                "sha256:f9ad3d41f01e98eb066b6e05c5b184fd1e925fadec48eb165b4e01c72a1ef3a7" +                "sha256:110ed090fec6bce1aabe3c72d9258a9de82207adeaa5a05cd75c635880312f9a", +                "sha256:ccd88716b890ecbe10920659450a635d2d25de499b9a638525a48b48261d989b"              ], -            "version": "==1.4.16" +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", +            "version": "==1.4.25"          },          "mccabe": {              "hashes": [ @@ -833,31 +852,32 @@          },          "nodeenv": {              "hashes": [ -                "sha256:5b2438f2e42af54ca968dd1b374d14a1194848955187b0e5e4be1f73813a5212" +                "sha256:4b0b77afa3ba9b54f4b6396e60b0c83f59eaeb2d63dc3cc7a70f7f4af96c82bc"              ], -            "version": "==1.3.5" +            "version": "==1.4.0"          },          "pep8-naming": {              "hashes": [ -                "sha256:5d9f1056cb9427ce344e98d1a7f5665710e2f20f748438e308995852cfa24164", -                "sha256:f3b4a5f9dd72b991bf7d8e2a341d2e1aa3a884a769b5aaac4f56825c1763bf3a" +                "sha256:a1dd47dd243adfe8a83616e27cf03164960b507530f155db94e10b36a6cd6724", +                "sha256:f43bfe3eea7e0d73e8b5d07d6407ab47f2476ccaeff6937c84275cd30b016738"              ],              "index": "pypi", -            "version": "==0.10.0" +            "version": "==0.11.1"          },          "pre-commit": {              "hashes": [ -                "sha256:5559e09afcac7808933951ffaf4ff9aac524f31efbc3f24d021540b6c579813c", -                "sha256:703e2e34cbe0eedb0d319eff9f7b83e2022bb5a3ab5289a6a8841441076514d0" +                "sha256:1657663fdd63a321a4a739915d7d03baedd555b25054449090f97bb0cb30a915", +                "sha256:e8b1315c585052e729ab7e99dcca5698266bedce9067d21dc909c23e3ceed626"              ],              "index": "pypi", -            "version": "==2.4.0" +            "version": "==2.6.0"          },          "pycodestyle": {              "hashes": [                  "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367",                  "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"              ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",              "version": "==2.6.0"          },          "pydocstyle": { @@ -865,6 +885,7 @@                  "sha256:da7831660b7355307b32778c4a0dbfb137d89254ef31a2b2978f50fc0b4d7586",                  "sha256:f4f5d210610c2d153fae39093d44224c17429e2ad7da12a8b419aba5c2f614b5"              ], +            "markers": "python_version >= '3.5'",              "version": "==5.0.2"          },          "pyflakes": { @@ -872,6 +893,7 @@                  "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92",                  "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"              ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",              "version": "==2.2.0"          },          "pyyaml": { @@ -896,6 +918,7 @@                  "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",                  "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"              ], +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",              "version": "==1.15.0"          },          "snowballstemmer": { @@ -922,10 +945,11 @@          },          "virtualenv": {              "hashes": [ -                "sha256:a116629d4e7f4d03433b8afa27f43deba09d48bc48f5ecefa4f015a178efb6cf", -                "sha256:a730548b27366c5e6cbdf6f97406d861cccece2e22275e8e1a757aeff5e00c70" +                "sha256:7b54fd606a1b85f83de49ad8d80dbec08e983a2d2f96685045b262ebc7481ee5", +                "sha256:8cd7b2a4850b003a11be2fc213e206419efab41115cc14bca20e69654f2ac08e"              ], -            "version": "==20.0.21" +            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", +            "version": "==20.0.30"          }      }  } @@ -1,6 +1,6 @@  # Python Utility Bot -[](https://discord.gg/2B963hn) +[](https://discord.gg/2B963hn)  [](https://dev.azure.com/python-discord/Python%20Discord/_build/latest?definitionId=1&branchName=master)  [](https://dev.azure.com/python-discord/Python%20Discord/_apis/build/status/Bot?branchName=master)  [](https://dev.azure.com/python-discord/Python%20Discord/_apis/build/status/Bot?branchName=master) diff --git a/bot/__init__.py b/bot/__init__.py index d63086fe2..3ee70c4e9 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -2,10 +2,14 @@ import asyncio  import logging  import os  import sys +from functools import partial, partialmethod  from logging import Logger, handlers  from pathlib import Path  import coloredlogs +from discord.ext import commands + +from bot.command import Command  TRACE_LEVEL = logging.TRACE = 5  logging.addLevelName(TRACE_LEVEL, "TRACE") @@ -66,3 +70,9 @@ logging.getLogger(__name__)  # On Windows, the selector event loop is required for aiodns.  if os.name == "nt":      asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + + +# Monkey-patch discord.py decorators to use the Command subclass which supports root aliases. +# Must be patched before any cogs are added. +commands.command = partial(commands.command, cls=Command) +commands.GroupMixin.command = partialmethod(commands.GroupMixin.command, cls=Command) diff --git a/bot/__main__.py b/bot/__main__.py index 4e0d4a111..fe2cf90e6 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -24,45 +24,49 @@ sentry_sdk.init(      ]  ) +allowed_roles = [discord.Object(id_) for id_ in constants.MODERATION_ROLES]  bot = Bot(      command_prefix=when_mentioned_or(constants.Bot.prefix),      activity=discord.Game(name="Commands: !help"),      case_insensitive=True,      max_messages=10_000, +    allowed_mentions=discord.AllowedMentions(everyone=False, roles=allowed_roles)  )  # Internal/debug +bot.load_extension("bot.cogs.config_verifier")  bot.load_extension("bot.cogs.error_handler")  bot.load_extension("bot.cogs.filtering")  bot.load_extension("bot.cogs.logging")  bot.load_extension("bot.cogs.security") -bot.load_extension("bot.cogs.config_verifier")  # Commands, etc  bot.load_extension("bot.cogs.antimalware")  bot.load_extension("bot.cogs.antispam")  bot.load_extension("bot.cogs.bot")  bot.load_extension("bot.cogs.clean") +bot.load_extension("bot.cogs.doc")  bot.load_extension("bot.cogs.extensions")  bot.load_extension("bot.cogs.help") - -bot.load_extension("bot.cogs.doc")  bot.load_extension("bot.cogs.verification")  # Feature cogs  bot.load_extension("bot.cogs.alias")  bot.load_extension("bot.cogs.defcon") +bot.load_extension("bot.cogs.dm_relay")  bot.load_extension("bot.cogs.duck_pond")  bot.load_extension("bot.cogs.eval") +bot.load_extension("bot.cogs.filter_lists")  bot.load_extension("bot.cogs.information")  bot.load_extension("bot.cogs.jams")  bot.load_extension("bot.cogs.moderation") -bot.load_extension("bot.cogs.python_news")  bot.load_extension("bot.cogs.off_topic_names") +bot.load_extension("bot.cogs.python_news")  bot.load_extension("bot.cogs.reddit")  bot.load_extension("bot.cogs.reminders")  bot.load_extension("bot.cogs.site")  bot.load_extension("bot.cogs.snekbox") +bot.load_extension("bot.cogs.source")  bot.load_extension("bot.cogs.stats")  bot.load_extension("bot.cogs.sync")  bot.load_extension("bot.cogs.tags") @@ -70,7 +74,6 @@ bot.load_extension("bot.cogs.token_remover")  bot.load_extension("bot.cogs.utils")  bot.load_extension("bot.cogs.watchchannels")  bot.load_extension("bot.cogs.webhook_remover") -bot.load_extension("bot.cogs.wolfram")  if constants.HelpChannels.enable:      bot.load_extension("bot.cogs.help_channels") diff --git a/bot/bot.py b/bot/bot.py index 313652d11..d25074fd9 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -2,7 +2,8 @@ import asyncio  import logging  import socket  import warnings -from typing import Optional +from collections import defaultdict +from typing import Dict, Optional  import aiohttp  import aioredis @@ -34,6 +35,7 @@ class Bot(commands.Bot):          self.redis_ready = asyncio.Event()          self.redis_closed = False          self.api_client = api.APIClient(loop=self.loop) +        self.filter_list_cache = defaultdict(dict)          self._connector = None          self._resolver = None @@ -49,6 +51,13 @@ class Bot(commands.Bot):          self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") +    async def cache_filter_list_data(self) -> None: +        """Cache all the data in the FilterList on the site.""" +        full_cache = await self.api_client.get('bot/filter-lists') + +        for item in full_cache: +            self.insert_item_into_filter_list_cache(item) +      async def _create_redis_session(self) -> None:          """          Create the Redis connection pool, and then open the redis event gate. @@ -73,11 +82,74 @@ class Bot(commands.Bot):          self.redis_closed = False          self.redis_ready.set() +    def _recreate(self) -> None: +        """Re-create the connector, aiohttp session, the APIClient and the Redis session.""" +        # Use asyncio for DNS resolution instead of threads so threads aren't spammed. +        # Doesn't seem to have any state with regards to being closed, so no need to worry? +        self._resolver = aiohttp.AsyncResolver() + +        # Its __del__ does send a warning but it doesn't always show up for some reason. +        if self._connector and not self._connector._closed: +            log.warning( +                "The previous connector was not closed; it will remain open and be overwritten" +            ) + +        if self.redis_session and not self.redis_session.closed: +            log.warning( +                "The previous redis pool was not closed; it will remain open and be overwritten" +            ) + +        # Create the redis session +        self.loop.create_task(self._create_redis_session()) + +        # Use AF_INET as its socket family to prevent HTTPS related problems both locally +        # and in production. +        self._connector = aiohttp.TCPConnector( +            resolver=self._resolver, +            family=socket.AF_INET, +        ) + +        # Client.login() will call HTTPClient.static_login() which will create a session using +        # this connector attribute. +        self.http.connector = self._connector + +        # Its __del__ does send a warning but it doesn't always show up for some reason. +        if self.http_session and not self.http_session.closed: +            log.warning( +                "The previous session was not closed; it will remain open and be overwritten" +            ) + +        self.http_session = aiohttp.ClientSession(connector=self._connector) +        self.api_client.recreate(force=True, connector=self._connector) + +        # Build the FilterList cache +        self.loop.create_task(self.cache_filter_list_data()) +      def add_cog(self, cog: commands.Cog) -> None:          """Adds a "cog" to the bot and logs the operation."""          super().add_cog(cog)          log.info(f"Cog loaded: {cog.qualified_name}") +    def add_command(self, command: commands.Command) -> None: +        """Add `command` as normal and then add its root aliases to the bot.""" +        super().add_command(command) +        self._add_root_aliases(command) + +    def remove_command(self, name: str) -> Optional[commands.Command]: +        """ +        Remove a command/alias as normal and then remove its root aliases from the bot. + +        Individual root aliases cannot be removed by this function. +        To remove them, either remove the entire command or manually edit `bot.all_commands`. +        """ +        command = super().remove_command(name) +        if command is None: +            # Even if it's a root alias, there's no way to get the Bot instance to remove the alias. +            return + +        self._remove_root_aliases(command) +        return command +      def clear(self) -> None:          """          Clears the internal state of the bot and recreates the connector and sessions. @@ -113,52 +185,25 @@ class Bot(commands.Bot):              self.redis_ready.clear()              await self.redis_session.wait_closed() +    def insert_item_into_filter_list_cache(self, item: Dict[str, str]) -> None: +        """Add an item to the bots filter_list_cache.""" +        type_ = item["type"] +        allowed = item["allowed"] +        content = item["content"] + +        self.filter_list_cache[f"{type_}.{allowed}"][content] = { +            "id": item["id"], +            "comment": item["comment"], +            "created_at": item["created_at"], +            "updated_at": item["updated_at"], +        } +      async def login(self, *args, **kwargs) -> None:          """Re-create the connector and set up sessions before logging into Discord."""          self._recreate()          await self.stats.create_socket()          await super().login(*args, **kwargs) -    def _recreate(self) -> None: -        """Re-create the connector, aiohttp session, the APIClient and the Redis session.""" -        # Use asyncio for DNS resolution instead of threads so threads aren't spammed. -        # Doesn't seem to have any state with regards to being closed, so no need to worry? -        self._resolver = aiohttp.AsyncResolver() - -        # Its __del__ does send a warning but it doesn't always show up for some reason. -        if self._connector and not self._connector._closed: -            log.warning( -                "The previous connector was not closed; it will remain open and be overwritten" -            ) - -        if self.redis_session and not self.redis_session.closed: -            log.warning( -                "The previous redis pool was not closed; it will remain open and be overwritten" -            ) - -        # Create the redis session -        self.loop.create_task(self._create_redis_session()) - -        # Use AF_INET as its socket family to prevent HTTPS related problems both locally -        # and in production. -        self._connector = aiohttp.TCPConnector( -            resolver=self._resolver, -            family=socket.AF_INET, -        ) - -        # Client.login() will call HTTPClient.static_login() which will create a session using -        # this connector attribute. -        self.http.connector = self._connector - -        # Its __del__ does send a warning but it doesn't always show up for some reason. -        if self.http_session and not self.http_session.closed: -            log.warning( -                "The previous session was not closed; it will remain open and be overwritten" -            ) - -        self.http_session = aiohttp.ClientSession(connector=self._connector) -        self.api_client.recreate(force=True, connector=self._connector) -      async def on_guild_available(self, guild: discord.Guild) -> None:          """          Set the internal guild available event when constants.Guild.id becomes available. @@ -210,3 +255,24 @@ class Bot(commands.Bot):              scope.set_extra("kwargs", kwargs)              log.exception(f"Unhandled exception in {event}.") + +    def _add_root_aliases(self, command: commands.Command) -> None: +        """Recursively add root aliases for `command` and any of its subcommands.""" +        if isinstance(command, commands.Group): +            for subcommand in command.commands: +                self._add_root_aliases(subcommand) + +        for alias in getattr(command, "root_aliases", ()): +            if alias in self.all_commands: +                raise commands.CommandRegistrationError(alias, alias_conflict=True) + +            self.all_commands[alias] = command + +    def _remove_root_aliases(self, command: commands.Command) -> None: +        """Recursively remove root aliases for `command` and any of its subcommands.""" +        if isinstance(command, commands.Group): +            for subcommand in command.commands: +                self._remove_root_aliases(subcommand) + +        for alias in getattr(command, "root_aliases", ()): +            self.all_commands.pop(alias, None) diff --git a/bot/cogs/alias.py b/bot/cogs/alias.py index 55c7efe65..c6ba8d6f3 100644 --- a/bot/cogs/alias.py +++ b/bot/cogs/alias.py @@ -3,13 +3,12 @@ import logging  from discord import Colour, Embed  from discord.ext.commands import ( -    Cog, Command, Context, Greedy, +    Cog, Command, Context,      clean_content, command, group,  )  from bot.bot import Bot -from bot.cogs.extensions import Extension -from bot.converters import FetchedMember, TagNameConverter +from bot.converters import TagNameConverter  from bot.pagination import LinePaginator  log = logging.getLogger(__name__) @@ -51,56 +50,6 @@ class Alias (Cog):              ctx, embed, empty=False, max_lines=20          ) -    @command(name="resources", aliases=("resource",), hidden=True) -    async def site_resources_alias(self, ctx: Context) -> None: -        """Alias for invoking <prefix>site resources.""" -        await self.invoke(ctx, "site resources") - -    @command(name="tools", hidden=True) -    async def site_tools_alias(self, ctx: Context) -> None: -        """Alias for invoking <prefix>site tools.""" -        await self.invoke(ctx, "site tools") - -    @command(name="watch", hidden=True) -    async def bigbrother_watch_alias(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: -        """Alias for invoking <prefix>bigbrother watch [user] [reason].""" -        await self.invoke(ctx, "bigbrother watch", user, reason=reason) - -    @command(name="unwatch", hidden=True) -    async def bigbrother_unwatch_alias(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: -        """Alias for invoking <prefix>bigbrother unwatch [user] [reason].""" -        await self.invoke(ctx, "bigbrother unwatch", user, reason=reason) - -    @command(name="home", hidden=True) -    async def site_home_alias(self, ctx: Context) -> None: -        """Alias for invoking <prefix>site home.""" -        await self.invoke(ctx, "site home") - -    @command(name="faq", hidden=True) -    async def site_faq_alias(self, ctx: Context) -> None: -        """Alias for invoking <prefix>site faq.""" -        await self.invoke(ctx, "site faq") - -    @command(name="rules", aliases=("rule",), hidden=True) -    async def site_rules_alias(self, ctx: Context, rules: Greedy[int], *_: str) -> None: -        """Alias for invoking <prefix>site rules.""" -        await self.invoke(ctx, "site rules", *rules) - -    @command(name="reload", hidden=True) -    async def extensions_reload_alias(self, ctx: Context, *extensions: Extension) -> None: -        """Alias for invoking <prefix>extensions reload [extensions...].""" -        await self.invoke(ctx, "extensions reload", *extensions) - -    @command(name="defon", hidden=True) -    async def defcon_enable_alias(self, ctx: Context) -> None: -        """Alias for invoking <prefix>defcon enable.""" -        await self.invoke(ctx, "defcon enable") - -    @command(name="defoff", hidden=True) -    async def defcon_disable_alias(self, ctx: Context) -> None: -        """Alias for invoking <prefix>defcon disable.""" -        await self.invoke(ctx, "defcon disable") -      @command(name="exception", hidden=True)      async def tags_get_traceback_alias(self, ctx: Context) -> None:          """Alias for invoking <prefix>tags get traceback.""" @@ -132,21 +81,6 @@ class Alias (Cog):          """Alias for invoking <prefix>docs get [symbol]."""          await self.invoke(ctx, "docs get", symbol) -    @command(name="nominate", hidden=True) -    async def nomination_add_alias(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: -        """Alias for invoking <prefix>talentpool add [user] [reason].""" -        await self.invoke(ctx, "talentpool add", user, reason=reason) - -    @command(name="unnominate", hidden=True) -    async def nomination_end_alias(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: -        """Alias for invoking <prefix>nomination end [user] [reason].""" -        await self.invoke(ctx, "nomination end", user, reason=reason) - -    @command(name="nominees", hidden=True) -    async def nominees_alias(self, ctx: Context) -> None: -        """Alias for invoking <prefix>tp watched.""" -        await self.invoke(ctx, "talentpool watched") -  def setup(bot: Bot) -> None:      """Load the Alias cog.""" diff --git a/bot/cogs/antimalware.py b/bot/cogs/antimalware.py index ea257442e..7894ec48f 100644 --- a/bot/cogs/antimalware.py +++ b/bot/cogs/antimalware.py @@ -6,7 +6,7 @@ from discord import Embed, Message, NotFound  from discord.ext.commands import Cog  from bot.bot import Bot -from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES, URLs +from bot.constants import Channels, STAFF_ROLES, URLs  log = logging.getLogger(__name__) @@ -27,7 +27,7 @@ TXT_EMBED_DESCRIPTION = (  DISALLOWED_EMBED_DESCRIPTION = (      "It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " -    f"We currently allow the following file types: **{', '.join(AntiMalwareConfig.whitelist)}**.\n\n" +    "We currently allow the following file types: **{joined_whitelist}**.\n\n"      "Feel free to ask in {meta_channel_mention} if you think this is a mistake."  ) @@ -38,6 +38,16 @@ class AntiMalware(Cog):      def __init__(self, bot: Bot):          self.bot = bot +    def _get_whitelisted_file_formats(self) -> list: +        """Get the file formats currently on the whitelist.""" +        return self.bot.filter_list_cache['FILE_FORMAT.True'].keys() + +    def _get_disallowed_extensions(self, message: Message) -> t.Iterable[str]: +        """Get an iterable containing all the disallowed extensions of attachments.""" +        file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} +        extensions_blocked = file_extensions - set(self._get_whitelisted_file_formats()) +        return extensions_blocked +      @Cog.listener()      async def on_message(self, message: Message) -> None:          """Identify messages with prohibited attachments.""" @@ -45,13 +55,17 @@ class AntiMalware(Cog):          if not message.attachments or not message.guild:              return +        # Ignore webhook and bot messages +        if message.webhook_id or message.author.bot: +            return +          # Check if user is staff, if is, return          # Since we only care that roles exist to iterate over, check for the attr rather than a User/Member instance          if hasattr(message.author, "roles") and any(role.id in STAFF_ROLES for role in message.author.roles):              return          embed = Embed() -        extensions_blocked = self.get_disallowed_extensions(message) +        extensions_blocked = self._get_disallowed_extensions(message)          blocked_extensions_str = ', '.join(extensions_blocked)          if ".py" in extensions_blocked:              # Short-circuit on *.py files to provide a pastebin link @@ -63,6 +77,7 @@ class AntiMalware(Cog):          elif extensions_blocked:              meta_channel = self.bot.get_channel(Channels.meta)              embed.description = DISALLOWED_EMBED_DESCRIPTION.format( +                joined_whitelist=', '.join(self._get_whitelisted_file_formats()),                  blocked_extensions_str=blocked_extensions_str,                  meta_channel_mention=meta_channel.mention,              ) @@ -81,13 +96,6 @@ class AntiMalware(Cog):              except NotFound:                  log.info(f"Tried to delete message `{message.id}`, but message could not be found.") -    @classmethod -    def get_disallowed_extensions(cls, message: Message) -> t.Iterable[str]: -        """Get an iterable containing all the disallowed extensions of attachments.""" -        file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} -        extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) -        return extensions_blocked -  def setup(bot: Bot) -> None:      """Load the AntiMalware cog.""" diff --git a/bot/cogs/antispam.py b/bot/cogs/antispam.py index 0bcca578d..3ad487d8c 100644 --- a/bot/cogs/antispam.py +++ b/bot/cogs/antispam.py @@ -27,14 +27,18 @@ log = logging.getLogger(__name__)  RULE_FUNCTION_MAPPING = {      'attachments': rules.apply_attachments,      'burst': rules.apply_burst, -    'burst_shared': rules.apply_burst_shared, +    # burst shared is temporarily disabled due to a bug +    # 'burst_shared': rules.apply_burst_shared,      'chars': rules.apply_chars,      'discord_emojis': rules.apply_discord_emojis,      'duplicates': rules.apply_duplicates,      'links': rules.apply_links,      'mentions': rules.apply_mentions,      'newlines': rules.apply_newlines, -    'role_mentions': rules.apply_role_mentions +    'role_mentions': rules.apply_role_mentions, +    # the everyone filter is temporarily disabled until +    # it has been improved. +    # 'everyone_ping': rules.apply_everyone_ping,  } @@ -219,7 +223,6 @@ class AntiSpam(Cog):              # Get context and make sure the bot becomes the actor of infraction by patching the `author` attributes              context = await self.bot.get_context(msg)              context.author = self.bot.user -            context.message.author = self.bot.user              # Since we're going to invoke the tempmute command directly, we need to manually call the converter.              dt_remove_role_after = await self.expiration_date_converter.convert(context, f"{remove_role_after}S") diff --git a/bot/cogs/bot.py b/bot/cogs/bot.py index a79b37d25..ddd1cef8d 100644 --- a/bot/cogs/bot.py +++ b/bot/cogs/bot.py @@ -9,6 +9,7 @@ from discord.ext.commands import Cog, Context, command, group  from bot.bot import Bot  from bot.cogs.token_remover import TokenRemover +from bot.cogs.webhook_remover import WEBHOOK_URL_RE  from bot.constants import Categories, Channels, DEBUG_MODE, Guild, MODERATION_ROLES, Roles, URLs  from bot.decorators import with_role  from bot.utils.messages import wait_for_deletion @@ -72,10 +73,14 @@ class BotCog(Cog, name="Bot"):      @command(name='embed')      @with_role(*MODERATION_ROLES) -    async def embed_command(self, ctx: Context, *, text: str) -> None: -        """Send the input within an embed to the current channel.""" +    async def embed_command(self, ctx: Context, channel: Optional[TextChannel], *, text: str) -> None: +        """Send the input within an embed to either a specified channel or the current channel."""          embed = Embed(description=text) -        await ctx.send(embed=embed) + +        if channel is None: +            await ctx.send(embed=embed) +        else: +            await channel.send(embed=embed)      def codeblock_stripping(self, msg: str, bad_ticks: bool) -> Optional[Tuple[Tuple[str, ...], str]]:          """ @@ -236,6 +241,7 @@ class BotCog(Cog, name="Bot"):              and not msg.author.bot              and len(msg.content.splitlines()) > 3              and not TokenRemover.find_token_in_message(msg) +            and not WEBHOOK_URL_RE.search(msg.content)          )          if parse_codeblock:  # no token in the msg @@ -333,7 +339,7 @@ class BotCog(Cog, name="Bot"):                          self.codeblock_message_ids[msg.id] = bot_message.id                          self.bot.loop.create_task( -                            wait_for_deletion(bot_message, user_ids=(msg.author.id,), client=self.bot) +                            wait_for_deletion(bot_message, (msg.author.id,), self.bot)                          )                      else:                          return diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index 368d91c85..f436e531a 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -45,6 +45,7 @@ class Clean(Cog):          bots_only: bool = False,          user: User = None,          regex: Optional[str] = None, +        until_message: Optional[Message] = None,      ) -> None:          """A helper function that does the actual message cleaning."""          def predicate_bots_only(message: Message) -> bool: @@ -129,6 +130,20 @@ class Clean(Cog):                  if not self.cleaning:                      return +                # If we are looking for specific message. +                if until_message: + +                    # we could use ID's here however in case if the message we are looking for gets deleted, +                    # we won't have a way to figure that out thus checking for datetime should be more reliable +                    if message.created_at < until_message.created_at: +                        # means we have found the message until which we were supposed to be deleting. +                        break + +                    # Since we will be using `delete_messages` method of a TextChannel and we need message objects to +                    # use it as well as to send logs we will start appending messages here instead adding them from +                    # purge. +                    messages.append(message) +                  # If the message passes predicate, let's save it.                  if predicate is None or predicate(message):                      message_ids.append(message.id) @@ -138,7 +153,14 @@ class Clean(Cog):          # Now let's delete the actual messages with purge.          self.mod_log.ignore(Event.message_delete, *message_ids)          for channel in channels: -            messages += await channel.purge(limit=amount, check=predicate) +            if until_message: +                for i in range(0, len(messages), 100): +                    # while purge automatically handles the amount of messages +                    # delete_messages only allows for up to 100 messages at once +                    # thus we need to paginate the amount to always be <= 100 +                    await channel.delete_messages(messages[i:i + 100]) +            else: +                messages += await channel.purge(limit=amount, check=predicate)          # Reverse the list to restore chronological order          if messages: @@ -221,6 +243,17 @@ class Clean(Cog):          """Delete all messages that match a certain regex, stop cleaning after traversing `amount` messages."""          await self._clean_messages(amount, ctx, regex=regex, channels=channels) +    @clean_group.command(name="message", aliases=["messages"]) +    @with_role(*MODERATION_ROLES) +    async def clean_message(self, ctx: Context, message: Message) -> None: +        """Delete all messages until certain message, stop cleaning after hitting the `message`.""" +        await self._clean_messages( +            CleanMessages.message_limit, +            ctx, +            channels=[message.channel], +            until_message=message +        ) +      @clean_group.command(name="stop", aliases=["cancel", "abort"])      @with_role(*MODERATION_ROLES)      async def clean_cancel(self, ctx: Context) -> None: diff --git a/bot/cogs/defcon.py b/bot/cogs/defcon.py index 4c0ad5914..9087ac454 100644 --- a/bot/cogs/defcon.py +++ b/bot/cogs/defcon.py @@ -10,7 +10,7 @@ from discord.ext.commands import Cog, Context, group  from bot.bot import Bot  from bot.cogs.moderation import ModLog -from bot.constants import Channels, Colours, Emojis, Event, Icons, Roles +from bot.constants import Channels, Colours, Emojis, Event, Icons, MODERATION_ROLES, Roles  from bot.decorators import with_role  log = logging.getLogger(__name__) @@ -119,7 +119,7 @@ class Defcon(Cog):                  )      @group(name='defcon', aliases=('dc',), invoke_without_command=True) -    @with_role(Roles.admins, Roles.owners) +    @with_role(*MODERATION_ROLES)      async def defcon_group(self, ctx: Context) -> None:          """Check the DEFCON status or run a subcommand."""          await ctx.send_help(ctx.command) @@ -162,8 +162,8 @@ class Defcon(Cog):              self.bot.stats.gauge("defcon.threshold", days) -    @defcon_group.command(name='enable', aliases=('on', 'e')) -    @with_role(Roles.admins, Roles.owners) +    @defcon_group.command(name='enable', aliases=('on', 'e'), root_aliases=("defon",)) +    @with_role(*MODERATION_ROLES)      async def enable_command(self, ctx: Context) -> None:          """          Enable DEFCON mode. Useful in a pinch, but be sure you know what you're doing! @@ -175,8 +175,8 @@ class Defcon(Cog):          await self._defcon_action(ctx, days=0, action=Action.ENABLED)          await self.update_channel_topic() -    @defcon_group.command(name='disable', aliases=('off', 'd')) -    @with_role(Roles.admins, Roles.owners) +    @defcon_group.command(name='disable', aliases=('off', 'd'), root_aliases=("defoff",)) +    @with_role(*MODERATION_ROLES)      async def disable_command(self, ctx: Context) -> None:          """Disable DEFCON mode. Useful in a pinch, but be sure you know what you're doing!"""          self.enabled = False @@ -184,7 +184,7 @@ class Defcon(Cog):          await self.update_channel_topic()      @defcon_group.command(name='status', aliases=('s',)) -    @with_role(Roles.admins, Roles.owners) +    @with_role(*MODERATION_ROLES)      async def status_command(self, ctx: Context) -> None:          """Check the current status of DEFCON mode."""          embed = Embed( @@ -196,7 +196,7 @@ class Defcon(Cog):          await ctx.send(embed=embed)      @defcon_group.command(name='days') -    @with_role(Roles.admins, Roles.owners) +    @with_role(*MODERATION_ROLES)      async def days_command(self, ctx: Context, days: int) -> None:          """Set how old an account must be to join the server, in days, with DEFCON mode enabled."""          self.days = timedelta(days=days) diff --git a/bot/cogs/dm_relay.py b/bot/cogs/dm_relay.py new file mode 100644 index 000000000..0d8f340b4 --- /dev/null +++ b/bot/cogs/dm_relay.py @@ -0,0 +1,124 @@ +import logging +from typing import Optional + +import discord +from discord import Color +from discord.ext import commands +from discord.ext.commands import Cog + +from bot import constants +from bot.bot import Bot +from bot.converters import UserMentionOrID +from bot.utils import RedisCache +from bot.utils.checks import in_whitelist_check, with_role_check +from bot.utils.messages import send_attachments +from bot.utils.webhooks import send_webhook + +log = logging.getLogger(__name__) + + +class DMRelay(Cog): +    """Relay direct messages to and from the bot.""" + +    # RedisCache[str, t.Union[discord.User.id, discord.Member.id]] +    dm_cache = RedisCache() + +    def __init__(self, bot: Bot): +        self.bot = bot +        self.webhook_id = constants.Webhooks.dm_log +        self.webhook = None +        self.bot.loop.create_task(self.fetch_webhook()) + +    @commands.command(aliases=("reply",)) +    async def send_dm(self, ctx: commands.Context, member: Optional[UserMentionOrID], *, message: str) -> None: +        """ +        Allows you to send a DM to a user from the bot. + +        If `member` is not provided, it will send to the last user who DM'd the bot. + +        This feature should be used extremely sparingly. Use ModMail if you need to have a serious +        conversation with a user. This is just for responding to extraordinary DMs, having a little +        fun with users, and telling people they are DMing the wrong bot. + +        NOTE: This feature will be removed if it is overused. +        """ +        if not member: +            user_id = await self.dm_cache.get("last_user") +            member = ctx.guild.get_member(user_id) if user_id else None + +        # If we still don't have a Member at this point, give up +        if not member: +            log.debug("This bot has never gotten a DM, or the RedisCache has been cleared.") +            await ctx.message.add_reaction("❌") +            return + +        try: +            await member.send(message) +        except discord.errors.Forbidden: +            log.debug("User has disabled DMs.") +            await ctx.message.add_reaction("❌") +        else: +            await ctx.message.add_reaction("✅") +            self.bot.stats.incr("dm_relay.dm_sent") + +    async def fetch_webhook(self) -> None: +        """Fetches the webhook object, so we can post to it.""" +        await self.bot.wait_until_guild_available() + +        try: +            self.webhook = await self.bot.fetch_webhook(self.webhook_id) +        except discord.HTTPException: +            log.exception(f"Failed to fetch webhook with id `{self.webhook_id}`") + +    @Cog.listener() +    async def on_message(self, message: discord.Message) -> None: +        """Relays the message's content and attachments to the dm_log channel.""" +        # Only relay DMs from humans +        if message.author.bot or message.guild or self.webhook is None: +            return + +        if message.clean_content: +            await send_webhook( +                webhook=self.webhook, +                content=message.clean_content, +                username=f"{message.author.display_name} ({message.author.id})", +                avatar_url=message.author.avatar_url +            ) +            await self.dm_cache.set("last_user", message.author.id) +            self.bot.stats.incr("dm_relay.dm_received") + +        # Handle any attachments +        if message.attachments: +            try: +                await send_attachments(message, self.webhook) +            except (discord.errors.Forbidden, discord.errors.NotFound): +                e = discord.Embed( +                    description=":x: **This message contained an attachment, but it could not be retrieved**", +                    color=Color.red() +                ) +                await send_webhook( +                    webhook=self.webhook, +                    embed=e, +                    username=f"{message.author.display_name} ({message.author.id})", +                    avatar_url=message.author.avatar_url +                ) +            except discord.HTTPException: +                log.exception("Failed to send an attachment to the webhook") + +    def cog_check(self, ctx: commands.Context) -> bool: +        """Only allow moderators to invoke the commands in this cog.""" +        checks = [ +            with_role_check(ctx, *constants.MODERATION_ROLES), +            in_whitelist_check( +                ctx, +                channels=[constants.Channels.dm_log], +                redirect=None, +                fail_silently=True, +            ) +        ] +        return all(checks) + + +def setup(bot: Bot) -> None: +    """Load the DMRelay  cog.""" +    bot.add_cog(DMRelay(bot)) diff --git a/bot/cogs/doc.py b/bot/cogs/doc.py index 204cffb37..30c793c75 100644 --- a/bot/cogs/doc.py +++ b/bot/cogs/doc.py @@ -23,6 +23,7 @@ from bot.constants import MODERATION_ROLES, RedirectOutput  from bot.converters import ValidPythonIdentifier, ValidURL  from bot.decorators import with_role  from bot.pagination import LinePaginator +from bot.utils.messages import wait_for_deletion  log = logging.getLogger(__name__) @@ -391,7 +392,8 @@ class Doc(commands.Cog):                      await error_message.delete(delay=NOT_FOUND_DELETE_DELAY)                      await ctx.message.delete(delay=NOT_FOUND_DELETE_DELAY)              else: -                await ctx.send(embed=doc_embed) +                msg = await ctx.send(embed=doc_embed) +                await wait_for_deletion(msg, (ctx.author.id,), client=self.bot)      @docs_group.command(name='set', aliases=('s',))      @with_role(*MODERATION_ROLES) diff --git a/bot/cogs/duck_pond.py b/bot/cogs/duck_pond.py index 5b6a7fd62..7021069fa 100644 --- a/bot/cogs/duck_pond.py +++ b/bot/cogs/duck_pond.py @@ -1,5 +1,5 @@  import logging -from typing import Optional, Union +from typing import Union  import discord  from discord import Color, Embed, Member, Message, RawReactionActionEvent, User, errors @@ -7,7 +7,8 @@ from discord.ext.commands import Cog  from bot import constants  from bot.bot import Bot -from bot.utils.messages import send_attachments, sub_clyde +from bot.utils.messages import send_attachments +from bot.utils.webhooks import send_webhook  log = logging.getLogger(__name__) @@ -18,6 +19,7 @@ class DuckPond(Cog):      def __init__(self, bot: Bot):          self.bot = bot          self.webhook_id = constants.Webhooks.duck_pond +        self.webhook = None          self.bot.loop.create_task(self.fetch_webhook())      async def fetch_webhook(self) -> None: @@ -47,24 +49,6 @@ class DuckPond(Cog):                          return True          return False -    async def send_webhook( -        self, -        content: Optional[str] = None, -        username: Optional[str] = None, -        avatar_url: Optional[str] = None, -        embed: Optional[Embed] = None, -    ) -> None: -        """Send a webhook to the duck_pond channel.""" -        try: -            await self.webhook.send( -                content=content, -                username=sub_clyde(username), -                avatar_url=avatar_url, -                embed=embed -            ) -        except discord.HTTPException: -            log.exception("Failed to send a message to the Duck Pool webhook") -      async def count_ducks(self, message: Message) -> int:          """          Count the number of ducks in the reactions of a specific message. @@ -94,10 +78,9 @@ class DuckPond(Cog):      async def relay_message(self, message: Message) -> None:          """Relays the message's content and attachments to the duck pond channel.""" -        clean_content = message.clean_content - -        if clean_content: -            await self.send_webhook( +        if message.clean_content: +            await send_webhook( +                webhook=self.webhook,                  content=message.clean_content,                  username=message.author.display_name,                  avatar_url=message.author.avatar_url @@ -111,7 +94,8 @@ class DuckPond(Cog):                      description=":x: **This message contained an attachment, but it could not be retrieved**",                      color=Color.red()                  ) -                await self.send_webhook( +                await send_webhook( +                    webhook=self.webhook,                      embed=e,                      username=message.author.display_name,                      avatar_url=message.author.avatar_url diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index 5de961116..f9d4de638 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -2,12 +2,13 @@ import contextlib  import logging  import typing as t +from discord import Embed  from discord.ext.commands import Cog, Context, errors  from sentry_sdk import push_scope  from bot.api import ResponseCodeError  from bot.bot import Bot -from bot.constants import Channels +from bot.constants import Channels, Colours  from bot.converters import TagNameConverter  from bot.utils.checks import InWhitelistCheckFailure @@ -20,6 +21,14 @@ class ErrorHandler(Cog):      def __init__(self, bot: Bot):          self.bot = bot +    def _get_error_embed(self, title: str, body: str) -> Embed: +        """Return an embed that contains the exception.""" +        return Embed( +            title=title, +            colour=Colours.soft_red, +            description=body +        ) +      @Cog.listener()      async def on_command_error(self, ctx: Context, e: errors.CommandError) -> None:          """ @@ -162,25 +171,34 @@ class ErrorHandler(Cog):          prepared_help_command = self.get_help_command(ctx)          if isinstance(e, errors.MissingRequiredArgument): -            await ctx.send(f"Missing required argument `{e.param.name}`.") +            embed = self._get_error_embed("Missing required argument", e.param.name) +            await ctx.send(embed=embed)              await prepared_help_command              self.bot.stats.incr("errors.missing_required_argument")          elif isinstance(e, errors.TooManyArguments): -            await ctx.send("Too many arguments provided.") +            embed = self._get_error_embed("Too many arguments", str(e)) +            await ctx.send(embed=embed)              await prepared_help_command              self.bot.stats.incr("errors.too_many_arguments")          elif isinstance(e, errors.BadArgument): -            await ctx.send(f"Bad argument: {e}\n") +            embed = self._get_error_embed("Bad argument", str(e)) +            await ctx.send(embed=embed)              await prepared_help_command              self.bot.stats.incr("errors.bad_argument")          elif isinstance(e, errors.BadUnionArgument): -            await ctx.send(f"Bad argument: {e}\n```{e.errors[-1]}```") +            embed = self._get_error_embed("Bad argument", f"{e}\n{e.errors[-1]}") +            await ctx.send(embed=embed)              self.bot.stats.incr("errors.bad_union_argument")          elif isinstance(e, errors.ArgumentParsingError): -            await ctx.send(f"Argument parsing error: {e}") +            embed = self._get_error_embed("Argument parsing error", str(e)) +            await ctx.send(embed=embed)              self.bot.stats.incr("errors.argument_parsing_error")          else: -            await ctx.send("Something about your input seems off. Check the arguments:") +            embed = self._get_error_embed( +                "Input error", +                "Something about your input seems off. Check the arguments and try again." +            ) +            await ctx.send(embed=embed)              await prepared_help_command              self.bot.stats.incr("errors.other_user_input_error") diff --git a/bot/cogs/extensions.py b/bot/cogs/extensions.py index 365f198ff..396e406b0 100644 --- a/bot/cogs/extensions.py +++ b/bot/cogs/extensions.py @@ -107,7 +107,7 @@ class Extensions(commands.Cog):          await ctx.send(msg) -    @extensions_group.command(name="reload", aliases=("r",)) +    @extensions_group.command(name="reload", aliases=("r",), root_aliases=("reload",))      async def reload_command(self, ctx: Context, *extensions: Extension) -> None:          r"""          Reload extensions given their fully qualified or unqualified names. diff --git a/bot/cogs/filter_lists.py b/bot/cogs/filter_lists.py new file mode 100644 index 000000000..c15adc461 --- /dev/null +++ b/bot/cogs/filter_lists.py @@ -0,0 +1,273 @@ +import logging +from typing import Optional + +from discord import Colour, Embed +from discord.ext.commands import BadArgument, Cog, Context, IDConverter, group + +from bot import constants +from bot.api import ResponseCodeError +from bot.bot import Bot +from bot.converters import ValidDiscordServerInvite, ValidFilterListType +from bot.pagination import LinePaginator +from bot.utils.checks import with_role_check + +log = logging.getLogger(__name__) + + +class FilterLists(Cog): +    """Commands for blacklisting and whitelisting things.""" + +    methods_with_filterlist_types = [ +        "allow_add", +        "allow_delete", +        "allow_get", +        "deny_add", +        "deny_delete", +        "deny_get", +    ] + +    def __init__(self, bot: Bot) -> None: +        self.bot = bot +        self.bot.loop.create_task(self._amend_docstrings()) + +    async def _amend_docstrings(self) -> None: +        """Add the valid FilterList types to the docstrings, so they'll appear in !help invocations.""" +        await self.bot.wait_until_guild_available() + +        # Add valid filterlist types to the docstrings +        valid_types = await ValidFilterListType.get_valid_types(self.bot) +        valid_types = [f"`{type_.lower()}`" for type_ in valid_types] + +        for method_name in self.methods_with_filterlist_types: +            command = getattr(self, method_name) +            command.help = ( +                f"{command.help}\n\nValid **list_type** values are {', '.join(valid_types)}." +            ) + +    async def _add_data( +        self, +        ctx: Context, +        allowed: bool, +        list_type: ValidFilterListType, +        content: str, +        comment: Optional[str] = None, +    ) -> None: +        """Add an item to a filterlist.""" +        allow_type = "whitelist" if allowed else "blacklist" + +        # If this is a server invite, we gotta validate it. +        if list_type == "GUILD_INVITE": +            guild_data = await self._validate_guild_invite(ctx, content) +            content = guild_data.get("id") + +            # Unless the user has specified another comment, let's +            # use the server name as the comment so that the list +            # of guild IDs will be more easily readable when we +            # display it. +            if not comment: +                comment = guild_data.get("name") + +        # If it's a file format, let's make sure it has a leading dot. +        elif list_type == "FILE_FORMAT" and not content.startswith("."): +            content = f".{content}" + +        # Try to add the item to the database +        log.trace(f"Trying to add the {content} item to the {list_type} {allow_type}") +        payload = { +            "allowed": allowed, +            "type": list_type, +            "content": content, +            "comment": comment, +        } + +        try: +            item = await self.bot.api_client.post( +                "bot/filter-lists", +                json=payload +            ) +        except ResponseCodeError as e: +            if e.status == 400: +                await ctx.message.add_reaction("❌") +                log.debug( +                    f"{ctx.author} tried to add data to a {allow_type}, but the API returned 400, " +                    "probably because the request violated the UniqueConstraint." +                ) +                raise BadArgument( +                    f"Unable to add the item to the {allow_type}. " +                    "The item probably already exists. Keep in mind that a " +                    "blacklist and a whitelist for the same item cannot co-exist, " +                    "and we do not permit any duplicates." +                ) +            raise + +        # Insert the item into the cache +        self.bot.insert_item_into_filter_list_cache(item) +        await ctx.message.add_reaction("✅") + +    async def _delete_data(self, ctx: Context, allowed: bool, list_type: ValidFilterListType, content: str) -> None: +        """Remove an item from a filterlist.""" +        allow_type = "whitelist" if allowed else "blacklist" + +        # If this is a server invite, we need to convert it. +        if list_type == "GUILD_INVITE" and not IDConverter()._get_id_match(content): +            guild_data = await self._validate_guild_invite(ctx, content) +            content = guild_data.get("id") + +        # If it's a file format, let's make sure it has a leading dot. +        elif list_type == "FILE_FORMAT" and not content.startswith("."): +            content = f".{content}" + +        # Find the content and delete it. +        log.trace(f"Trying to delete the {content} item from the {list_type} {allow_type}") +        item = self.bot.filter_list_cache[f"{list_type}.{allowed}"].get(content) + +        if item is not None: +            try: +                await self.bot.api_client.delete( +                    f"bot/filter-lists/{item['id']}" +                ) +                del self.bot.filter_list_cache[f"{list_type}.{allowed}"][content] +                await ctx.message.add_reaction("✅") +            except ResponseCodeError as e: +                log.debug( +                    f"{ctx.author} tried to delete an item with the id {item['id']}, but " +                    f"the API raised an unexpected error: {e}" +                ) +                await ctx.message.add_reaction("❌") +        else: +            await ctx.message.add_reaction("❌") + +    async def _list_all_data(self, ctx: Context, allowed: bool, list_type: ValidFilterListType) -> None: +        """Paginate and display all items in a filterlist.""" +        allow_type = "whitelist" if allowed else "blacklist" +        result = self.bot.filter_list_cache[f"{list_type}.{allowed}"] + +        # Build a list of lines we want to show in the paginator +        lines = [] +        for content, metadata in result.items(): +            line = f"• `{content}`" + +            if comment := metadata.get("comment"): +                line += f" - {comment}" + +            lines.append(line) +        lines = sorted(lines) + +        # Build the embed +        list_type_plural = list_type.lower().replace("_", " ").title() + "s" +        embed = Embed( +            title=f"{allow_type.title()}ed {list_type_plural} ({len(result)} total)", +            colour=Colour.blue() +        ) +        log.trace(f"Trying to list {len(result)} items from the {list_type.lower()} {allow_type}") + +        if result: +            await LinePaginator.paginate(lines, ctx, embed, max_lines=15, empty=False) +        else: +            embed.description = "Hmmm, seems like there's nothing here yet." +            await ctx.send(embed=embed) +            await ctx.message.add_reaction("❌") + +    async def _sync_data(self, ctx: Context) -> None: +        """Syncs the filterlists with the API.""" +        try: +            log.trace("Attempting to sync FilterList cache with data from the API.") +            await self.bot.cache_filter_list_data() +            await ctx.message.add_reaction("✅") +        except ResponseCodeError as e: +            log.debug( +                f"{ctx.author} tried to sync FilterList cache data but " +                f"the API raised an unexpected error: {e}" +            ) +            await ctx.message.add_reaction("❌") + +    @staticmethod +    async def _validate_guild_invite(ctx: Context, invite: str) -> dict: +        """ +        Validates a guild invite, and returns the guild info as a dict. + +        Will raise a BadArgument if the guild invite is invalid. +        """ +        log.trace(f"Attempting to validate whether or not {invite} is a guild invite.") +        validator = ValidDiscordServerInvite() +        guild_data = await validator.convert(ctx, invite) + +        # If we make it this far without raising a BadArgument, the invite is +        # valid. Let's return a dict of guild information. +        log.trace(f"{invite} validated as server invite. Converting to ID.") +        return guild_data + +    @group(aliases=("allowlist", "allow", "al", "wl")) +    async def whitelist(self, ctx: Context) -> None: +        """Group for whitelisting commands.""" +        if not ctx.invoked_subcommand: +            await ctx.send_help(ctx.command) + +    @group(aliases=("denylist", "deny", "bl", "dl")) +    async def blacklist(self, ctx: Context) -> None: +        """Group for blacklisting commands.""" +        if not ctx.invoked_subcommand: +            await ctx.send_help(ctx.command) + +    @whitelist.command(name="add", aliases=("a", "set")) +    async def allow_add( +        self, +        ctx: Context, +        list_type: ValidFilterListType, +        content: str, +        *, +        comment: Optional[str] = None, +    ) -> None: +        """Add an item to the specified allowlist.""" +        await self._add_data(ctx, True, list_type, content, comment) + +    @blacklist.command(name="add", aliases=("a", "set")) +    async def deny_add( +        self, +        ctx: Context, +        list_type: ValidFilterListType, +        content: str, +        *, +        comment: Optional[str] = None, +    ) -> None: +        """Add an item to the specified denylist.""" +        await self._add_data(ctx, False, list_type, content, comment) + +    @whitelist.command(name="remove", aliases=("delete", "rm",)) +    async def allow_delete(self, ctx: Context, list_type: ValidFilterListType, content: str) -> None: +        """Remove an item from the specified allowlist.""" +        await self._delete_data(ctx, True, list_type, content) + +    @blacklist.command(name="remove", aliases=("delete", "rm",)) +    async def deny_delete(self, ctx: Context, list_type: ValidFilterListType, content: str) -> None: +        """Remove an item from the specified denylist.""" +        await self._delete_data(ctx, False, list_type, content) + +    @whitelist.command(name="get", aliases=("list", "ls", "fetch", "show")) +    async def allow_get(self, ctx: Context, list_type: ValidFilterListType) -> None: +        """Get the contents of a specified allowlist.""" +        await self._list_all_data(ctx, True, list_type) + +    @blacklist.command(name="get", aliases=("list", "ls", "fetch", "show")) +    async def deny_get(self, ctx: Context, list_type: ValidFilterListType) -> None: +        """Get the contents of a specified denylist.""" +        await self._list_all_data(ctx, False, list_type) + +    @whitelist.command(name="sync", aliases=("s",)) +    async def allow_sync(self, ctx: Context) -> None: +        """Syncs both allowlists and denylists with the API.""" +        await self._sync_data(ctx) + +    @blacklist.command(name="sync", aliases=("s",)) +    async def deny_sync(self, ctx: Context) -> None: +        """Syncs both allowlists and denylists with the API.""" +        await self._sync_data(ctx) + +    def cog_check(self, ctx: Context) -> bool: +        """Only allow moderators to invoke the commands in this cog.""" +        return with_role_check(ctx, *constants.MODERATION_ROLES) + + +def setup(bot: Bot) -> None: +    """Load the FilterLists cog.""" +    bot.add_cog(FilterLists(bot)) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 76ea68660..99b659bff 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -2,7 +2,7 @@ import asyncio  import logging  import re  from datetime import datetime, timedelta -from typing import List, Mapping, Optional, Union +from typing import List, Mapping, Optional, Tuple, Union  import dateutil  import discord.errors @@ -11,6 +11,7 @@ from discord import Colour, HTTPException, Member, Message, NotFound, TextChanne  from discord.ext.commands import Cog  from discord.utils import escape_markdown +from bot.api import ResponseCodeError  from bot.bot import Bot  from bot.cogs.moderation import ModLog  from bot.constants import ( @@ -18,49 +19,22 @@ from bot.constants import (      Filter, Icons, URLs  )  from bot.utils.redis_cache import RedisCache +from bot.utils.regex import INVITE_RE  from bot.utils.scheduling import Scheduler -from bot.utils.time import wait_until  log = logging.getLogger(__name__) -INVITE_RE = re.compile( -    r"(?:discord(?:[\.,]|dot)gg|"                     # Could be discord.gg/ -    r"discord(?:[\.,]|dot)com(?:\/|slash)invite|"     # or discord.com/invite/ -    r"discordapp(?:[\.,]|dot)com(?:\/|slash)invite|"  # or discordapp.com/invite/ -    r"discord(?:[\.,]|dot)me|"                        # or discord.me -    r"discord(?:[\.,]|dot)io"                         # or discord.io. -    r")(?:[\/]|slash)"                                # / or 'slash' -    r"([a-zA-Z0-9]+)",                                # the invite code itself -    flags=re.IGNORECASE -) - +# Regular expressions  SPOILER_RE = re.compile(r"(\|\|.+?\|\|)", re.DOTALL)  URL_RE = re.compile(r"(https?://[^\s]+)", flags=re.IGNORECASE)  ZALGO_RE = re.compile(r"[\u0300-\u036F\u0489]") -WORD_WATCHLIST_PATTERNS = [ -    re.compile(fr'\b{expression}\b', flags=re.IGNORECASE) for expression in Filter.word_watchlist -] -TOKEN_WATCHLIST_PATTERNS = [ -    re.compile(fr'{expression}', flags=re.IGNORECASE) for expression in Filter.token_watchlist -] -WATCHLIST_PATTERNS = WORD_WATCHLIST_PATTERNS + TOKEN_WATCHLIST_PATTERNS - +# Other constants.  DAYS_BETWEEN_ALERTS = 3 - - -def expand_spoilers(text: str) -> str: -    """Return a string containing all interpretations of a spoilered message.""" -    split_text = SPOILER_RE.split(text) -    return ''.join( -        split_text[0::2] + split_text[1::2] + split_text -    ) - -  OFFENSIVE_MSG_DELETE_TIME = timedelta(days=Filter.offensive_msg_delete_days) -class Filtering(Cog, Scheduler): +class Filtering(Cog):      """Filtering out invites, blacklisting domains, and warning us of certain regular expressions."""      # Redis cache mapping a user ID to the last timestamp a bad nickname alert was sent @@ -68,8 +42,7 @@ class Filtering(Cog, Scheduler):      def __init__(self, bot: Bot):          self.bot = bot -        super().__init__() - +        self.scheduler = Scheduler(self.__class__.__name__)          self.name_lock = asyncio.Lock()          staff_mistake_str = "If you believe this was a mistake, please let staff know!" @@ -127,6 +100,22 @@ class Filtering(Cog, Scheduler):          self.bot.loop.create_task(self.reschedule_offensive_msg_deletion()) +    def cog_unload(self) -> None: +        """Cancel scheduled tasks.""" +        self.scheduler.cancel_all() + +    def _get_filterlist_items(self, list_type: str, *, allowed: bool) -> list: +        """Fetch items from the filter_list_cache.""" +        return self.bot.filter_list_cache[f"{list_type.upper()}.{allowed}"].keys() + +    @staticmethod +    def _expand_spoilers(text: str) -> str: +        """Return a string containing all interpretations of a spoilered message.""" +        split_text = SPOILER_RE.split(text) +        return ''.join( +            split_text[0::2] + split_text[1::2] + split_text +        ) +      @property      def mod_log(self) -> ModLog:          """Get currently loaded ModLog cog instance.""" @@ -136,7 +125,10 @@ class Filtering(Cog, Scheduler):      async def on_message(self, msg: Message) -> None:          """Invoke message filter for new messages."""          await self._filter_message(msg) -        await self.check_bad_words_in_name(msg.author) + +        # Ignore webhook messages. +        if msg.webhook_id is None: +            await self.check_bad_words_in_name(msg.author)      @Cog.listener()      async def on_message_edit(self, before: Message, after: Message) -> None: @@ -151,12 +143,12 @@ class Filtering(Cog, Scheduler):              delta = relativedelta(after.edited_at, before.edited_at).microseconds          await self._filter_message(after, delta) -    @staticmethod -    def get_name_matches(name: str) -> List[re.Match]: +    def get_name_matches(self, name: str) -> List[re.Match]:          """Check bad words from passed string (name). Return list of matches."""          matches = [] -        for pattern in WATCHLIST_PATTERNS: -            if match := pattern.search(name): +        watchlist_patterns = self._get_filterlist_items('filter_token', allowed=False) +        for pattern in watchlist_patterns: +            if match := re.search(pattern, name, flags=re.IGNORECASE):                  matches.append(match)          return matches @@ -200,24 +192,67 @@ class Filtering(Cog, Scheduler):              # Update time when alert sent              await self.name_alerts.set(member.id, datetime.utcnow().timestamp()) -    async def _filter_message(self, msg: Message, delta: Optional[int] = None) -> None: -        """Filter the input message to see if it violates any of our rules, and then respond accordingly.""" +    async def filter_eval(self, result: str, msg: Message) -> bool: +        """ +        Filter the result of an !eval to see if it violates any of our rules, and then respond accordingly. + +        Also requires the original message, to check whether to filter and for mod logs. +        Returns whether a filter was triggered or not. +        """ +        filter_triggered = False          # Should we filter this message? -        role_whitelisted = False +        if self._check_filter(msg): +            for filter_name, _filter in self.filters.items(): +                # Is this specific filter enabled in the config? +                # We also do not need to worry about filters that take the full message, +                # since all we have is an arbitrary string. +                if _filter["enabled"] and _filter["content_only"]: +                    match = await _filter["function"](result) -        if type(msg.author) is Member:  # Only Member has roles, not User. -            for role in msg.author.roles: -                if role.id in Filter.role_whitelist: -                    role_whitelisted = True +                    if match: +                        # If this is a filter (not a watchlist), we set the variable so we know +                        # that it has been triggered +                        if _filter["type"] == "filter": +                            filter_triggered = True -        filter_message = ( -            msg.channel.id not in Filter.channel_whitelist  # Channel not in whitelist -            and not role_whitelisted                        # Role not in whitelist -            and not msg.author.bot                          # Author not a bot -        ) +                        # We do not have to check against DM channels since !eval cannot be used there. +                        channel_str = f"in {msg.channel.mention}" + +                        message_content, additional_embeds, additional_embeds_msg = self._add_stats( +                            filter_name, match, result +                        ) + +                        message = ( +                            f"The {filter_name} {_filter['type']} was triggered " +                            f"by **{msg.author}** " +                            f"(`{msg.author.id}`) {channel_str} using !eval with " +                            f"[the following message]({msg.jump_url}):\n\n" +                            f"{message_content}" +                        ) + +                        log.debug(message) -        # If none of the above, we can start filtering. -        if filter_message: +                        # Send pretty mod log embed to mod-alerts +                        await self.mod_log.send_log_message( +                            icon_url=Icons.filtering, +                            colour=Colour(Colours.soft_red), +                            title=f"{_filter['type'].title()} triggered!", +                            text=message, +                            thumbnail=msg.author.avatar_url_as(static_format="png"), +                            channel_id=Channels.mod_alerts, +                            ping_everyone=Filter.ping_everyone, +                            additional_embeds=additional_embeds, +                            additional_embeds_msg=additional_embeds_msg +                        ) + +                        break  # We don't want multiple filters to trigger + +        return filter_triggered + +    async def _filter_message(self, msg: Message, delta: Optional[int] = None) -> None: +        """Filter the input message to see if it violates any of our rules, and then respond accordingly.""" +        # Should we filter this message? +        if self._check_filter(msg):              for filter_name, _filter in self.filters.items():                  # Is this specific filter enabled in the config?                  if _filter["enabled"]: @@ -267,25 +302,25 @@ class Filtering(Cog, Scheduler):                                  'delete_date': delete_date                              } -                            await self.bot.api_client.post('bot/offensive-messages', json=data) -                            self.schedule_task(msg.id, data) -                            log.trace(f"Offensive message {msg.id} will be deleted on {delete_date}") +                            try: +                                await self.bot.api_client.post('bot/offensive-messages', json=data) +                            except ResponseCodeError as e: +                                if e.status == 400 and "already exists" in e.response_json.get("id", [""])[0]: +                                    log.debug(f"Offensive message {msg.id} already exists.") +                                else: +                                    log.error(f"Offensive message {msg.id} failed to post: {e}") +                            else: +                                self.schedule_msg_delete(data) +                                log.trace(f"Offensive message {msg.id} will be deleted on {delete_date}")                          if is_private:                              channel_str = "via DM"                          else:                              channel_str = f"in {msg.channel.mention}" -                        # Word and match stats for watch_regex -                        if filter_name == "watch_regex": -                            surroundings = match.string[max(match.start() - 10, 0): match.end() + 10] -                            message_content = ( -                                f"**Match:** '{match[0]}'\n" -                                f"**Location:** '...{escape_markdown(surroundings)}...'\n" -                                f"\n**Original Message:**\n{escape_markdown(msg.content)}" -                            ) -                        else:  # Use content of discord Message -                            message_content = msg.content +                        message_content, additional_embeds, additional_embeds_msg = self._add_stats( +                            filter_name, match, msg.content +                        )                          message = (                              f"The {filter_name} {_filter['type']} was triggered " @@ -297,30 +332,6 @@ class Filtering(Cog, Scheduler):                          log.debug(message) -                        self.bot.stats.incr(f"filters.{filter_name}") - -                        additional_embeds = None -                        additional_embeds_msg = None - -                        # The function returns True for invalid invites. -                        # They have no data so additional embeds can't be created for them. -                        if filter_name == "filter_invites" and match is not True: -                            additional_embeds = [] -                            for invite, data in match.items(): -                                embed = discord.Embed(description=( -                                    f"**Members:**\n{data['members']}\n" -                                    f"**Active:**\n{data['active']}" -                                )) -                                embed.set_author(name=data["name"]) -                                embed.set_thumbnail(url=data["icon"]) -                                embed.set_footer(text=f"Guild Invite Code: {invite}") -                                additional_embeds.append(embed) -                            additional_embeds_msg = "For the following guild(s):" - -                        elif filter_name == "watch_rich_embeds": -                            additional_embeds = msg.embeds -                            additional_embeds_msg = "With the following embed(s):" -                          # Send pretty mod log embed to mod-alerts                          await self.mod_log.send_log_message(                              icon_url=Icons.filtering, @@ -329,15 +340,71 @@ class Filtering(Cog, Scheduler):                              text=message,                              thumbnail=msg.author.avatar_url_as(static_format="png"),                              channel_id=Channels.mod_alerts, -                            ping_everyone=Filter.ping_everyone, +                            ping_everyone=Filter.ping_everyone if not is_private else False,                              additional_embeds=additional_embeds,                              additional_embeds_msg=additional_embeds_msg                          )                          break  # We don't want multiple filters to trigger +    def _add_stats(self, name: str, match: Union[re.Match, dict, bool, List[discord.Embed]], content: str) -> Tuple[ +        str, Optional[List[discord.Embed]], Optional[str] +    ]: +        """Adds relevant statistical information to the relevant filter and increments the bot's stats.""" +        # Word and match stats for watch_regex +        if name == "watch_regex": +            surroundings = match.string[max(match.start() - 10, 0): match.end() + 10] +            message_content = ( +                f"**Match:** '{match[0]}'\n" +                f"**Location:** '...{escape_markdown(surroundings)}...'\n" +                f"\n**Original Message:**\n{escape_markdown(content)}" +            ) +        else:  # Use original content +            message_content = content + +        additional_embeds = None +        additional_embeds_msg = None + +        self.bot.stats.incr(f"filters.{name}") + +        # The function returns True for invalid invites. +        # They have no data so additional embeds can't be created for them. +        if name == "filter_invites" and match is not True: +            additional_embeds = [] +            for _, data in match.items(): +                embed = discord.Embed(description=( +                    f"**Members:**\n{data['members']}\n" +                    f"**Active:**\n{data['active']}" +                )) +                embed.set_author(name=data["name"]) +                embed.set_thumbnail(url=data["icon"]) +                embed.set_footer(text=f"Guild ID: {data['id']}") +                additional_embeds.append(embed) +            additional_embeds_msg = "For the following guild(s):" + +        elif name == "watch_rich_embeds": +            additional_embeds = match +            additional_embeds_msg = "With the following embed(s):" + +        return message_content, additional_embeds, additional_embeds_msg +      @staticmethod -    async def _has_watch_regex_match(text: str) -> Union[bool, re.Match]: +    def _check_filter(msg: Message) -> bool: +        """Check whitelists to see if we should filter this message.""" +        role_whitelisted = False + +        if type(msg.author) is Member:  # Only Member has roles, not User. +            for role in msg.author.roles: +                if role.id in Filter.role_whitelist: +                    role_whitelisted = True + +        return ( +            msg.channel.id not in Filter.channel_whitelist  # Channel not in whitelist +            and not role_whitelisted                        # Role not in whitelist +            and not msg.author.bot                          # Author not a bot +        ) + +    async def _has_watch_regex_match(self, text: str) -> Union[bool, re.Match]:          """          Return True if `text` matches any regex from `word_watchlist` or `token_watchlist` configs. @@ -345,26 +412,27 @@ class Filtering(Cog, Scheduler):          matched as-is. Spoilers are expanded, if any, and URLs are ignored.          """          if SPOILER_RE.search(text): -            text = expand_spoilers(text) +            text = self._expand_spoilers(text)          # Make sure it's not a URL          if URL_RE.search(text):              return False -        for pattern in WATCHLIST_PATTERNS: -            match = pattern.search(text) +        watchlist_patterns = self._get_filterlist_items('filter_token', allowed=False) +        for pattern in watchlist_patterns: +            match = re.search(pattern, text, flags=re.IGNORECASE)              if match:                  return match -    @staticmethod -    async def _has_urls(text: str) -> bool: +    async def _has_urls(self, text: str) -> bool:          """Returns True if the text contains one of the blacklisted URLs from the config file."""          if not URL_RE.search(text):              return False          text = text.lower() +        domain_blacklist = self._get_filterlist_items("domain_name", allowed=False) -        for url in Filter.domain_blacklist: +        for url in domain_blacklist:              if url.lower() in text:                  return True @@ -388,7 +456,7 @@ class Filtering(Cog, Scheduler):          Attempts to catch some of common ways to try to cheat the system.          """ -        # Remove backslashes to prevent escape character around fuckery like +        # Remove backslashes to prevent escape character aroundfuckery like          # discord\.gg/gdudes-pony-farm          text = text.replace("\\", "") @@ -409,9 +477,22 @@ class Filtering(Cog, Scheduler):                  # between invalid and expired invites                  return True -            guild_id = int(guild.get("id")) +            guild_id = guild.get("id") +            guild_invite_whitelist = self._get_filterlist_items("guild_invite", allowed=True) +            guild_invite_blacklist = self._get_filterlist_items("guild_invite", allowed=False) -            if guild_id not in Filter.guild_invite_whitelist: +            # Is this invite allowed? +            guild_partnered_or_verified = ( +                'PARTNERED' in guild.get("features", []) +                or 'VERIFIED' in guild.get("features", []) +            ) +            invite_not_allowed = ( +                guild_id in guild_invite_blacklist           # Blacklisted guilds are never permitted. +                or guild_id not in guild_invite_whitelist    # Whitelisted guilds are always permitted. +                and not guild_partnered_or_verified          # Otherwise guilds have to be Verified or Partnered. +            ) + +            if invite_not_allowed:                  guild_icon_hash = guild["icon"]                  guild_icon = (                      "https://cdn.discordapp.com/icons/" @@ -420,6 +501,7 @@ class Filtering(Cog, Scheduler):                  invite_data[invite] = {                      "name": guild["name"], +                    "id": guild['id'],                      "icon": guild_icon,                      "members": response["approximate_member_count"],                      "active": response["approximate_presence_count"] @@ -428,7 +510,7 @@ class Filtering(Cog, Scheduler):          return invite_data if invite_data else False      @staticmethod -    async def _has_rich_embed(msg: Message) -> bool: +    async def _has_rich_embed(msg: Message) -> Union[bool, List[discord.Embed]]:          """Determines if `msg` contains any rich embeds not auto-generated from a URL."""          if msg.embeds:              for embed in msg.embeds: @@ -437,7 +519,7 @@ class Filtering(Cog, Scheduler):                      if not embed.url or embed.url not in urls:                          # If `embed.url` does not exist or if `embed.url` is not part of the content                          # of the message, it's unlikely to be an auto-generated embed by Discord. -                        return True +                        return msg.embeds                      else:                          log.trace(                              "Found a rich embed sent by a regular user account, " @@ -457,12 +539,10 @@ class Filtering(Cog, Scheduler):          except discord.errors.Forbidden:              await channel.send(f"{filtered_member.mention} {reason}") -    async def _scheduled_task(self, msg: dict) -> None: +    def schedule_msg_delete(self, msg: dict) -> None:          """Delete an offensive message once its deletion date is reached."""          delete_at = dateutil.parser.isoparse(msg['delete_date']).replace(tzinfo=None) - -        await wait_until(delete_at) -        await self.delete_offensive_msg(msg) +        self.scheduler.schedule_at(delete_at, msg['id'], self.delete_offensive_msg(msg))      async def reschedule_offensive_msg_deletion(self) -> None:          """Get all the pending message deletion from the API and reschedule them.""" @@ -477,7 +557,7 @@ class Filtering(Cog, Scheduler):              if delete_at < now:                  await self.delete_offensive_msg(msg)              else: -                self.schedule_task(msg['id'], msg) +                self.schedule_msg_delete(msg)      async def delete_offensive_msg(self, msg: Mapping[str, str]) -> None:          """Delete an offensive message, and then delete it from the db.""" diff --git a/bot/cogs/help.py b/bot/cogs/help.py index 542f19139..99d503f5c 100644 --- a/bot/cogs/help.py +++ b/bot/cogs/help.py @@ -1,50 +1,28 @@  import itertools  import logging -from asyncio import TimeoutError  from collections import namedtuple  from contextlib import suppress  from typing import List, Union -from discord import Colour, Embed, Member, Message, NotFound, Reaction, User +from discord import Colour, Embed  from discord.ext.commands import Bot, Cog, Command, Context, Group, HelpCommand  from fuzzywuzzy import fuzz, process +from fuzzywuzzy.utils import full_process  from bot import constants -from bot.constants import Channels, Emojis, STAFF_ROLES +from bot.constants import Channels, STAFF_ROLES  from bot.decorators import redirect_output  from bot.pagination import LinePaginator +from bot.utils.messages import wait_for_deletion  log = logging.getLogger(__name__)  COMMANDS_PER_PAGE = 8 -DELETE_EMOJI = Emojis.trashcan  PREFIX = constants.Bot.prefix  Category = namedtuple("Category", ["name", "description", "cogs"]) -async def help_cleanup(bot: Bot, author: Member, message: Message) -> None: -    """ -    Runs the cleanup for the help command. - -    Adds the :trashcan: reaction that, when clicked, will delete the help message. -    After a 300 second timeout, the reaction will be removed. -    """ -    def check(reaction: Reaction, user: User) -> bool: -        """Checks the reaction is :trashcan:, the author is original author and messages are the same.""" -        return str(reaction) == DELETE_EMOJI and user.id == author.id and reaction.message.id == message.id - -    await message.add_reaction(DELETE_EMOJI) - -    try: -        await bot.wait_for("reaction_add", check=check, timeout=300) -        await message.delete() -    except TimeoutError: -        await message.remove_reaction(DELETE_EMOJI, bot.user) -    except NotFound: -        pass - -  class HelpQueryNotFound(ValueError):      """      Raised when a HelpSession Query doesn't match a command or cog. @@ -146,7 +124,13 @@ class CustomHelpCommand(HelpCommand):          Will return an instance of the `HelpQueryNotFound` exception with the error message and possible matches.          """          choices = await self.get_all_help_choices() -        result = process.extractBests(string, choices, scorer=fuzz.ratio, score_cutoff=60) + +        # Run fuzzywuzzy's processor beforehand, and avoid matching if processed string is empty +        # This avoids fuzzywuzzy from raising a warning on inputs with only non-alphanumeric characters +        if (processed := full_process(string)): +            result = process.extractBests(processed, choices, scorer=fuzz.ratio, score_cutoff=60, processor=None) +        else: +            result = []          return HelpQueryNotFound(f'Query "{string}" not found.', dict(result)) @@ -183,7 +167,9 @@ class CustomHelpCommand(HelpCommand):          command_details = f"**```{PREFIX}{name} {command.signature}```**\n"          # show command aliases -        aliases = ", ".join(f"`{alias}`" if not parent else f"`{parent} {alias}`" for alias in command.aliases) +        aliases = [f"`{alias}`" if not parent else f"`{parent} {alias}`" for alias in command.aliases] +        aliases += [f"`{alias}`" for alias in getattr(command, "root_aliases", ())] +        aliases = ", ".join(sorted(aliases))          if aliases:              command_details += f"**Can also use:** {aliases}\n\n" @@ -200,7 +186,7 @@ class CustomHelpCommand(HelpCommand):          """Send help for a single command."""          embed = await self.command_formatting(command)          message = await self.context.send(embed=embed) -        await help_cleanup(self.context.bot, self.context.author, message) +        await wait_for_deletion(message, (self.context.author.id,), self.context.bot)      @staticmethod      def get_commands_brief_details(commands_: List[Command], return_as_list: bool = False) -> Union[List[str], str]: @@ -239,7 +225,7 @@ class CustomHelpCommand(HelpCommand):              embed.description += f"\n**Subcommands:**\n{command_details}"          message = await self.context.send(embed=embed) -        await help_cleanup(self.context.bot, self.context.author, message) +        await wait_for_deletion(message, (self.context.author.id,), self.context.bot)      async def send_cog_help(self, cog: Cog) -> None:          """Send help for a cog.""" @@ -255,7 +241,7 @@ class CustomHelpCommand(HelpCommand):              embed.description += f"\n\n**Commands:**\n{command_details}"          message = await self.context.send(embed=embed) -        await help_cleanup(self.context.bot, self.context.author, message) +        await wait_for_deletion(message, (self.context.author.id,), self.context.bot)      @staticmethod      def _category_key(command: Command) -> str: @@ -299,7 +285,7 @@ class CustomHelpCommand(HelpCommand):              embed,              prefix=description,              max_lines=COMMANDS_PER_PAGE, -            max_size=2040, +            max_size=2000,          )      async def send_bot_help(self, mapping: dict) -> None: @@ -346,7 +332,7 @@ class CustomHelpCommand(HelpCommand):              # add any remaining command help that didn't get added in the last iteration above.              pages.append(page) -        await LinePaginator.paginate(pages, self.context, embed=embed, max_lines=1, max_size=2040) +        await LinePaginator.paginate(pages, self.context, embed=embed, max_lines=1, max_size=2000)  class Help(Cog): diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 187adfe51..0f9cac89e 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -1,5 +1,4 @@  import asyncio -import inspect  import json  import logging  import random @@ -35,12 +34,9 @@ and will be yours until it has been inactive for {constants.HelpChannels.idle_mi  is closed manually with `!close`. When that happens, it will be set to **dormant** and moved into \  the **Help: Dormant** category. -You may claim a new channel once every {constants.HelpChannels.claim_minutes} minutes. If you \ -currently cannot send a message in this channel, it means you are on cooldown and need to wait. -  Try to write the best question you can by providing a detailed description and telling us what \  you've tried already. For more information on asking a good question, \ -check out our guide on [asking good questions]({ASKING_GUIDE_URL}). +check out our guide on **[asking good questions]({ASKING_GUIDE_URL})**.  """  DORMANT_MSG = f""" @@ -51,20 +47,13 @@ channel until it becomes available again.  If your question wasn't answered yet, you can claim a new help channel from the \  **Help: Available** category by simply asking your question again. Consider rephrasing the \  question to maximize your chance of getting a good answer. If you're not sure how, have a look \ -through our guide for [asking a good question]({ASKING_GUIDE_URL}). +through our guide for **[asking a good question]({ASKING_GUIDE_URL})**.  """  CoroutineFunc = t.Callable[..., t.Coroutine] -class TaskData(t.NamedTuple): -    """Data for a scheduled task.""" - -    wait_time: int -    callback: t.Awaitable - - -class HelpChannels(Scheduler, commands.Cog): +class HelpChannels(commands.Cog):      """      Manage the help channel system of the guild. @@ -113,10 +102,13 @@ class HelpChannels(Scheduler, commands.Cog):      # RedisCache[discord.TextChannel.id, UtcPosixTimestamp]      claim_times = RedisCache() -    def __init__(self, bot: Bot): -        super().__init__() +    # This cache maps a help channel to original question message in same channel. +    # RedisCache[discord.TextChannel.id, discord.Message.id] +    question_messages = RedisCache() +    def __init__(self, bot: Bot):          self.bot = bot +        self.scheduler = Scheduler(self.__class__.__name__)          # Categories          self.available_category: discord.CategoryChannel = None @@ -145,7 +137,7 @@ class HelpChannels(Scheduler, commands.Cog):          for task in self.queue_tasks:              task.cancel() -        self.cancel_all() +        self.scheduler.cancel_all()      def create_channel_queue(self) -> asyncio.Queue:          """ @@ -223,16 +215,14 @@ class HelpChannels(Scheduler, commands.Cog):          log.trace("close command invoked; checking if the channel is in-use.")          if ctx.channel.category == self.in_use_category:              if await self.dormant_check(ctx): - -                # Remove the claimant and the cooldown role -                await self.help_channel_claimants.delete(ctx.channel.id)                  await self.remove_cooldown_role(ctx.author)                  # Ignore missing task when cooldown has passed but the channel still isn't dormant. -                self.cancel_task(ctx.author.id, ignore_missing=True) +                if ctx.author.id in self.scheduler: +                    self.scheduler.cancel(ctx.author.id)                  await self.move_to_dormant(ctx.channel, "command") -                self.cancel_task(ctx.channel.id) +                self.scheduler.cancel(ctx.channel.id)          else:              log.debug(f"{ctx.author} invoked command 'dormant' outside an in-use help channel") @@ -371,10 +361,18 @@ class HelpChannels(Scheduler, commands.Cog):          channels = list(self.get_category_channels(self.available_category))          missing = constants.HelpChannels.max_available - len(channels) -        log.trace(f"Moving {missing} missing channels to the Available category.") +        # If we've got less than `max_available` channel available, we should add some. +        if missing > 0: +            log.trace(f"Moving {missing} missing channels to the Available category.") +            for _ in range(missing): +                await self.move_to_available() -        for _ in range(missing): -            await self.move_to_available() +        # If for some reason we have more than `max_available` channels available, +        # we should move the superfluous ones over to dormant. +        elif missing < 0: +            log.trace(f"Moving {abs(missing)} superfluous available channels over to the Dormant category.") +            for channel in channels[:abs(missing)]: +                await self.move_to_dormant(channel, "auto")      async def init_categories(self) -> None:          """Get the help category objects. Remove the cog if retrieval fails.""" @@ -439,8 +437,11 @@ class HelpChannels(Scheduler, commands.Cog):          if not message or not message.embeds:              return False -        embed = message.embeds[0] -        return message.author == self.bot.user and embed.description.strip() == description.strip() +        bot_msg_desc = message.embeds[0].description +        if bot_msg_desc is discord.Embed.Empty: +            log.trace("Last message was a bot embed but it was empty.") +            return False +        return message.author == self.bot.user and bot_msg_desc.strip() == description.strip()      @staticmethod      def is_in_category(channel: discord.TextChannel, category_id: int) -> bool: @@ -474,16 +475,15 @@ class HelpChannels(Scheduler, commands.Cog):          else:              # Cancel the existing task, if any.              if has_task: -                self.cancel_task(channel.id) - -            data = TaskData(idle_seconds - time_elapsed, self.move_idle_channel(channel)) +                self.scheduler.cancel(channel.id) +            delay = idle_seconds - time_elapsed              log.info(                  f"#{channel} ({channel.id}) is still active; " -                f"scheduling it to be moved after {data.wait_time} seconds." +                f"scheduling it to be moved after {delay} seconds."              ) -            self.schedule_task(channel.id, data) +            self.scheduler.schedule_later(delay, channel.id, self.move_idle_channel(channel))      async def move_to_bottom_position(self, channel: discord.TextChannel, category_id: int, **options) -> None:          """ @@ -550,6 +550,7 @@ class HelpChannels(Scheduler, commands.Cog):          """          log.info(f"Moving #{channel} ({channel.id}) to the Dormant category.") +        await self.help_channel_claimants.delete(channel.id)          await self.move_to_bottom_position(              channel=channel,              category_id=constants.Categories.help_dormant, @@ -572,6 +573,8 @@ class HelpChannels(Scheduler, commands.Cog):          embed = discord.Embed(description=DORMANT_MSG)          await channel.send(embed=embed) +        await self.unpin(channel) +          log.trace(f"Pushing #{channel} ({channel.id}) into the channel queue.")          self.channel_queue.put_nowait(channel)          self.report_stats() @@ -588,8 +591,7 @@ class HelpChannels(Scheduler, commands.Cog):          timeout = constants.HelpChannels.idle_minutes * 60          log.trace(f"Scheduling #{channel} ({channel.id}) to become dormant in {timeout} sec.") -        data = TaskData(timeout, self.move_idle_channel(channel)) -        self.schedule_task(channel.id, data) +        self.scheduler.schedule_later(timeout, channel.id, self.move_idle_channel(channel))          self.report_stats()      async def notify(self) -> None: @@ -624,11 +626,13 @@ class HelpChannels(Scheduler, commands.Cog):              channel = self.bot.get_channel(constants.HelpChannels.notify_channel)              mentions = " ".join(f"<@&{role}>" for role in constants.HelpChannels.notify_roles) +            allowed_roles = [discord.Object(id_) for id_ in constants.HelpChannels.notify_roles]              message = await channel.send(                  f"{mentions} A new available help channel is needed but there "                  f"are no more dormant ones. Consider freeing up some in-use channels manually by " -                f"using the `{constants.Bot.prefix}dormant` command within the channels." +                f"using the `{constants.Bot.prefix}dormant` command within the channels.", +                allowed_mentions=discord.AllowedMentions(everyone=False, roles=allowed_roles)              )              self.bot.stats.incr("help.out_of_channel_alerts") @@ -688,6 +692,9 @@ class HelpChannels(Scheduler, commands.Cog):              log.info(f"Channel #{channel} was claimed by `{message.author.id}`.")              await self.move_to_in_use(channel)              await self.revoke_send_permissions(message.author) + +            await self.pin(message) +              # Add user with channel for dormant check.              await self.help_channel_claimants.set(channel.id, message.author.id) @@ -722,15 +729,28 @@ class HelpChannels(Scheduler, commands.Cog):          log.info(f"Claimant of #{msg.channel} ({msg.author}) deleted message, channel is empty now. Rescheduling task.")          # Cancel existing dormant task before scheduling new. -        self.cancel_task(msg.channel.id) +        self.scheduler.cancel(msg.channel.id) -        task = TaskData(constants.HelpChannels.deleted_idle_minutes * 60, self.move_idle_channel(msg.channel)) -        self.schedule_task(msg.channel.id, task) +        delay = constants.HelpChannels.deleted_idle_minutes * 60 +        self.scheduler.schedule_later(delay, msg.channel.id, self.move_idle_channel(msg.channel))      async def is_empty(self, channel: discord.TextChannel) -> bool: -        """Return True if the most recent message in `channel` is the bot's `AVAILABLE_MSG`.""" -        msg = await self.get_last_message(channel) -        return self.match_bot_embed(msg, AVAILABLE_MSG) +        """Return True if there's an AVAILABLE_MSG and the messages leading up are bot messages.""" +        log.trace(f"Checking if #{channel} ({channel.id}) is empty.") + +        # A limit of 100 results in a single API call. +        # If AVAILABLE_MSG isn't found within 100 messages, then assume the channel is not empty. +        # Not gonna do an extensive search for it cause it's too expensive. +        async for msg in channel.history(limit=100): +            if not msg.author.bot: +                log.trace(f"#{channel} ({channel.id}) has a non-bot message.") +                return False + +            if self.match_bot_embed(msg, AVAILABLE_MSG): +                log.trace(f"#{channel} ({channel.id}) has the available message embed.") +                return True + +        return False      async def check_cooldowns(self) -> None:          """Remove expired cooldowns and re-schedule active ones.""" @@ -752,8 +772,8 @@ class HelpChannels(Scheduler, commands.Cog):                  await self.remove_cooldown_role(member)              else:                  # The member is still on a cooldown; re-schedule it for the remaining time. -                remaining = cooldown - in_use_time.seconds -                await self.schedule_cooldown_expiration(member, remaining) +                delay = cooldown - in_use_time.seconds +                self.scheduler.schedule_later(delay, member.id, self.remove_cooldown_role(member))      async def add_cooldown_role(self, member: discord.Member) -> None:          """Add the help cooldown role to `member`.""" @@ -804,16 +824,11 @@ class HelpChannels(Scheduler, commands.Cog):          # Cancel the existing task, if any.          # Would mean the user somehow bypassed the lack of permissions (e.g. user is guild owner). -        self.cancel_task(member.id, ignore_missing=True) - -        await self.schedule_cooldown_expiration(member, constants.HelpChannels.claim_minutes * 60) - -    async def schedule_cooldown_expiration(self, member: discord.Member, seconds: int) -> None: -        """Schedule the cooldown role for `member` to be removed after a duration of `seconds`.""" -        log.trace(f"Scheduling removal of {member}'s ({member.id}) cooldown.") +        if member.id in self.scheduler: +            self.scheduler.cancel(member.id) -        callback = self.remove_cooldown_role(member) -        self.schedule_task(member.id, TaskData(seconds, callback)) +        delay = constants.HelpChannels.claim_minutes * 60 +        self.scheduler.schedule_later(delay, member.id, self.remove_cooldown_role(member))      async def send_available_message(self, channel: discord.TextChannel) -> None:          """Send the available message by editing a dormant message or sending a new message.""" @@ -842,6 +857,47 @@ class HelpChannels(Scheduler, commands.Cog):          log.trace(f"Channel #{channel} ({channel_id}) retrieved.")          return channel +    async def pin_wrapper(self, msg_id: int, channel: discord.TextChannel, *, pin: bool) -> bool: +        """ +        Pin message `msg_id` in `channel` if `pin` is True or unpin if it's False. + +        Return True if successful and False otherwise. +        """ +        channel_str = f"#{channel} ({channel.id})" +        if pin: +            func = self.bot.http.pin_message +            verb = "pin" +        else: +            func = self.bot.http.unpin_message +            verb = "unpin" + +        try: +            await func(channel.id, msg_id) +        except discord.HTTPException as e: +            if e.code == 10008: +                log.debug(f"Message {msg_id} in {channel_str} doesn't exist; can't {verb}.") +            else: +                log.exception( +                    f"Error {verb}ning message {msg_id} in {channel_str}: {e.status} ({e.code})" +                ) +            return False +        else: +            log.trace(f"{verb.capitalize()}ned message {msg_id} in {channel_str}.") +            return True + +    async def pin(self, message: discord.Message) -> None: +        """Pin an initial question `message` and store it in a cache.""" +        if await self.pin_wrapper(message.id, message.channel, pin=True): +            await self.question_messages.set(message.channel.id, message.id) + +    async def unpin(self, channel: discord.TextChannel) -> None: +        """Unpin the initial question message sent in `channel`.""" +        msg_id = await self.question_messages.pop(channel.id) +        if msg_id is None: +            log.debug(f"#{channel} ({channel.id}) doesn't have a message pinned.") +        else: +            await self.pin_wrapper(msg_id, channel, pin=False) +      async def wait_for_dormant_channel(self) -> discord.TextChannel:          """Wait for a dormant channel to become available in the queue and return it."""          log.trace("Waiting for a dormant channel.") @@ -855,21 +911,6 @@ class HelpChannels(Scheduler, commands.Cog):          return channel -    async def _scheduled_task(self, data: TaskData) -> None: -        """Await the `data.callback` coroutine after waiting for `data.wait_time` seconds.""" -        try: -            log.trace(f"Waiting {data.wait_time} seconds before awaiting callback.") -            await asyncio.sleep(data.wait_time) - -            # Use asyncio.shield to prevent callback from cancelling itself. -            # The parent task (_scheduled_task) will still get cancelled. -            log.trace("Done waiting; now awaiting the callback.") -            await asyncio.shield(data.callback) -        finally: -            if inspect.iscoroutine(data.callback): -                log.trace("Explicitly closing coroutine.") -                data.callback.close() -  def validate_config() -> None:      """Raise a ValueError if the cog's config is invalid.""" diff --git a/bot/cogs/information.py b/bot/cogs/information.py index f0bd1afdb..55ecb2836 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -4,9 +4,9 @@ import pprint  import textwrap  from collections import Counter, defaultdict  from string import Template -from typing import Any, Mapping, Optional, Union +from typing import Any, Mapping, Optional, Tuple, Union -from discord import ChannelType, Colour, Embed, Guild, Member, Message, Role, Status, utils +from discord import ChannelType, Colour, CustomActivity, Embed, Guild, Member, Message, Role, Status, utils  from discord.abc import GuildChannel  from discord.ext.commands import BucketType, Cog, Context, Paginator, command, group  from discord.utils import escape_markdown @@ -20,6 +20,12 @@ from bot.utils.time import time_since  log = logging.getLogger(__name__) +STATUS_EMOTES = { +    Status.offline: constants.Emojis.status_offline, +    Status.dnd: constants.Emojis.status_dnd, +    Status.idle: constants.Emojis.status_idle +} +  class Information(Cog):      """A cog with commands for generating embeds with server info, such as server stats and user info.""" @@ -116,10 +122,7 @@ class Information(Cog):              parsed_roles.append(role)          if failed_roles: -            await ctx.send( -                ":x: I could not convert the following role names to a role: \n- " -                "\n- ".join(failed_roles) -            ) +            await ctx.send(f":x: Could not retrieve the following roles: {', '.join(failed_roles)}")          for role in parsed_roles:              h, s, v = colorsys.rgb_to_hsv(*role.colour.to_rgb()) @@ -214,53 +217,88 @@ class Information(Cog):          # Custom status          custom_status = ''          for activity in user.activities: -            # Check activity.state for None value if user has a custom status set -            # This guards against a custom status with an emoji but no text, which will cause -            # escape_markdown to raise an exception -            # This can be reworked after a move to d.py 1.3.0+, which adds a CustomActivity class -            if activity.name == 'Custom Status' and activity.state: -                state = escape_markdown(activity.state) -                custom_status = f'Status: {state}\n' +            if isinstance(activity, CustomActivity): +                state = "" + +                if activity.name: +                    state = escape_markdown(activity.name) + +                emoji = "" +                if activity.emoji: +                    # If an emoji is unicode use the emoji, else write the emote like :abc: +                    if not activity.emoji.id: +                        emoji += activity.emoji.name + " " +                    else: +                        emoji += f"`:{activity.emoji.name}:` " + +                custom_status = f'Status: {emoji}{state}\n'          name = str(user)          if user.nick:              name = f"{user.nick} ({name})" -        joined = time_since(user.joined_at, precision="days") +        badges = [] + +        for badge, is_set in user.public_flags: +            if is_set and (emoji := getattr(constants.Emojis, f"badge_{badge}", None)): +                badges.append(emoji) + +        joined = time_since(user.joined_at, max_units=3)          roles = ", ".join(role.mention for role in user.roles[1:]) -        description = [ -            textwrap.dedent(f""" -                **User Information** -                Created: {created} -                Profile: {user.mention} -                ID: {user.id} -                {custom_status} -                **Member Information** -                Joined: {joined} -                Roles: {roles or None} -            """).strip() +        desktop_status = STATUS_EMOTES.get(user.desktop_status, constants.Emojis.status_online) +        web_status = STATUS_EMOTES.get(user.web_status, constants.Emojis.status_online) +        mobile_status = STATUS_EMOTES.get(user.mobile_status, constants.Emojis.status_online) + +        fields = [ +            ( +                "User information", +                textwrap.dedent(f""" +                    Created: {created} +                    Profile: {user.mention} +                    ID: {user.id} +                    {custom_status} +                """).strip() +            ), +            ( +                "Member information", +                textwrap.dedent(f""" +                    Joined: {joined} +                    Roles: {roles or None} +                """).strip() +            ), +            ( +                "Status", +                textwrap.dedent(f""" +                    {desktop_status} Desktop +                    {web_status} Web +                    {mobile_status} Mobile +                """).strip() +            )          ]          # Show more verbose output in moderation channels for infractions and nominations          if ctx.channel.id in constants.MODERATION_CHANNELS: -            description.append(await self.expanded_user_infraction_counts(user)) -            description.append(await self.user_nomination_counts(user)) +            fields.append(await self.expanded_user_infraction_counts(user)) +            fields.append(await self.user_nomination_counts(user))          else: -            description.append(await self.basic_user_infraction_counts(user)) +            fields.append(await self.basic_user_infraction_counts(user))          # Let's build the embed now          embed = Embed(              title=name, -            description="\n\n".join(description) +            description=" ".join(badges)          ) +        for field_name, field_content in fields: +            embed.add_field(name=field_name, value=field_content, inline=False) +          embed.set_thumbnail(url=user.avatar_url_as(static_format="png"))          embed.colour = user.top_role.colour if roles else Colour.blurple()          return embed -    async def basic_user_infraction_counts(self, member: Member) -> str: +    async def basic_user_infraction_counts(self, member: Member) -> Tuple[str, str]:          """Gets the total and active infraction counts for the given `member`."""          infractions = await self.bot.api_client.get(              'bot/infractions', @@ -273,11 +311,11 @@ class Information(Cog):          total_infractions = len(infractions)          active_infractions = sum(infraction['active'] for infraction in infractions) -        infraction_output = f"**Infractions**\nTotal: {total_infractions}\nActive: {active_infractions}" +        infraction_output = f"Total: {total_infractions}\nActive: {active_infractions}" -        return infraction_output +        return "Infractions", infraction_output -    async def expanded_user_infraction_counts(self, member: Member) -> str: +    async def expanded_user_infraction_counts(self, member: Member) -> Tuple[str, str]:          """          Gets expanded infraction counts for the given `member`. @@ -291,9 +329,9 @@ class Information(Cog):              }          ) -        infraction_output = ["**Infractions**"] +        infraction_output = []          if not infractions: -            infraction_output.append("This user has never received an infraction.") +            infraction_output.append("No infractions")          else:              # Count infractions split by `type` and `active` status for this user              infraction_types = set() @@ -316,9 +354,9 @@ class Information(Cog):                  infraction_output.append(line) -        return "\n".join(infraction_output) +        return "Infractions", "\n".join(infraction_output) -    async def user_nomination_counts(self, member: Member) -> str: +    async def user_nomination_counts(self, member: Member) -> Tuple[str, str]:          """Gets the active and historical nomination counts for the given `member`."""          nominations = await self.bot.api_client.get(              'bot/nominations', @@ -327,21 +365,21 @@ class Information(Cog):              }          ) -        output = ["**Nominations**"] +        output = []          if not nominations: -            output.append("This user has never been nominated.") +            output.append("No nominations")          else:              count = len(nominations)              is_currently_nominated = any(nomination["active"] for nomination in nominations)              nomination_noun = "nomination" if count == 1 else "nominations"              if is_currently_nominated: -                output.append(f"This user is **currently** nominated ({count} {nomination_noun} in total).") +                output.append(f"This user is **currently** nominated\n({count} {nomination_noun} in total)")              else:                  output.append(f"This user has {count} historical {nomination_noun}, but is currently not nominated.") -        return "\n".join(output) +        return "Nominations", "\n".join(output)      def format_fields(self, mapping: Mapping[str, Any], field_width: Optional[int] = None) -> str:          """Format a mapping to be readable to a human.""" @@ -379,7 +417,7 @@ class Information(Cog):          return out.rstrip()      @cooldown_with_role_bypass(2, 60 * 3, BucketType.member, bypass_roles=constants.STAFF_ROLES) -    @group(invoke_without_command=True) +    @group(invoke_without_command=True, enabled=False)      @in_whitelist(channels=(constants.Channels.bot_commands,), roles=constants.STAFF_ROLES)      async def raw(self, ctx: Context, *, message: Message, json: bool = False) -> None:          """Shows information about the raw API response.""" @@ -414,7 +452,7 @@ class Information(Cog):          for page in paginator.pages:              await ctx.send(page) -    @raw.command() +    @raw.command(enabled=False)      async def json(self, ctx: Context, message: Message) -> None:          """Shows information about the raw API response in a copy-pasteable Python format."""          await ctx.invoke(self.raw, message=message, json=True) diff --git a/bot/cogs/jams.py b/bot/cogs/jams.py index 1d062b0c2..b3102db2f 100644 --- a/bot/cogs/jams.py +++ b/bot/cogs/jams.py @@ -1,6 +1,7 @@  import logging +import typing as t -from discord import Member, PermissionOverwrite, utils +from discord import CategoryChannel, Guild, Member, PermissionOverwrite, Role  from discord.ext import commands  from more_itertools import unique_everseen @@ -10,6 +11,9 @@ from bot.decorators import with_role  log = logging.getLogger(__name__) +MAX_CHANNELS = 50 +CATEGORY_NAME = "Code Jam" +  class CodeJams(commands.Cog):      """Manages the code-jam related parts of our server.""" @@ -40,22 +44,47 @@ class CodeJams(commands.Cog):              )              return -        code_jam_category = utils.get(ctx.guild.categories, name="Code Jam") +        team_channel = await self.create_channels(ctx.guild, team_name, members) +        await self.add_roles(ctx.guild, members) -        if code_jam_category is None: -            log.info("Code Jam category not found, creating it.") +        await ctx.send( +            f":ok_hand: Team created: {team_channel}\n" +            f"**Team Leader:** {members[0].mention}\n" +            f"**Team Members:** {' '.join(member.mention for member in members[1:])}" +        ) -            category_overwrites = { -                ctx.guild.default_role: PermissionOverwrite(read_messages=False), -                ctx.guild.me: PermissionOverwrite(read_messages=True) -            } +    async def get_category(self, guild: Guild) -> CategoryChannel: +        """ +        Return a code jam category. -            code_jam_category = await ctx.guild.create_category_channel( -                "Code Jam", -                overwrites=category_overwrites, -                reason="It's code jam time!" -            ) +        If all categories are full or none exist, create a new category. +        """ +        for category in guild.categories: +            # Need 2 available spaces: one for the text channel and one for voice. +            if category.name == CATEGORY_NAME and MAX_CHANNELS - len(category.channels) >= 2: +                return category + +        return await self.create_category(guild) + +    @staticmethod +    async def create_category(guild: Guild) -> CategoryChannel: +        """Create a new code jam category and return it.""" +        log.info("Creating a new code jam category.") + +        category_overwrites = { +            guild.default_role: PermissionOverwrite(read_messages=False), +            guild.me: PermissionOverwrite(read_messages=True) +        } + +        return await guild.create_category_channel( +            CATEGORY_NAME, +            overwrites=category_overwrites, +            reason="It's code jam time!" +        ) +    @staticmethod +    def get_overwrites(members: t.List[Member], guild: Guild) -> t.Dict[t.Union[Member, Role], PermissionOverwrite]: +        """Get code jam team channels permission overwrites."""          # First member is always the team leader          team_channel_overwrites = {              members[0]: PermissionOverwrite( @@ -64,8 +93,8 @@ class CodeJams(commands.Cog):                  manage_webhooks=True,                  connect=True              ), -            ctx.guild.default_role: PermissionOverwrite(read_messages=False, connect=False), -            ctx.guild.get_role(Roles.verified): PermissionOverwrite( +            guild.default_role: PermissionOverwrite(read_messages=False, connect=False), +            guild.get_role(Roles.verified): PermissionOverwrite(                  read_messages=False,                  connect=False              ) @@ -78,8 +107,16 @@ class CodeJams(commands.Cog):                  connect=True              ) +        return team_channel_overwrites + +    async def create_channels(self, guild: Guild, team_name: str, members: t.List[Member]) -> str: +        """Create team text and voice channels. Return the mention for the text channel.""" +        # Get permission overwrites and category +        team_channel_overwrites = self.get_overwrites(members, guild) +        code_jam_category = await self.get_category(guild) +          # Create a text channel for the team -        team_channel = await ctx.guild.create_text_channel( +        team_channel = await guild.create_text_channel(              team_name,              overwrites=team_channel_overwrites,              category=code_jam_category @@ -88,26 +125,25 @@ class CodeJams(commands.Cog):          # Create a voice channel for the team          team_voice_name = " ".join(team_name.split("-")).title() -        await ctx.guild.create_voice_channel( +        await guild.create_voice_channel(              team_voice_name,              overwrites=team_channel_overwrites,              category=code_jam_category          ) +        return team_channel.mention + +    @staticmethod +    async def add_roles(guild: Guild, members: t.List[Member]) -> None: +        """Assign team leader and jammer roles."""          # Assign team leader role -        await members[0].add_roles(ctx.guild.get_role(Roles.team_leaders)) +        await members[0].add_roles(guild.get_role(Roles.team_leaders))          # Assign rest of roles -        jammer_role = ctx.guild.get_role(Roles.jammers) +        jammer_role = guild.get_role(Roles.jammers)          for member in members:              await member.add_roles(jammer_role) -        await ctx.send( -            f":ok_hand: Team created: {team_channel.mention}\n" -            f"**Team Leader:** {members[0].mention}\n" -            f"**Team Members:** {' '.join(member.mention for member in members[1:])}" -        ) -  def setup(bot: Bot) -> None:      """Load the CodeJams cog.""" diff --git a/bot/cogs/moderation/__init__.py b/bot/cogs/moderation/__init__.py index 6880ca1bd..995187ef0 100644 --- a/bot/cogs/moderation/__init__.py +++ b/bot/cogs/moderation/__init__.py @@ -1,15 +1,19 @@  from bot.bot import Bot +from .incidents import Incidents  from .infractions import Infractions  from .management import ModManagement  from .modlog import ModLog  from .silence import Silence +from .slowmode import Slowmode  from .superstarify import Superstarify  def setup(bot: Bot) -> None: -    """Load the Infractions, ModManagement, ModLog, Silence, and Superstarify cogs.""" +    """Load the Incidents, Infractions, ModManagement, ModLog, Silence, Slowmode and Superstarify cogs.""" +    bot.add_cog(Incidents(bot))      bot.add_cog(Infractions(bot))      bot.add_cog(ModLog(bot))      bot.add_cog(ModManagement(bot))      bot.add_cog(Silence(bot)) +    bot.add_cog(Slowmode(bot))      bot.add_cog(Superstarify(bot)) diff --git a/bot/cogs/moderation/incidents.py b/bot/cogs/moderation/incidents.py new file mode 100644 index 000000000..3605ab1d2 --- /dev/null +++ b/bot/cogs/moderation/incidents.py @@ -0,0 +1,407 @@ +import asyncio +import logging +import typing as t +from datetime import datetime +from enum import Enum + +import discord +from discord.ext.commands import Cog + +from bot.bot import Bot +from bot.constants import Channels, Colours, Emojis, Guild, Webhooks +from bot.utils.messages import sub_clyde + +log = logging.getLogger(__name__) + +# Amount of messages for `crawl_task` to process at most on start-up - limited to 50 +# as in practice, there should never be this many messages, and if there are, +# something has likely gone very wrong +CRAWL_LIMIT = 50 + +# Seconds for `crawl_task` to sleep after adding reactions to a message +CRAWL_SLEEP = 2 + + +class Signal(Enum): +    """ +    Recognized incident status signals. + +    This binds emoji to actions. The bot will only react to emoji linked here. +    All other signals are seen as invalid. +    """ + +    ACTIONED = Emojis.incident_actioned +    NOT_ACTIONED = Emojis.incident_unactioned +    INVESTIGATING = Emojis.incident_investigating + + +# Reactions from non-mod roles will be removed +ALLOWED_ROLES: t.Set[int] = set(Guild.moderation_roles) + +# Message must have all of these emoji to pass the `has_signals` check +ALL_SIGNALS: t.Set[str] = {signal.value for signal in Signal} + +# An embed coupled with an optional file to be dispatched +# If the file is not None, the embed attempts to show it in its body +FileEmbed = t.Tuple[discord.Embed, t.Optional[discord.File]] + + +async def download_file(attachment: discord.Attachment) -> t.Optional[discord.File]: +    """ +    Download & return `attachment` file. + +    If the download fails, the reason is logged and None will be returned. +    404 and 403 errors are only logged at debug level. +    """ +    log.debug(f"Attempting to download attachment: {attachment.filename}") +    try: +        return await attachment.to_file() +    except (discord.NotFound, discord.Forbidden) as exc: +        log.debug(f"Failed to download attachment: {exc}") +    except Exception: +        log.exception("Failed to download attachment") + + +async def make_embed(incident: discord.Message, outcome: Signal, actioned_by: discord.Member) -> FileEmbed: +    """ +    Create an embed representation of `incident` for the #incidents-archive channel. + +    The name & discriminator of `actioned_by` and `outcome` will be presented in the +    embed footer. Additionally, the embed is coloured based on `outcome`. + +    The author of `incident` is not shown in the embed. It is assumed that this piece +    of information will be relayed in other ways, e.g. webhook username. + +    As mentions in embeds do not ping, we do not need to use `incident.clean_content`. + +    If `incident` contains attachments, the first attachment will be downloaded and +    returned alongside the embed. The embed attempts to display the attachment. +    Should the download fail, we fallback on linking the `proxy_url`, which should +    remain functional for some time after the original message is deleted. +    """ +    log.trace(f"Creating embed for {incident.id=}") + +    if outcome is Signal.ACTIONED: +        colour = Colours.soft_green +        footer = f"Actioned by {actioned_by}" +    else: +        colour = Colours.soft_red +        footer = f"Rejected by {actioned_by}" + +    embed = discord.Embed( +        description=incident.content, +        timestamp=datetime.utcnow(), +        colour=colour, +    ) +    embed.set_footer(text=footer, icon_url=actioned_by.avatar_url) + +    if incident.attachments: +        attachment = incident.attachments[0]  # User-sent messages can only contain one attachment +        file = await download_file(attachment) + +        if file is not None: +            embed.set_image(url=f"attachment://{attachment.filename}")  # Embed displays the attached file +        else: +            embed.set_author(name="[Failed to relay attachment]", url=attachment.proxy_url)  # Embed links the file +    else: +        file = None + +    return embed, file + + +def is_incident(message: discord.Message) -> bool: +    """True if `message` qualifies as an incident, False otherwise.""" +    conditions = ( +        message.channel.id == Channels.incidents,  # Message sent in #incidents +        not message.author.bot,                    # Not by a bot +        not message.content.startswith("#"),       # Doesn't start with a hash +        not message.pinned,                        # And isn't header +    ) +    return all(conditions) + + +def own_reactions(message: discord.Message) -> t.Set[str]: +    """Get the set of reactions placed on `message` by the bot itself.""" +    return {str(reaction.emoji) for reaction in message.reactions if reaction.me} + + +def has_signals(message: discord.Message) -> bool: +    """True if `message` already has all `Signal` reactions, False otherwise.""" +    return ALL_SIGNALS.issubset(own_reactions(message)) + + +async def add_signals(incident: discord.Message) -> None: +    """ +    Add `Signal` member emoji to `incident` as reactions. + +    If the emoji has already been placed on `incident` by the bot, it will be skipped. +    """ +    existing_reacts = own_reactions(incident) + +    for signal_emoji in Signal: +        if signal_emoji.value in existing_reacts:  # This would not raise, but it is a superfluous API call +            log.trace(f"Skipping emoji as it's already been placed: {signal_emoji}") +        else: +            log.trace(f"Adding reaction: {signal_emoji}") +            await incident.add_reaction(signal_emoji.value) + + +class Incidents(Cog): +    """ +    Automation for the #incidents channel. + +    This cog does not provide a command API, it only reacts to the following events. + +    On start-up: +        * Crawl #incidents and add missing `Signal` emoji where appropriate +        * This is to retro-actively add the available options for messages which +          were sent while the bot wasn't listening +        * Pinned messages and message starting with # do not qualify as incidents +        * See: `crawl_incidents` + +    On message: +        * Add `Signal` member emoji if message qualifies as an incident +        * Ignore messages starting with # +            * Use this if verbal communication is necessary +            * Each such message must be deleted manually once appropriate +        * See: `on_message` + +    On reaction: +        * Remove reaction if not permitted +            * User does not have any of the roles in `ALLOWED_ROLES` +            * Used emoji is not a `Signal` member +        * If `Signal.ACTIONED` or `Signal.NOT_ACTIONED` were chosen, attempt to +          relay the incident message to #incidents-archive +        * If relay successful, delete original message +        * See: `on_raw_reaction_add` + +    Please refer to function docstrings for implementation details. +    """ + +    def __init__(self, bot: Bot) -> None: +        """Prepare `event_lock` and schedule `crawl_task` on start-up.""" +        self.bot = bot + +        self.event_lock = asyncio.Lock() +        self.crawl_task = self.bot.loop.create_task(self.crawl_incidents()) + +    async def crawl_incidents(self) -> None: +        """ +        Crawl #incidents and add missing emoji where necessary. + +        This is to catch-up should an incident be reported while the bot wasn't listening. +        After adding each reaction, we take a short break to avoid drowning in ratelimits. + +        Once this task is scheduled, listeners that change messages should await it. +        The crawl assumes that the channel history doesn't change as we go over it. + +        Behaviour is configured by: `CRAWL_LIMIT`, `CRAWL_SLEEP`. +        """ +        await self.bot.wait_until_guild_available() +        incidents: discord.TextChannel = self.bot.get_channel(Channels.incidents) + +        log.debug(f"Crawling messages in #incidents: {CRAWL_LIMIT=}, {CRAWL_SLEEP=}") +        async for message in incidents.history(limit=CRAWL_LIMIT): + +            if not is_incident(message): +                log.trace(f"Skipping message {message.id}: not an incident") +                continue + +            if has_signals(message): +                log.trace(f"Skipping message {message.id}: already has all signals") +                continue + +            await add_signals(message) +            await asyncio.sleep(CRAWL_SLEEP) + +        log.debug("Crawl task finished!") + +    async def archive(self, incident: discord.Message, outcome: Signal, actioned_by: discord.Member) -> bool: +        """ +        Relay an embed representation of `incident` to the #incidents-archive channel. + +        The following pieces of information are relayed: +            * Incident message content (as embed description) +            * Incident attachment (if image, shown in archive embed) +            * Incident author name (as webhook author) +            * Incident author avatar (as webhook avatar) +            * Resolution signal `outcome` (as embed colour & footer) +            * Moderator `actioned_by` (name & discriminator shown in footer) + +        If `incident` contains an attachment, we try to add it to the archive embed. There is +        no handing of extensions / file types - we simply dispatch the attachment file with the +        webhook, and try to display it in the embed. Testing indicates that if the attachment +        cannot be displayed (e.g. a text file), it's invisible in the embed, with no error. + +        Return True if the relay finishes successfully. If anything goes wrong, meaning +        not all information was relayed, return False. This signals that the original +        message is not safe to be deleted, as we will lose some information. +        """ +        log.debug(f"Archiving incident: {incident.id} (outcome: {outcome}, actioned by: {actioned_by})") +        embed, attachment_file = await make_embed(incident, outcome, actioned_by) + +        try: +            webhook = await self.bot.fetch_webhook(Webhooks.incidents_archive) +            await webhook.send( +                embed=embed, +                username=sub_clyde(incident.author.name), +                avatar_url=incident.author.avatar_url, +                file=attachment_file, +            ) +        except Exception: +            log.exception(f"Failed to archive incident {incident.id} to #incidents-archive") +            return False +        else: +            log.trace("Message archived successfully!") +            return True + +    def make_confirmation_task(self, incident: discord.Message, timeout: int = 5) -> asyncio.Task: +        """ +        Create a task to wait `timeout` seconds for `incident` to be deleted. + +        If `timeout` passes, this will raise `asyncio.TimeoutError`, signaling that we haven't +        been able to confirm that the message was deleted. +        """ +        log.trace(f"Confirmation task will wait {timeout=} seconds for {incident.id=} to be deleted") + +        def check(payload: discord.RawReactionActionEvent) -> bool: +            return payload.message_id == incident.id + +        coroutine = self.bot.wait_for(event="raw_message_delete", check=check, timeout=timeout) +        return self.bot.loop.create_task(coroutine) + +    async def process_event(self, reaction: str, incident: discord.Message, member: discord.Member) -> None: +        """ +        Process a `reaction_add` event in #incidents. + +        First, we check that the reaction is a recognized `Signal` member, and that it was sent by +        a permitted user (at least one role in `ALLOWED_ROLES`). If not, the reaction is removed. + +        If the reaction was either `Signal.ACTIONED` or `Signal.NOT_ACTIONED`, we attempt to relay +        the report to #incidents-archive. If successful, the original message is deleted. + +        We do not release `event_lock` until we receive the corresponding `message_delete` event. +        This ensures that if there is a racing event awaiting the lock, it will fail to find the +        message, and will abort. There is a `timeout` to ensure that this doesn't hold the lock +        forever should something go wrong. +        """ +        members_roles: t.Set[int] = {role.id for role in member.roles} +        if not members_roles & ALLOWED_ROLES:  # Intersection is truthy on at least 1 common element +            log.debug(f"Removing invalid reaction: user {member} is not permitted to send signals") +            await incident.remove_reaction(reaction, member) +            return + +        try: +            signal = Signal(reaction) +        except ValueError: +            log.debug(f"Removing invalid reaction: emoji {reaction} is not a valid signal") +            await incident.remove_reaction(reaction, member) +            return + +        log.trace(f"Received signal: {signal}") + +        if signal not in (Signal.ACTIONED, Signal.NOT_ACTIONED): +            log.debug("Reaction was valid, but no action is currently defined for it") +            return + +        relay_successful = await self.archive(incident, signal, actioned_by=member) +        if not relay_successful: +            log.trace("Original message will not be deleted as we failed to relay it to the archive") +            return + +        timeout = 5  # Seconds +        confirmation_task = self.make_confirmation_task(incident, timeout) + +        log.trace("Deleting original message") +        await incident.delete() + +        log.trace(f"Awaiting deletion confirmation: {timeout=} seconds") +        try: +            await confirmation_task +        except asyncio.TimeoutError: +            log.warning(f"Did not receive incident deletion confirmation within {timeout} seconds!") +        else: +            log.trace("Deletion was confirmed") + +    async def resolve_message(self, message_id: int) -> t.Optional[discord.Message]: +        """ +        Get `discord.Message` for `message_id` from cache, or API. + +        We first look into the local cache to see if the message is present. + +        If not, we try to fetch the message from the API. This is necessary for messages +        which were sent before the bot's current session. + +        In an edge-case, it is also possible that the message was already deleted, and +        the API will respond with a 404. In such a case, None will be returned. +        This signals that the event for `message_id` should be ignored. +        """ +        await self.bot.wait_until_guild_available()  # First make sure that the cache is ready +        log.trace(f"Resolving message for: {message_id=}") +        message: t.Optional[discord.Message] = self.bot._connection._get_message(message_id) + +        if message is not None: +            log.trace("Message was found in cache") +            return message + +        log.trace("Message not found, attempting to fetch") +        try: +            message = await self.bot.get_channel(Channels.incidents).fetch_message(message_id) +        except discord.NotFound: +            log.trace("Message doesn't exist, it was likely already relayed") +        except Exception: +            log.exception(f"Failed to fetch message {message_id}!") +        else: +            log.trace("Message fetched successfully!") +            return message + +    @Cog.listener() +    async def on_raw_reaction_add(self, payload: discord.RawReactionActionEvent) -> None: +        """ +        Pre-process `payload` and pass it to `process_event` if appropriate. + +        We abort instantly if `payload` doesn't relate to a message sent in #incidents, +        or if it was sent by a bot. + +        If `payload` relates to a message in #incidents, we first ensure that `crawl_task` has +        finished, to make sure we don't mutate channel state as we're crawling it. + +        Next, we acquire `event_lock` - to prevent racing, events are processed one at a time. + +        Once we have the lock, the `discord.Message` object for this event must be resolved. +        If the lock was previously held by an event which successfully relayed the incident, +        this will fail and we abort the current event. + +        Finally, with both the lock and the `discord.Message` instance in our hands, we delegate +        to `process_event` to handle the event. + +        The justification for using a raw listener is the need to receive events for messages +        which were not cached in the current session. As a result, a certain amount of +        complexity is introduced, but at the moment this doesn't appear to be avoidable. +        """ +        if payload.channel_id != Channels.incidents or payload.member.bot: +            return + +        log.trace(f"Received reaction add event in #incidents, waiting for crawler: {self.crawl_task.done()=}") +        await self.crawl_task + +        log.trace(f"Acquiring event lock: {self.event_lock.locked()=}") +        async with self.event_lock: +            message = await self.resolve_message(payload.message_id) + +            if message is None: +                log.debug("Listener will abort as related message does not exist!") +                return + +            if not is_incident(message): +                log.debug("Ignoring event for a non-incident message") +                return + +            await self.process_event(str(payload.emoji), message, payload.member) +            log.trace("Releasing event lock") + +    @Cog.listener() +    async def on_message(self, message: discord.Message) -> None: +        """Pass `message` to `add_signals` if and only if it satisfies `is_incident`.""" +        if is_incident(message): +            await add_signals(message) diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 3b28526b2..8df642428 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -64,7 +64,7 @@ class Infractions(InfractionScheduler, commands.Cog):      @command()      async def kick(self, ctx: Context, user: Member, *, reason: t.Optional[str] = None) -> None:          """Kick a user for the given reason.""" -        await self.apply_kick(ctx, user, reason, active=False) +        await self.apply_kick(ctx, user, reason)      @command()      async def ban(self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None) -> None: @@ -134,7 +134,7 @@ class Infractions(InfractionScheduler, commands.Cog):      @command(hidden=True, aliases=['shadowkick', 'skick'])      async def shadow_kick(self, ctx: Context, user: Member, *, reason: t.Optional[str] = None) -> None:          """Kick a user for the given reason without notifying the user.""" -        await self.apply_kick(ctx, user, reason, hidden=True, active=False) +        await self.apply_kick(ctx, user, reason, hidden=True)      @command(hidden=True, aliases=['shadowban', 'sban'])      async def shadow_ban(self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None) -> None: diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index c39c7f3bc..672bb0e9c 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -135,11 +135,11 @@ class ModManagement(commands.Cog):          if 'expires_at' in request_data:              # A scheduled task should only exist if the old infraction wasn't permanent              if old_infraction['expires_at']: -                self.infractions_cog.cancel_task(new_infraction['id']) +                self.infractions_cog.scheduler.cancel(new_infraction['id'])              # If the infraction was not marked as permanent, schedule a new expiration task              if request_data['expires_at']: -                self.infractions_cog.schedule_task(new_infraction['id'], new_infraction) +                self.infractions_cog.schedule_expiration(new_infraction)              log_text += f"""                  Previous expiry: {old_infraction['expires_at'] or "Permanent"} @@ -268,12 +268,12 @@ class ModManagement(commands.Cog):              User: {self.bot.get_user(user_id)} (`{user_id}`)              Type: **{infraction["type"]}**              Shadow: {hidden} -            Reason: {infraction["reason"] or "*None*"}              Created: {created}              Expires: {expires}              Remaining: {remaining}              Actor: {actor.mention if actor else actor_id}              ID: `{infraction["id"]}` +            Reason: {infraction["reason"] or "*None*"}              {"**===============**" if active else "==============="}          """) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index 41472c64c..5f30d3744 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -24,7 +24,6 @@ GUILD_CHANNEL = t.Union[discord.CategoryChannel, discord.TextChannel, discord.Vo  CHANNEL_CHANGES_UNSUPPORTED = ("permissions",)  CHANNEL_CHANGES_SUPPRESSED = ("_overwrites", "position") -MEMBER_CHANGES_SUPPRESSED = ("status", "activities", "_client_status", "nick")  ROLE_CHANGES_UNSUPPORTED = ("colour", "permissions")  VOICE_STATE_ATTRIBUTES = { @@ -121,8 +120,17 @@ class ModLog(Cog, name="ModLog"):              else:                  content = "@everyone" +        # Truncate content to 2000 characters and append an ellipsis. +        if content and len(content) > 2000: +            content = content[:2000 - 3] + "..." +          channel = self.bot.get_channel(channel_id) -        log_message = await channel.send(content=content, embed=embed, files=files) +        log_message = await channel.send( +            content=content, +            embed=embed, +            files=files, +            allowed_mentions=discord.AllowedMentions(everyone=True) +        )          if additional_embeds:              if additional_embeds_msg: @@ -452,6 +460,21 @@ class ModLog(Cog, name="ModLog"):              channel_id=Channels.mod_log          ) +    @staticmethod +    def get_role_diff(before: t.List[discord.Role], after: t.List[discord.Role]) -> t.List[str]: +        """Return a list of strings describing the roles added and removed.""" +        changes = [] +        before_roles = set(before) +        after_roles = set(after) + +        for role in (before_roles - after_roles): +            changes.append(f"**Role removed:** {role.name} (`{role.id}`)") + +        for role in (after_roles - before_roles): +            changes.append(f"**Role added:** {role.name} (`{role.id}`)") + +        return changes +      @Cog.listener()      async def on_member_update(self, before: discord.Member, after: discord.Member) -> None:          """Log member update event to user log.""" @@ -462,74 +485,27 @@ class ModLog(Cog, name="ModLog"):              self._ignored[Event.member_update].remove(before.id)              return -        diff = DeepDiff(before, after) -        changes = [] -        done = [] - -        diff_values = {} +        changes = self.get_role_diff(before.roles, after.roles) -        diff_values.update(diff.get("values_changed", {})) -        diff_values.update(diff.get("type_changes", {})) -        diff_values.update(diff.get("iterable_item_removed", {})) -        diff_values.update(diff.get("iterable_item_added", {})) - -        diff_user = DeepDiff(before._user, after._user) - -        diff_values.update(diff_user.get("values_changed", {})) -        diff_values.update(diff_user.get("type_changes", {})) -        diff_values.update(diff_user.get("iterable_item_removed", {})) -        diff_values.update(diff_user.get("iterable_item_added", {})) - -        for key, value in diff_values.items(): -            if not key:  # Not sure why, but it happens -                continue - -            key = key[5:]  # Remove "root." prefix - -            if "[" in key: -                key = key.split("[", 1)[0] +        # The regex is a simple way to exclude all sequence and mapping types. +        diff = DeepDiff(before, after, exclude_regex_paths=r".*\[.*") -            if "." in key: -                key = key.split(".", 1)[0] +        # A type change seems to always take precedent over a value change. Furthermore, it will +        # include the value change along with the type change anyway. Therefore, it's OK to +        # "overwrite" values_changed; in practice there will never even be anything to overwrite. +        diff_values = {**diff.get("values_changed", {}), **diff.get("type_changes", {})} -            if key in done or key in MEMBER_CHANGES_SUPPRESSED: +        for attr, value in diff_values.items(): +            if not attr:  # Not sure why, but it happens.                  continue -            if key == "_roles": -                new_roles = after.roles -                old_roles = before.roles - -                for role in old_roles: -                    if role not in new_roles: -                        changes.append(f"**Role removed:** {role.name} (`{role.id}`)") - -                for role in new_roles: -                    if role not in old_roles: -                        changes.append(f"**Role added:** {role.name} (`{role.id}`)") - -            else: -                new = value.get("new_value") -                old = value.get("old_value") - -                if new and old: -                    changes.append(f"**{key.title()}:** `{old}` **→** `{new}`") - -            done.append(key) - -        if before.name != after.name: -            changes.append( -                f"**Username:** `{before.name}` **→** `{after.name}`" -            ) +            attr = attr[5:]  # Remove "root." prefix. +            attr = attr.replace("_", " ").replace(".", " ").capitalize() -        if before.discriminator != after.discriminator: -            changes.append( -                f"**Discriminator:** `{before.discriminator}` **→** `{after.discriminator}`" -            ) +            new = value.get("new_value") +            old = value.get("old_value") -        if before.display_name != after.display_name: -            changes.append( -                f"**Display name:** `{before.display_name}` **→** `{after.display_name}`" -            ) +            changes.append(f"**{attr}:** `{old}` **→** `{new}`")          if not changes:              return @@ -543,8 +519,10 @@ class ModLog(Cog, name="ModLog"):          message = f"**{member_str}** (`{after.id}`)\n{message}"          await self.send_log_message( -            Icons.user_update, Colour.blurple(), -            "Member updated", message, +            icon_url=Icons.user_update, +            colour=Colour.blurple(), +            title="Member updated", +            text=message,              thumbnail=after.avatar_url_as(static_format="png"),              channel_id=Channels.user_log          ) diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index d75a72ddb..051f6c52c 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -1,4 +1,3 @@ -import asyncio  import logging  import textwrap  import typing as t @@ -23,15 +22,19 @@ from .utils import UserSnowflake  log = logging.getLogger(__name__) -class InfractionScheduler(Scheduler): +class InfractionScheduler:      """Handles the application, pardoning, and expiration of infractions."""      def __init__(self, bot: Bot, supported_infractions: t.Container[str]): -        super().__init__() -          self.bot = bot +        self.scheduler = Scheduler(self.__class__.__name__) +          self.bot.loop.create_task(self.reschedule_infractions(supported_infractions)) +    def cog_unload(self) -> None: +        """Cancel scheduled tasks.""" +        self.scheduler.cancel_all() +      @property      def mod_log(self) -> ModLog:          """Get the currently loaded ModLog cog instance.""" @@ -49,7 +52,7 @@ class InfractionScheduler(Scheduler):          )          for infraction in infractions:              if infraction["expires_at"] is not None and infraction["type"] in supported_infractions: -                self.schedule_task(infraction["id"], infraction) +                self.schedule_expiration(infraction)      async def reapply_infraction(          self, @@ -155,9 +158,10 @@ class InfractionScheduler(Scheduler):                  await action_coro                  if expiry:                      # Schedule the expiration of the infraction. -                    self.schedule_task(infraction["id"], infraction) +                    self.schedule_expiration(infraction)              except discord.HTTPException as e:                  # Accordingly display that applying the infraction failed. +                # Don't use ctx.message.author; antispam only patches ctx.author.                  confirm_msg = ":x: failed to apply"                  expiry_msg = ""                  log_content = ctx.author.mention @@ -187,6 +191,7 @@ class InfractionScheduler(Scheduler):          await ctx.send(f"{dm_result}{confirm_msg}{infr_message}.")          # Send a log message to the mod log. +        # Don't use ctx.message.author for the actor; antispam only patches ctx.author.          log.trace(f"Sending apply mod log for infraction #{id_}.")          await self.mod_log.send_log_message(              icon_url=icon, @@ -195,7 +200,7 @@ class InfractionScheduler(Scheduler):              thumbnail=user.avatar_url_as(static_format="png"),              text=textwrap.dedent(f"""                  Member: {user.mention} (`{user.id}`) -                Actor: {ctx.message.author}{dm_log_text}{expiry_log_text} +                Actor: {ctx.author}{dm_log_text}{expiry_log_text}                  Reason: {reason}              """),              content=log_content, @@ -239,7 +244,7 @@ class InfractionScheduler(Scheduler):          log_text = await self.deactivate_infraction(response[0], send_log=False)          log_text["Member"] = f"{user.mention}(`{user.id}`)" -        log_text["Actor"] = str(ctx.message.author) +        log_text["Actor"] = str(ctx.author)          log_content = None          id_ = response[0]['id']          footer = f"ID: {id_}" @@ -278,7 +283,7 @@ class InfractionScheduler(Scheduler):                  # Cancel pending expiration task.                  if infraction["expires_at"] is not None: -                    self.cancel_task(infraction["id"]) +                    self.scheduler.cancel(infraction["id"])          # Accordingly display whether the user was successfully notified via DM.          dm_emoji = "" @@ -415,7 +420,7 @@ class InfractionScheduler(Scheduler):          # Cancel the expiration task.          if infraction["expires_at"] is not None: -            self.cancel_task(infraction["id"]) +            self.scheduler.cancel(infraction["id"])          # Send a log message to the mod log.          if send_log: @@ -449,7 +454,7 @@ class InfractionScheduler(Scheduler):          """          raise NotImplementedError -    async def _scheduled_task(self, infraction: utils.Infraction) -> None: +    def schedule_expiration(self, infraction: utils.Infraction) -> None:          """          Marks an infraction expired after the delay from time of scheduling to time of expiration. @@ -457,8 +462,4 @@ class InfractionScheduler(Scheduler):          expiration task is cancelled.          """          expiry = dateutil.parser.isoparse(infraction["expires_at"]).replace(tzinfo=None) -        await time.wait_until(expiry) - -        # Because deactivate_infraction() explicitly cancels this scheduled task, it is shielded -        # to avoid prematurely cancelling itself. -        await asyncio.shield(self.deactivate_infraction(infraction)) +        self.scheduler.schedule_at(expiry, infraction["id"], self.deactivate_infraction(infraction)) diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index c8ab6443b..f8a6592bc 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -1,7 +1,7 @@  import asyncio  import logging  from contextlib import suppress -from typing import NamedTuple, Optional +from typing import Optional  from discord import TextChannel  from discord.ext import commands, tasks @@ -16,13 +16,6 @@ from bot.utils.scheduling import Scheduler  log = logging.getLogger(__name__) -class TaskData(NamedTuple): -    """Data for a scheduled task.""" - -    delay: int -    ctx: Context - -  class SilenceNotifier(tasks.Loop):      """Loop notifier for posting notices to `alert_channel` containing added channels.""" @@ -61,25 +54,17 @@ class SilenceNotifier(tasks.Loop):              await self._alert_channel.send(f"<@&{Roles.moderators}> currently silenced channels: {channels_text}") -class Silence(Scheduler, commands.Cog): +class Silence(commands.Cog):      """Commands for stopping channel messages for `verified` role in a channel."""      def __init__(self, bot: Bot): -        super().__init__()          self.bot = bot +        self.scheduler = Scheduler(self.__class__.__name__)          self.muted_channels = set() +          self._get_instance_vars_task = self.bot.loop.create_task(self._get_instance_vars())          self._get_instance_vars_event = asyncio.Event() -    async def _scheduled_task(self, task: TaskData) -> None: -        """Calls `self.unsilence` on expired silenced channel to unsilence it.""" -        await asyncio.sleep(task.delay) -        log.info("Unsilencing channel after set delay.") - -        # Because `self.unsilence` explicitly cancels this scheduled task, it is shielded -        # to avoid prematurely cancelling itself -        await asyncio.shield(task.ctx.invoke(self.unsilence)) -      async def _get_instance_vars(self) -> None:          """Get instance variables after they're available to get from the guild."""          await self.bot.wait_until_guild_available() @@ -109,12 +94,7 @@ class Silence(Scheduler, commands.Cog):          await ctx.send(f"{Emojis.check_mark} silenced current channel for {duration} minute(s).") -        task_data = TaskData( -            delay=duration*60, -            ctx=ctx -        ) - -        self.schedule_task(ctx.channel.id, task_data) +        self.scheduler.schedule_later(duration * 60, ctx.channel.id, ctx.invoke(self.unsilence))      @commands.command(aliases=("unhush",))      async def unsilence(self, ctx: Context) -> None: @@ -164,7 +144,7 @@ class Silence(Scheduler, commands.Cog):          if current_overwrite.send_messages is False:              await channel.set_permissions(self._verified_role, **dict(current_overwrite, send_messages=None))              log.info(f"Unsilenced channel #{channel} ({channel.id}).") -            self.cancel_task(channel.id) +            self.scheduler.cancel(channel.id)              self.notifier.remove_channel(channel)              self.muted_channels.discard(channel)              return True @@ -172,7 +152,8 @@ class Silence(Scheduler, commands.Cog):          return False      def cog_unload(self) -> None: -        """Send alert with silenced channels on unload.""" +        """Send alert with silenced channels and cancel scheduled tasks on unload.""" +        self.scheduler.cancel_all()          if self.muted_channels:              channels_string = ''.join(channel.mention for channel in self.muted_channels)              message = f"<@&{Roles.moderators}> channels left silenced on cog unload: {channels_string}" diff --git a/bot/cogs/moderation/slowmode.py b/bot/cogs/moderation/slowmode.py new file mode 100644 index 000000000..1d055afac --- /dev/null +++ b/bot/cogs/moderation/slowmode.py @@ -0,0 +1,97 @@ +import logging +from datetime import datetime +from typing import Optional + +from dateutil.relativedelta import relativedelta +from discord import TextChannel +from discord.ext.commands import Cog, Context, group + +from bot.bot import Bot +from bot.constants import Emojis, MODERATION_ROLES +from bot.converters import DurationDelta +from bot.decorators import with_role_check +from bot.utils import time + +log = logging.getLogger(__name__) + +SLOWMODE_MAX_DELAY = 21600  # seconds + + +class Slowmode(Cog): +    """Commands for getting and setting slowmode delays of text channels.""" + +    def __init__(self, bot: Bot) -> None: +        self.bot = bot + +    @group(name='slowmode', aliases=['sm'], invoke_without_command=True) +    async def slowmode_group(self, ctx: Context) -> None: +        """Get or set the slowmode delay for the text channel this was invoked in or a given text channel.""" +        await ctx.send_help(ctx.command) + +    @slowmode_group.command(name='get', aliases=['g']) +    async def get_slowmode(self, ctx: Context, channel: Optional[TextChannel]) -> None: +        """Get the slowmode delay for a text channel.""" +        # Use the channel this command was invoked in if one was not given +        if channel is None: +            channel = ctx.channel + +        delay = relativedelta(seconds=channel.slowmode_delay) +        humanized_delay = time.humanize_delta(delay) + +        await ctx.send(f'The slowmode delay for {channel.mention} is {humanized_delay}.') + +    @slowmode_group.command(name='set', aliases=['s']) +    async def set_slowmode(self, ctx: Context, channel: Optional[TextChannel], delay: DurationDelta) -> None: +        """Set the slowmode delay for a text channel.""" +        # Use the channel this command was invoked in if one was not given +        if channel is None: +            channel = ctx.channel + +        # Convert `dateutil.relativedelta.relativedelta` to `datetime.timedelta` +        # Must do this to get the delta in a particular unit of time +        utcnow = datetime.utcnow() +        slowmode_delay = (utcnow + delay - utcnow).total_seconds() + +        humanized_delay = time.humanize_delta(delay) + +        # Ensure the delay is within discord's limits +        if slowmode_delay <= SLOWMODE_MAX_DELAY: +            log.info(f'{ctx.author} set the slowmode delay for #{channel} to {humanized_delay}.') + +            await channel.edit(slowmode_delay=slowmode_delay) +            await ctx.send( +                f'{Emojis.check_mark} The slowmode delay for {channel.mention} is now {humanized_delay}.' +            ) + +        else: +            log.info( +                f'{ctx.author} tried to set the slowmode delay of #{channel} to {humanized_delay}, ' +                'which is not between 0 and 6 hours.' +            ) + +            await ctx.send( +                f'{Emojis.cross_mark} The slowmode delay must be between 0 and 6 hours.' +            ) + +    @slowmode_group.command(name='reset', aliases=['r']) +    async def reset_slowmode(self, ctx: Context, channel: Optional[TextChannel]) -> None: +        """Reset the slowmode delay for a text channel to 0 seconds.""" +        # Use the channel this command was invoked in if one was not given +        if channel is None: +            channel = ctx.channel + +        log.info(f'{ctx.author} reset the slowmode delay for #{channel} to 0 seconds.') + +        await channel.edit(slowmode_delay=0) +        await ctx.send( +            f'{Emojis.check_mark} The slowmode delay for {channel.mention} has been reset to 0 seconds.' +        ) + +    def cog_check(self, ctx: Context) -> bool: +        """Only allow moderators to invoke the commands in this cog.""" +        return with_role_check(ctx, *MODERATION_ROLES) + + +def setup(bot: Bot) -> None: +    """Load the Slowmode cog.""" +    bot.add_cog(Slowmode(bot)) diff --git a/bot/cogs/moderation/superstarify.py b/bot/cogs/moderation/superstarify.py index 45a010f00..867de815a 100644 --- a/bot/cogs/moderation/superstarify.py +++ b/bot/cogs/moderation/superstarify.py @@ -146,7 +146,7 @@ class Superstarify(InfractionScheduler, Cog):          log.debug(f"Changing nickname of {member} to {forced_nick}.")          self.mod_log.ignore(constants.Event.member_update, member.id)          await member.edit(nick=forced_nick, reason=reason) -        self.schedule_task(id_, infraction) +        self.schedule_expiration(infraction)          # Send a DM to the user to notify them of their new infraction.          await utils.notify_infraction( diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py index fb55287b6..f21272102 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/cogs/moderation/utils.py @@ -70,7 +70,7 @@ async def post_infraction(      log.trace(f"Posting {infr_type} infraction for {user} to the API.")      payload = { -        "actor": ctx.message.author.id, +        "actor": ctx.author.id,  # Don't use ctx.message.author; antispam only patches ctx.author.          "hidden": hidden,          "reason": reason,          "type": infr_type, diff --git a/bot/cogs/off_topic_names.py b/bot/cogs/off_topic_names.py index 201579a0b..ce95450e0 100644 --- a/bot/cogs/off_topic_names.py +++ b/bot/cogs/off_topic_names.py @@ -4,46 +4,19 @@ import logging  from datetime import datetime, timedelta  from discord import Colour, Embed -from discord.ext.commands import BadArgument, Cog, Context, Converter, group +from discord.ext.commands import Cog, Context, group  from bot.api import ResponseCodeError  from bot.bot import Bot  from bot.constants import Channels, MODERATION_ROLES +from bot.converters import OffTopicName  from bot.decorators import with_role  from bot.pagination import LinePaginator -  CHANNELS = (Channels.off_topic_0, Channels.off_topic_1, Channels.off_topic_2)  log = logging.getLogger(__name__) -class OffTopicName(Converter): -    """A converter that ensures an added off-topic name is valid.""" - -    @staticmethod -    async def convert(ctx: Context, argument: str) -> str: -        """Attempt to replace any invalid characters with their approximate Unicode equivalent.""" -        allowed_characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ!?'`-" - -        # Chain multiple words to a single one -        argument = "-".join(argument.split()) - -        if not (2 <= len(argument) <= 96): -            raise BadArgument("Channel name must be between 2 and 96 chars long") - -        elif not all(c.isalnum() or c in allowed_characters for c in argument): -            raise BadArgument( -                "Channel name must only consist of " -                "alphanumeric characters, minus signs or apostrophes." -            ) - -        # Replace invalid characters with unicode alternatives. -        table = str.maketrans( -            allowed_characters, '𝖠𝖡𝖢𝖣𝖤𝖥𝖦𝖧𝖨𝖩𝖪𝖫𝖬𝖭𝖮𝖯𝖰𝖱𝖲𝖳𝖴𝖵𝖶𝖷𝖸𝖹ǃ?’’-' -        ) -        return argument.translate(table) - -  async def update_names(bot: Bot) -> None:      """Background updater task that performs the daily channel name update."""      while True: diff --git a/bot/cogs/python_news.py b/bot/cogs/python_news.py index adefd5c7c..0ab5738a4 100644 --- a/bot/cogs/python_news.py +++ b/bot/cogs/python_news.py @@ -10,7 +10,7 @@ from discord.ext.tasks import loop  from bot import constants  from bot.bot import Bot -from bot.utils.messages import sub_clyde +from bot.utils.webhooks import send_webhook  PEPS_RSS_URL = "https://www.python.org/dev/peps/peps.rss/" @@ -100,13 +100,21 @@ class PythonNews(Cog):              ):                  continue -            msg = await self.send_webhook( +            # Build an embed and send a webhook +            embed = discord.Embed(                  title=new["title"],                  description=new["summary"],                  timestamp=new_datetime,                  url=new["link"], -                webhook_profile_name=data["feed"]["title"], -                footer=data["feed"]["title"] +                colour=constants.Colours.soft_green +            ) +            embed.set_footer(text=data["feed"]["title"], icon_url=AVATAR_URL) +            msg = await send_webhook( +                webhook=self.webhook, +                username=data["feed"]["title"], +                embed=embed, +                avatar_url=AVATAR_URL, +                wait=True,              )              payload["data"]["pep"].append(pep_nr) @@ -161,15 +169,29 @@ class PythonNews(Cog):                  content = email_information["content"]                  link = THREAD_URL.format(id=thread["href"].split("/")[-2], list=maillist) -                msg = await self.send_webhook( + +                # Build an embed and send a message to the webhook +                embed = discord.Embed(                      title=thread_information["subject"],                      description=content[:500] + f"... [continue reading]({link})" if len(content) > 500 else content,                      timestamp=new_date,                      url=link, -                    author=f"{email_information['sender_name']} ({email_information['sender']['address']})", -                    author_url=MAILMAN_PROFILE_URL.format(id=email_information["sender"]["mailman_id"]), -                    webhook_profile_name=self.webhook_names[maillist], -                    footer=f"Posted to {self.webhook_names[maillist]}" +                    colour=constants.Colours.soft_green +                ) +                embed.set_author( +                    name=f"{email_information['sender_name']} ({email_information['sender']['address']})", +                    url=MAILMAN_PROFILE_URL.format(id=email_information["sender"]["mailman_id"]), +                ) +                embed.set_footer( +                    text=f"Posted to {self.webhook_names[maillist]}", +                    icon_url=AVATAR_URL, +                ) +                msg = await send_webhook( +                    webhook=self.webhook, +                    username=self.webhook_names[maillist], +                    embed=embed, +                    avatar_url=AVATAR_URL, +                    wait=True,                  )                  payload["data"][maillist].append(thread_information["thread_id"]) @@ -182,38 +204,6 @@ class PythonNews(Cog):          await self.bot.api_client.put("bot/bot-settings/news", json=payload) -    async def send_webhook(self, -                           title: str, -                           description: str, -                           timestamp: datetime, -                           url: str, -                           webhook_profile_name: str, -                           footer: str, -                           author: t.Optional[str] = None, -                           author_url: t.Optional[str] = None, -                           ) -> discord.Message: -        """Send webhook entry and return sent message.""" -        embed = discord.Embed( -            title=title, -            description=description, -            timestamp=timestamp, -            url=url, -            colour=constants.Colours.soft_green -        ) -        if author and author_url: -            embed.set_author( -                name=author, -                url=author_url -            ) -        embed.set_footer(text=footer, icon_url=AVATAR_URL) - -        return await self.webhook.send( -            embed=embed, -            username=sub_clyde(webhook_profile_name), -            avatar_url=AVATAR_URL, -            wait=True -        ) -      async def get_thread_and_first_mail(self, maillist: str, thread_identifier: str) -> t.Tuple[t.Any, t.Any]:          """Get mail thread and first mail from mail.python.org based on `maillist` and `thread_identifier`."""          async with self.bot.http_session.get( diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index d853ab2ea..5d9e2c20b 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -10,6 +10,7 @@ from aiohttp import BasicAuth, ClientError  from discord import Colour, Embed, TextChannel  from discord.ext.commands import Cog, Context, group  from discord.ext.tasks import loop +from discord.utils import escape_markdown  from bot.bot import Bot  from bot.constants import Channels, ERROR_REPLIES, Emojis, Reddit as RedditConfig, STAFF_ROLES, Webhooks @@ -187,6 +188,8 @@ class Reddit(Cog):              author = data["author"]              title = textwrap.shorten(data["title"], width=64, placeholder="...") +            # Normal brackets interfere with Markdown. +            title = escape_markdown(title).replace("[", "⦋").replace("]", "⦌")              link = self.URL + data["permalink"]              embed.description += ( diff --git a/bot/cogs/reminders.py b/bot/cogs/reminders.py index c242d2920..08bce2153 100644 --- a/bot/cogs/reminders.py +++ b/bot/cogs/reminders.py @@ -9,31 +9,38 @@ from operator import itemgetter  import discord  from dateutil.parser import isoparse  from dateutil.relativedelta import relativedelta -from discord.ext.commands import Cog, Context, group +from discord.ext.commands import Cog, Context, Greedy, group  from bot.bot import Bot -from bot.constants import Guild, Icons, NEGATIVE_REPLIES, POSITIVE_REPLIES, STAFF_ROLES +from bot.constants import Guild, Icons, MODERATION_ROLES, POSITIVE_REPLIES, Roles, STAFF_ROLES  from bot.converters import Duration  from bot.pagination import LinePaginator -from bot.utils.checks import without_role_check +from bot.utils.checks import with_role_check, without_role_check +from bot.utils.messages import send_denial  from bot.utils.scheduling import Scheduler -from bot.utils.time import humanize_delta, wait_until +from bot.utils.time import humanize_delta  log = logging.getLogger(__name__)  WHITELISTED_CHANNELS = Guild.reminder_whitelist  MAXIMUM_REMINDERS = 5 +Mentionable = t.Union[discord.Member, discord.Role] -class Reminders(Scheduler, Cog): + +class Reminders(Cog):      """Provide in-channel reminder functionality."""      def __init__(self, bot: Bot):          self.bot = bot -        super().__init__() +        self.scheduler = Scheduler(self.__class__.__name__)          self.bot.loop.create_task(self.reschedule_reminders()) +    def cog_unload(self) -> None: +        """Cancel scheduled tasks.""" +        self.scheduler.cancel_all() +      async def reschedule_reminders(self) -> None:          """Get all current reminders from the API and reschedule them."""          await self.bot.wait_until_guild_available() @@ -56,7 +63,7 @@ class Reminders(Scheduler, Cog):                  late = relativedelta(now, remind_at)                  await self.send_reminder(reminder, late)              else: -                self.schedule_task(reminder["id"], reminder) +                self.schedule_reminder(reminder)      def ensure_valid_reminder(          self, @@ -99,17 +106,58 @@ class Reminders(Scheduler, Cog):          await ctx.send(embed=embed) -    async def _scheduled_task(self, reminder: dict) -> None: +    @staticmethod +    async def _check_mentions(ctx: Context, mentions: t.Iterable[Mentionable]) -> t.Tuple[bool, str]: +        """ +        Returns whether or not the list of mentions is allowed. + +        Conditions: +        - Role reminders are Mods+ +        - Reminders for other users are Helpers+ + +        If mentions aren't allowed, also return the type of mention(s) disallowed. +        """ +        if without_role_check(ctx, *STAFF_ROLES): +            return False, "members/roles" +        elif without_role_check(ctx, *MODERATION_ROLES): +            return all(isinstance(mention, discord.Member) for mention in mentions), "roles" +        else: +            return True, "" + +    @staticmethod +    async def validate_mentions(ctx: Context, mentions: t.Iterable[Mentionable]) -> bool: +        """ +        Filter mentions to see if the user can mention, and sends a denial if not allowed. + +        Returns whether or not the validation is successful. +        """ +        mentions_allowed, disallowed_mentions = await Reminders._check_mentions(ctx, mentions) + +        if not mentions or mentions_allowed: +            return True +        else: +            await send_denial(ctx, f"You can't mention other {disallowed_mentions} in your reminder!") +            return False + +    def get_mentionables(self, mention_ids: t.List[int]) -> t.Iterator[Mentionable]: +        """Converts Role and Member ids to their corresponding objects if possible.""" +        guild = self.bot.get_guild(Guild.id) +        for mention_id in mention_ids: +            if (mentionable := (guild.get_member(mention_id) or guild.get_role(mention_id))): +                yield mentionable + +    def schedule_reminder(self, reminder: dict) -> None:          """A coroutine which sends the reminder once the time is reached, and cancels the running task."""          reminder_id = reminder["id"]          reminder_datetime = isoparse(reminder['expiration']).replace(tzinfo=None) -        # Send the reminder message once the desired duration has passed -        await wait_until(reminder_datetime) -        await self.send_reminder(reminder) +        async def _remind() -> None: +            await self.send_reminder(reminder) + +            log.debug(f"Deleting reminder {reminder_id} (the user has been reminded).") +            await self._delete_reminder(reminder_id) -        log.debug(f"Deleting reminder {reminder_id} (the user has been reminded).") -        await self._delete_reminder(reminder_id) +        self.scheduler.schedule_at(reminder_datetime, reminder_id, _remind())      async def _delete_reminder(self, reminder_id: str, cancel_task: bool = True) -> None:          """Delete a reminder from the database, given its ID, and cancel the running task.""" @@ -117,15 +165,28 @@ class Reminders(Scheduler, Cog):          if cancel_task:              # Now we can remove it from the schedule list -            self.cancel_task(reminder_id) +            self.scheduler.cancel(reminder_id) + +    async def _edit_reminder(self, reminder_id: int, payload: dict) -> dict: +        """ +        Edits a reminder in the database given the ID and payload. + +        Returns the edited reminder. +        """ +        # Send the request to update the reminder in the database +        reminder = await self.bot.api_client.patch( +            'bot/reminders/' + str(reminder_id), +            json=payload +        ) +        return reminder      async def _reschedule_reminder(self, reminder: dict) -> None:          """Reschedule a reminder object."""          log.trace(f"Cancelling old task #{reminder['id']}") -        self.cancel_task(reminder["id"]) +        self.scheduler.cancel(reminder["id"])          log.trace(f"Scheduling new task #{reminder['id']}") -        self.schedule_task(reminder["id"], reminder) +        self.schedule_reminder(reminder)      async def send_reminder(self, reminder: dict, late: relativedelta = None) -> None:          """Send the reminder.""" @@ -152,36 +213,39 @@ class Reminders(Scheduler, Cog):                  name=f"Sorry it arrived {humanize_delta(late, max_units=2)} late!"              ) +        additional_mentions = ' '.join( +            mentionable.mention for mentionable in self.get_mentionables(reminder["mentions"]) +        ) +          await channel.send( -            content=user.mention, +            content=f"{user.mention} {additional_mentions}",              embed=embed          )          await self._delete_reminder(reminder["id"])      @group(name="remind", aliases=("reminder", "reminders", "remindme"), invoke_without_command=True) -    async def remind_group(self, ctx: Context, expiration: Duration, *, content: str) -> None: +    async def remind_group( +        self, ctx: Context, mentions: Greedy[Mentionable], expiration: Duration, *, content: str +    ) -> None:          """Commands for managing your reminders.""" -        await ctx.invoke(self.new_reminder, expiration=expiration, content=content) +        await ctx.invoke(self.new_reminder, mentions=mentions, expiration=expiration, content=content)      @remind_group.command(name="new", aliases=("add", "create")) -    async def new_reminder(self, ctx: Context, expiration: Duration, *, content: str) -> t.Optional[discord.Message]: +    async def new_reminder( +        self, ctx: Context, mentions: Greedy[Mentionable], expiration: Duration, *, content: str +    ) -> None:          """          Set yourself a simple reminder.          Expiration is parsed per: http://strftime.org/          """ -        embed = discord.Embed() -          # If the user is not staff, we need to verify whether or not to make a reminder at all.          if without_role_check(ctx, *STAFF_ROLES):              # If they don't have permission to set a reminder in this channel              if ctx.channel.id not in WHITELISTED_CHANNELS: -                embed.colour = discord.Colour.red() -                embed.title = random.choice(NEGATIVE_REPLIES) -                embed.description = "Sorry, you can't do that here!" - -                return await ctx.send(embed=embed) +                await send_denial(ctx, "Sorry, you can't do that here!") +                return              # Get their current active reminders              active_reminders = await self.bot.api_client.get( @@ -194,11 +258,18 @@ class Reminders(Scheduler, Cog):              # Let's limit this, so we don't get 10 000              # reminders from kip or something like that :P              if len(active_reminders) > MAXIMUM_REMINDERS: -                embed.colour = discord.Colour.red() -                embed.title = random.choice(NEGATIVE_REPLIES) -                embed.description = "You have too many active reminders!" +                await send_denial(ctx, "You have too many active reminders!") +                return -                return await ctx.send(embed=embed) +        # Remove duplicate mentions +        mentions = set(mentions) +        mentions.discard(ctx.author) + +        # Filter mentions to see if the user can mention members/roles +        if not await self.validate_mentions(ctx, mentions): +            return + +        mention_ids = [mention.id for mention in mentions]          # Now we can attempt to actually set the reminder.          reminder = await self.bot.api_client.post( @@ -208,25 +279,30 @@ class Reminders(Scheduler, Cog):                  'channel_id': ctx.message.channel.id,                  'jump_url': ctx.message.jump_url,                  'content': content, -                'expiration': expiration.isoformat() +                'expiration': expiration.isoformat(), +                'mentions': mention_ids,              }          )          now = datetime.utcnow() - timedelta(seconds=1)          humanized_delta = humanize_delta(relativedelta(expiration, now)) +        mention_string = ( +            f"Your reminder will arrive in {humanized_delta} " +            f"and will mention {len(mentions)} other(s)!" +        )          # Confirm to the user that it worked.          await self._send_confirmation(              ctx, -            on_success=f"Your reminder will arrive in {humanized_delta}!", +            on_success=mention_string,              reminder_id=reminder["id"],              delivery_dt=expiration,          ) -        self.schedule_task(reminder["id"], reminder) +        self.schedule_reminder(reminder)      @remind_group.command(name="list") -    async def list_reminders(self, ctx: Context) -> t.Optional[discord.Message]: +    async def list_reminders(self, ctx: Context) -> None:          """View a paginated embed of all reminders for your user."""          # Get all the user's reminders from the database.          data = await self.bot.api_client.get( @@ -239,7 +315,7 @@ class Reminders(Scheduler, Cog):          # Make a list of tuples so it can be sorted by time.          reminders = sorted(              ( -                (rem['content'], rem['expiration'], rem['id']) +                (rem['content'], rem['expiration'], rem['id'], rem['mentions'])                  for rem in data              ),              key=itemgetter(1) @@ -247,13 +323,19 @@ class Reminders(Scheduler, Cog):          lines = [] -        for content, remind_at, id_ in reminders: +        for content, remind_at, id_, mentions in reminders:              # Parse and humanize the time, make it pretty :D              remind_datetime = isoparse(remind_at).replace(tzinfo=None)              time = humanize_delta(relativedelta(remind_datetime, now)) +            mentions = ", ".join( +                # Both Role and User objects have the `name` attribute +                mention.name for mention in self.get_mentionables(mentions) +            ) +            mention_string = f"\n**Mentions:** {mentions}" if mentions else "" +              text = textwrap.dedent(f""" -            **Reminder #{id_}:** *expires in {time}* (ID: {id_}) +            **Reminder #{id_}:** *expires in {time}* (ID: {id_}){mention_string}              {content}              """).strip() @@ -266,7 +348,8 @@ class Reminders(Scheduler, Cog):          # Remind the user that they have no reminders :^)          if not lines:              embed.description = "No active reminders could be found." -            return await ctx.send(embed=embed) +            await ctx.send(embed=embed) +            return          # Construct the embed and paginate it.          embed.colour = discord.Colour.blurple() @@ -286,37 +369,39 @@ class Reminders(Scheduler, Cog):      @edit_reminder_group.command(name="duration", aliases=("time",))      async def edit_reminder_duration(self, ctx: Context, id_: int, expiration: Duration) -> None:          """ -         Edit one of your reminder's expiration. +        Edit one of your reminder's expiration.          Expiration is parsed per: http://strftime.org/          """ -        # Send the request to update the reminder in the database -        reminder = await self.bot.api_client.patch( -            'bot/reminders/' + str(id_), -            json={'expiration': expiration.isoformat()} -        ) - -        # Send a confirmation message to the channel -        await self._send_confirmation( -            ctx, -            on_success="That reminder has been edited successfully!", -            reminder_id=id_, -            delivery_dt=expiration, -        ) - -        await self._reschedule_reminder(reminder) +        await self.edit_reminder(ctx, id_, {'expiration': expiration.isoformat()})      @edit_reminder_group.command(name="content", aliases=("reason",))      async def edit_reminder_content(self, ctx: Context, id_: int, *, content: str) -> None:          """Edit one of your reminder's content.""" -        # Send the request to update the reminder in the database -        reminder = await self.bot.api_client.patch( -            'bot/reminders/' + str(id_), -            json={'content': content} -        ) +        await self.edit_reminder(ctx, id_, {"content": content}) + +    @edit_reminder_group.command(name="mentions", aliases=("pings",)) +    async def edit_reminder_mentions(self, ctx: Context, id_: int, mentions: Greedy[Mentionable]) -> None: +        """Edit one of your reminder's mentions.""" +        # Remove duplicate mentions +        mentions = set(mentions) +        mentions.discard(ctx.author) + +        # Filter mentions to see if the user can mention members/roles +        if not await self.validate_mentions(ctx, mentions): +            return + +        mention_ids = [mention.id for mention in mentions] +        await self.edit_reminder(ctx, id_, {"mentions": mention_ids}) + +    async def edit_reminder(self, ctx: Context, id_: int, payload: dict) -> None: +        """Edits a reminder with the given payload, then sends a confirmation message.""" +        if not await self._can_modify(ctx, id_): +            return +        reminder = await self._edit_reminder(id_, payload) -        # Parse the reminder expiration back into a datetime for the confirmation message -        expiration = isoparse(reminder['expiration']).replace(tzinfo=None) +        # Parse the reminder expiration back into a datetime +        expiration = isoparse(reminder["expiration"]).replace(tzinfo=None)          # Send a confirmation message to the channel          await self._send_confirmation( @@ -330,6 +415,8 @@ class Reminders(Scheduler, Cog):      @remind_group.command("delete", aliases=("remove", "cancel"))      async def delete_reminder(self, ctx: Context, id_: int) -> None:          """Delete one of your active reminders.""" +        if not await self._can_modify(ctx, id_): +            return          await self._delete_reminder(id_)          await self._send_confirmation(              ctx, @@ -338,6 +425,24 @@ class Reminders(Scheduler, Cog):              delivery_dt=None,          ) +    async def _can_modify(self, ctx: Context, reminder_id: t.Union[str, int]) -> bool: +        """ +        Check whether the reminder can be modified by the ctx author. + +        The check passes when the user is an admin, or if they created the reminder. +        """ +        if with_role_check(ctx, Roles.admins): +            return True + +        api_response = await self.bot.api_client.get(f"bot/reminders/{reminder_id}") +        if not api_response["author"] == ctx.author.id: +            log.debug(f"{ctx.author} is not the reminder author and does not pass the check.") +            await send_denial(ctx, "You can't modify reminders of other users!") +            return False + +        log.debug(f"{ctx.author} is the reminder author and passes the check.") +        return True +  def setup(bot: Bot) -> None:      """Load the Reminders cog.""" diff --git a/bot/cogs/site.py b/bot/cogs/site.py index ac29daa1d..2d3a3d9f3 100644 --- a/bot/cogs/site.py +++ b/bot/cogs/site.py @@ -23,7 +23,7 @@ class Site(Cog):          """Commands for getting info about our website."""          await ctx.send_help(ctx.command) -    @site_group.command(name="home", aliases=("about",)) +    @site_group.command(name="home", aliases=("about",), root_aliases=("home",))      async def site_main(self, ctx: Context) -> None:          """Info about the website itself."""          url = f"{URLs.site_schema}{URLs.site}/" @@ -40,7 +40,7 @@ class Site(Cog):          await ctx.send(embed=embed) -    @site_group.command(name="resources") +    @site_group.command(name="resources", root_aliases=("resources", "resource"))      async def site_resources(self, ctx: Context) -> None:          """Info about the site's Resources page."""          learning_url = f"{PAGES_URL}/resources" @@ -56,7 +56,7 @@ class Site(Cog):          await ctx.send(embed=embed) -    @site_group.command(name="tools") +    @site_group.command(name="tools", root_aliases=("tools",))      async def site_tools(self, ctx: Context) -> None:          """Info about the site's Tools page."""          tools_url = f"{PAGES_URL}/resources/tools" @@ -87,7 +87,7 @@ class Site(Cog):          await ctx.send(embed=embed) -    @site_group.command(name="faq") +    @site_group.command(name="faq", root_aliases=("faq",))      async def site_faq(self, ctx: Context) -> None:          """Info about the site's FAQ page."""          url = f"{PAGES_URL}/frequently-asked-questions" @@ -104,7 +104,7 @@ class Site(Cog):          await ctx.send(embed=embed) -    @site_group.command(aliases=['r', 'rule'], name='rules') +    @site_group.command(name="rules", aliases=("r", "rule"), root_aliases=("rules", "rule"))      async def site_rules(self, ctx: Context, *rules: int) -> None:          """Provides a link to all rules or, if specified, displays specific rule(s)."""          rules_embed = Embed(title='Rules', color=Colour.blurple()) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 1d3565b62..03bf454ac 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -193,7 +193,7 @@ class Snekbox(Cog):                  output, paste_link = await self.format_output(results["stdout"])              icon = self.get_status_emoji(results) -            msg = f"{ctx.author.mention} {icon} {msg}.\n\n```py\n{output}\n```" +            msg = f"{ctx.author.mention} {icon} {msg}.\n\n```\n{output}\n```"              if paste_link:                  msg = f"{msg}\nFull output: {paste_link}" @@ -203,10 +203,15 @@ class Snekbox(Cog):              else:                  self.bot.stats.incr("snekbox.python.success") -            response = await ctx.send(msg) -            self.bot.loop.create_task( -                wait_for_deletion(response, user_ids=(ctx.author.id,), client=ctx.bot) -            ) +            filter_cog = self.bot.get_cog("Filtering") +            filter_triggered = False +            if filter_cog: +                filter_triggered = await filter_cog.filter_eval(msg, ctx.message) +            if filter_triggered: +                response = await ctx.send("Attempt to circumvent filter detected. Moderator team has been alerted.") +            else: +                response = await ctx.send(msg) +            self.bot.loop.create_task(wait_for_deletion(response, (ctx.author.id,), ctx.bot))              log.info(f"{ctx.author}'s job had a return code of {results['returncode']}")          return response diff --git a/bot/cogs/source.py b/bot/cogs/source.py new file mode 100644 index 000000000..205e0ba81 --- /dev/null +++ b/bot/cogs/source.py @@ -0,0 +1,141 @@ +import inspect +from pathlib import Path +from typing import Optional, Tuple, Union + +from discord import Embed +from discord.ext import commands + +from bot.bot import Bot +from bot.constants import URLs + +SourceType = Union[commands.HelpCommand, commands.Command, commands.Cog, str, commands.ExtensionNotLoaded] + + +class SourceConverter(commands.Converter): +    """Convert an argument into a help command, tag, command, or cog.""" + +    async def convert(self, ctx: commands.Context, argument: str) -> SourceType: +        """Convert argument into source object.""" +        if argument.lower().startswith("help"): +            return ctx.bot.help_command + +        cog = ctx.bot.get_cog(argument) +        if cog: +            return cog + +        cmd = ctx.bot.get_command(argument) +        if cmd: +            return cmd + +        tags_cog = ctx.bot.get_cog("Tags") +        show_tag = True + +        if not tags_cog: +            show_tag = False +        elif argument.lower() in tags_cog._cache: +            return argument.lower() + +        raise commands.BadArgument( +            f"Unable to convert `{argument}` to valid command{', tag,' if show_tag else ''} or Cog." +        ) + + +class BotSource(commands.Cog): +    """Displays information about the bot's source code.""" + +    def __init__(self, bot: Bot): +        self.bot = bot + +    @commands.command(name="source", aliases=("src",)) +    async def source_command(self, ctx: commands.Context, *, source_item: SourceConverter = None) -> None: +        """Display information and a GitHub link to the source code of a command, tag, or cog.""" +        if not source_item: +            embed = Embed(title="Bot's GitHub Repository") +            embed.add_field(name="Repository", value=f"[Go to GitHub]({URLs.github_bot_repo})") +            embed.set_thumbnail(url="https://avatars1.githubusercontent.com/u/9919") +            await ctx.send(embed=embed) +            return + +        embed = await self.build_embed(source_item) +        await ctx.send(embed=embed) + +    def get_source_link(self, source_item: SourceType) -> Tuple[str, str, Optional[int]]: +        """ +        Build GitHub link of source item, return this link, file location and first line number. + +        Raise BadArgument if `source_item` is a dynamically-created object (e.g. via internal eval). +        """ +        if isinstance(source_item, commands.Command): +            if source_item.cog_name == "Alias": +                cmd_name = source_item.callback.__name__.replace("_alias", "") +                cmd = self.bot.get_command(cmd_name.replace("_", " ")) +                src = cmd.callback.__code__ +                filename = src.co_filename +            else: +                src = source_item.callback.__code__ +                filename = src.co_filename +        elif isinstance(source_item, str): +            tags_cog = self.bot.get_cog("Tags") +            filename = tags_cog._cache[source_item]["location"] +        else: +            src = type(source_item) +            try: +                filename = inspect.getsourcefile(src) +            except TypeError: +                raise commands.BadArgument("Cannot get source for a dynamically-created object.") + +        if not isinstance(source_item, str): +            try: +                lines, first_line_no = inspect.getsourcelines(src) +            except OSError: +                raise commands.BadArgument("Cannot get source for a dynamically-created object.") + +            lines_extension = f"#L{first_line_no}-L{first_line_no+len(lines)-1}" +        else: +            first_line_no = None +            lines_extension = "" + +        # Handle tag file location differently than others to avoid errors in some cases +        if not first_line_no: +            file_location = Path(filename).relative_to("/bot/") +        else: +            file_location = Path(filename).relative_to(Path.cwd()).as_posix() + +        url = f"{URLs.github_bot_repo}/blob/master/{file_location}{lines_extension}" + +        return url, file_location, first_line_no or None + +    async def build_embed(self, source_object: SourceType) -> Optional[Embed]: +        """Build embed based on source object.""" +        url, location, first_line = self.get_source_link(source_object) + +        if isinstance(source_object, commands.HelpCommand): +            title = "Help Command" +            description = source_object.__doc__.splitlines()[1] +        elif isinstance(source_object, commands.Command): +            if source_object.cog_name == "Alias": +                cmd_name = source_object.callback.__name__.replace("_alias", "") +                cmd = self.bot.get_command(cmd_name.replace("_", " ")) +                description = cmd.short_doc +            else: +                description = source_object.short_doc + +            title = f"Command: {source_object.qualified_name}" +        elif isinstance(source_object, str): +            title = f"Tag: {source_object}" +            description = "" +        else: +            title = f"Cog: {source_object.qualified_name}" +            description = source_object.description.splitlines()[0] + +        embed = Embed(title=title, description=description) +        embed.add_field(name="Source Code", value=f"[Go to GitHub]({url})") +        line_text = f":{first_line}" if first_line else "" +        embed.set_footer(text=f"{location}{line_text}") + +        return embed + + +def setup(bot: Bot) -> None: +    """Load the BotSource cog.""" +    bot.add_cog(BotSource(bot)) diff --git a/bot/cogs/sync/cog.py b/bot/cogs/sync/cog.py index 7cc3726b2..5ace957e7 100644 --- a/bot/cogs/sync/cog.py +++ b/bot/cogs/sync/cog.py @@ -34,18 +34,22 @@ class Sync(Cog):          for syncer in (self.role_syncer, self.user_syncer):              await syncer.sync(guild) -    async def patch_user(self, user_id: int, updated_information: Dict[str, Any]) -> None: +    async def patch_user(self, user_id: int, json: Dict[str, Any], ignore_404: bool = False) -> None:          """Send a PATCH request to partially update a user in the database."""          try: -            await self.bot.api_client.patch(f"bot/users/{user_id}", json=updated_information) +            await self.bot.api_client.patch(f"bot/users/{user_id}", json=json)          except ResponseCodeError as e:              if e.response.status != 404:                  raise -            log.warning("Unable to update user, got 404. Assuming race condition from join event.") +            if not ignore_404: +                log.warning("Unable to update user, got 404. Assuming race condition from join event.")      @Cog.listener()      async def on_guild_role_create(self, role: Role) -> None:          """Adds newly create role to the database table over the API.""" +        if role.guild.id != constants.Guild.id: +            return +          await self.bot.api_client.post(              'bot/roles',              json={ @@ -60,11 +64,17 @@ class Sync(Cog):      @Cog.listener()      async def on_guild_role_delete(self, role: Role) -> None:          """Deletes role from the database when it's deleted from the guild.""" +        if role.guild.id != constants.Guild.id: +            return +          await self.bot.api_client.delete(f'bot/roles/{role.id}')      @Cog.listener()      async def on_guild_role_update(self, before: Role, after: Role) -> None:          """Syncs role with the database if any of the stored attributes were updated.""" +        if after.guild.id != constants.Guild.id: +            return +          was_updated = (              before.name != after.name              or before.colour != after.colour @@ -93,6 +103,9 @@ class Sync(Cog):          previously left), it will update the user's information. If the user is not yet known by          the database, the user is added.          """ +        if member.guild.id != constants.Guild.id: +            return +          packed = {              'discriminator': int(member.discriminator),              'id': member.id, @@ -122,14 +135,20 @@ class Sync(Cog):      @Cog.listener()      async def on_member_remove(self, member: Member) -> None:          """Set the in_guild field to False when a member leaves the guild.""" -        await self.patch_user(member.id, updated_information={"in_guild": False}) +        if member.guild.id != constants.Guild.id: +            return + +        await self.patch_user(member.id, json={"in_guild": False})      @Cog.listener()      async def on_member_update(self, before: Member, after: Member) -> None:          """Update the roles of the member in the database if a change is detected.""" +        if after.guild.id != constants.Guild.id: +            return +          if before.roles != after.roles:              updated_information = {"roles": sorted(role.id for role in after.roles)} -            await self.patch_user(after.id, updated_information=updated_information) +            await self.patch_user(after.id, json=updated_information)      @Cog.listener()      async def on_user_update(self, before: User, after: User) -> None: @@ -140,7 +159,8 @@ class Sync(Cog):                  "name": after.name,                  "discriminator": int(after.discriminator),              } -            await self.patch_user(after.id, updated_information=updated_information) +            # A 404 likely means the user is in another guild. +            await self.patch_user(after.id, json=updated_information, ignore_404=True)      @commands.group(name='sync')      @commands.has_permissions(administrator=True) diff --git a/bot/cogs/sync/syncers.py b/bot/cogs/sync/syncers.py index 536455668..f7ba811bc 100644 --- a/bot/cogs/sync/syncers.py +++ b/bot/cogs/sync/syncers.py @@ -5,6 +5,7 @@ import typing as t  from collections import namedtuple  from functools import partial +import discord  from discord import Guild, HTTPException, Member, Message, Reaction, User  from discord.ext.commands import Context @@ -68,7 +69,11 @@ class Syncer(abc.ABC):                      )                      return None -            message = await channel.send(f"{self._CORE_DEV_MENTION}{msg_content}") +            allowed_roles = [discord.Object(constants.Roles.core_developers)] +            message = await channel.send( +                f"{self._CORE_DEV_MENTION}{msg_content}", +                allowed_mentions=discord.AllowedMentions(everyone=False, roles=allowed_roles) +            )          else:              await message.edit(content=msg_content) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index 6f03a3475..d01647312 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -47,6 +47,7 @@ class Tags(Cog):                          "description": file.read_text(encoding="utf8"),                      },                      "restricted_to": "developers", +                    "location": f"/bot/{file}"                  }                  # Convert to a list to allow negative indexing. @@ -235,7 +236,7 @@ class Tags(Cog):                  await wait_for_deletion(                      await ctx.send(embed=Embed.from_dict(tag['embed'])),                      [ctx.author.id], -                    client=self.bot +                    self.bot                  )              elif founds and len(tag_name) >= 3:                  await wait_for_deletion( @@ -246,7 +247,7 @@ class Tags(Cog):                          )                      ),                      [ctx.author.id], -                    client=self.bot +                    self.bot                  )          else: diff --git a/bot/cogs/utils.py b/bot/cogs/utils.py index 697bf60ce..d96abbd5a 100644 --- a/bot/cogs/utils.py +++ b/bot/cogs/utils.py @@ -7,11 +7,13 @@ from io import StringIO  from typing import Tuple, Union  from discord import Colour, Embed, utils -from discord.ext.commands import BadArgument, Cog, Context, command +from discord.ext.commands import BadArgument, Cog, Context, clean_content, command  from bot.bot import Bot  from bot.constants import Channels, MODERATION_ROLES, STAFF_ROLES  from bot.decorators import in_whitelist, with_role +from bot.pagination import LinePaginator +from bot.utils import messages  log = logging.getLogger(__name__) @@ -117,25 +119,18 @@ class Utils(Cog):      @command()      @in_whitelist(channels=(Channels.bot_commands,), roles=STAFF_ROLES)      async def charinfo(self, ctx: Context, *, characters: str) -> None: -        """Shows you information on up to 25 unicode characters.""" +        """Shows you information on up to 50 unicode characters."""          match = re.match(r"<(a?):(\w+):(\d+)>", characters)          if match: -            embed = Embed( -                title="Non-Character Detected", -                description=( -                    "Only unicode characters can be processed, but a custom Discord emoji " -                    "was found. Please remove it and try again." -                ) +            return await messages.send_denial( +                ctx, +                "**Non-Character Detected**\n" +                "Only unicode characters can be processed, but a custom Discord emoji " +                "was found. Please remove it and try again."              ) -            embed.colour = Colour.red() -            await ctx.send(embed=embed) -            return -        if len(characters) > 25: -            embed = Embed(title=f"Too many characters ({len(characters)}/25)") -            embed.colour = Colour.red() -            await ctx.send(embed=embed) -            return +        if len(characters) > 50: +            return await messages.send_denial(ctx, f"Too many characters ({len(characters)}/50)")          def get_info(char: str) -> Tuple[str, str]:              digit = f"{ord(char):x}" @@ -148,15 +143,14 @@ class Utils(Cog):              info = f"`{u_code.ljust(10)}`: {name} - {utils.escape_markdown(char)}"              return info, u_code -        charlist, rawlist = zip(*(get_info(c) for c in characters)) - -        embed = Embed(description="\n".join(charlist)) -        embed.set_author(name="Character Info") +        char_list, raw_list = zip(*(get_info(c) for c in characters)) +        embed = Embed().set_author(name="Character Info")          if len(characters) > 1: -            embed.add_field(name='Raw', value=f"`{''.join(rawlist)}`", inline=False) +            # Maximum length possible is 502 out of 1024, so there's no need to truncate. +            embed.add_field(name='Full Raw Text', value=f"`{''.join(raw_list)}`", inline=False) -        await ctx.send(embed=embed) +        await LinePaginator.paginate(char_list, ctx, embed, max_lines=10, max_size=2000, empty=False)      @command()      async def zen(self, ctx: Context, *, search_value: Union[int, str, None] = None) -> None: @@ -231,13 +225,15 @@ class Utils(Cog):      @command(aliases=("poll",))      @with_role(*MODERATION_ROLES) -    async def vote(self, ctx: Context, title: str, *options: str) -> None: +    async def vote(self, ctx: Context, title: clean_content(fix_channel_mentions=True), *options: str) -> None:          """          Build a quick voting poll with matching reactions with the provided options.          A maximum of 20 options can be provided, as Discord supports a max of 20          reactions on a single message.          """ +        if len(title) > 256: +            raise BadArgument("The title cannot be longer than 256 characters.")          if len(options) < 2:              raise BadArgument("Please provide at least 2 options.")          if len(options) > 20: diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index ae156cf70..9ae92a228 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -1,19 +1,36 @@ +import asyncio  import logging +import typing as t  from contextlib import suppress +from datetime import datetime, timedelta -from discord import Colour, Forbidden, Message, NotFound, Object -from discord.ext.commands import Cog, Context, command +import discord +from discord.ext import tasks +from discord.ext.commands import Cog, Context, command, group +from discord.utils import snowflake_time  from bot import constants  from bot.bot import Bot  from bot.cogs.moderation import ModLog -from bot.decorators import in_whitelist, without_role +from bot.decorators import in_whitelist, with_role, without_role  from bot.utils.checks import InWhitelistCheckFailure, without_role_check +from bot.utils.redis_cache import RedisCache  log = logging.getLogger(__name__) -WELCOME_MESSAGE = f""" -Hello! Welcome to the server, and thanks for verifying yourself! +# Sent via DMs once user joins the guild +ON_JOIN_MESSAGE = f""" +Hello! Welcome to Python Discord! + +As a new user, you have read-only access to a few select channels to give you a taste of what our server is like. + +In order to see the rest of the channels and to send messages, you first have to accept our rules. To do so, \ +please visit <#{constants.Channels.verification}>. Thank you! +""" + +# Sent via DMs once user verifies +VERIFIED_MESSAGE = f""" +Thanks for verifying yourself!  For your records, these are the documents you accepted: @@ -32,29 +49,471 @@ If you'd like to unsubscribe from the announcement notifications, simply send `!  <#{constants.Channels.bot_commands}>.  """ -BOT_MESSAGE_DELETE_DELAY = 10 +# Sent via DMs to users kicked for failing to verify +KICKED_MESSAGE = f""" +Hi! You have been automatically kicked from Python Discord as you have failed to accept our rules \ +within `{constants.Verification.kicked_after}` days. If this was an accident, please feel free to join us again! + +{constants.Guild.invite} +""" + +# Sent periodically in the verification channel +REMINDER_MESSAGE = f""" +<@&{constants.Roles.unverified}> + +Welcome to Python Discord! Please read the documents mentioned above and type `!accept` to gain permissions \ +to send messages in the community! + +You will be kicked if you don't verify within `{constants.Verification.kicked_after}` days. +""".strip() + +# An async function taking a Member param +Request = t.Callable[[discord.Member], t.Awaitable] + + +class StopExecution(Exception): +    """Signals that a task should halt immediately & alert admins.""" + +    def __init__(self, reason: discord.HTTPException) -> None: +        super().__init__() +        self.reason = reason + + +class Limit(t.NamedTuple): +    """Composition over config for throttling requests.""" + +    batch_size: int  # Amount of requests after which to pause +    sleep_secs: int  # Sleep this many seconds after each batch + + +def mention_role(role_id: int) -> discord.AllowedMentions: +    """Construct an allowed mentions instance that allows pinging `role_id`.""" +    return discord.AllowedMentions(roles=[discord.Object(role_id)]) + + +def is_verified(member: discord.Member) -> bool: +    """ +    Check whether `member` is considered verified. + +    Members are considered verified if they have at least 1 role other than +    the default role (@everyone) and the @Unverified role. +    """ +    unverified_roles = { +        member.guild.get_role(constants.Roles.unverified), +        member.guild.default_role, +    } +    return len(set(member.roles) - unverified_roles) > 0  class Verification(Cog): -    """User verification and role self-management.""" +    """ +    User verification and role management. + +    There are two internal tasks in this cog: + +    * `update_unverified_members` +        * Unverified members are given the @Unverified role after configured `unverified_after` days +        * Unverified members are kicked after configured `kicked_after` days +    * `ping_unverified` +        * Periodically ping the @Unverified role in the verification channel + +    Statistics are collected in the 'verification.' namespace. -    def __init__(self, bot: Bot): +    Moderators+ can use the `verification` command group to start or stop both internal +    tasks, if necessary. Settings are persisted in Redis across sessions. + +    Additionally, this cog offers the !accept, !subscribe and !unsubscribe commands, +    and keeps the verification channel clean by deleting messages. +    """ + +    # Persist task settings & last sent `REMINDER_MESSAGE` id +    # RedisCache[ +    #   "tasks_running": int (0 or 1), +    #   "last_reminder": int (discord.Message.id), +    # ] +    task_cache = RedisCache() + +    def __init__(self, bot: Bot) -> None: +        """Start internal tasks."""          self.bot = bot +        self.bot.loop.create_task(self._maybe_start_tasks()) + +    def cog_unload(self) -> None: +        """ +        Cancel internal tasks. + +        This is necessary, as tasks are not automatically cancelled on cog unload. +        """ +        self._stop_tasks(gracefully=False)      @property      def mod_log(self) -> ModLog:          """Get currently loaded ModLog cog instance."""          return self.bot.get_cog("ModLog") +    async def _maybe_start_tasks(self) -> None: +        """ +        Poll Redis to check whether internal tasks should start. + +        Redis must be interfaced with from an async function. +        """ +        log.trace("Checking whether background tasks should begin") +        setting: t.Optional[int] = await self.task_cache.get("tasks_running")  # This can be None if never set + +        if setting: +            log.trace("Background tasks will be started") +            self.update_unverified_members.start() +            self.ping_unverified.start() + +    def _stop_tasks(self, *, gracefully: bool) -> None: +        """ +        Stop the update users & ping @Unverified tasks. + +        If `gracefully` is True, the tasks will be able to finish their current iteration. +        Otherwise, they are cancelled immediately. +        """ +        log.info(f"Stopping internal tasks ({gracefully=})") +        if gracefully: +            self.update_unverified_members.stop() +            self.ping_unverified.stop() +        else: +            self.update_unverified_members.cancel() +            self.ping_unverified.cancel() + +    # region: automatically update unverified users + +    async def _verify_kick(self, n_members: int) -> bool: +        """ +        Determine whether `n_members` is a reasonable amount of members to kick. + +        First, `n_members` is checked against the size of the PyDis guild. If `n_members` are +        more than the configured `kick_confirmation_threshold` of the guild, the operation +        must be confirmed by staff in #core-dev. Otherwise, the operation is seen as safe. +        """ +        log.debug(f"Checking whether {n_members} members are safe to kick") + +        await self.bot.wait_until_guild_available()  # Ensure cache is populated before we grab the guild +        pydis = self.bot.get_guild(constants.Guild.id) + +        percentage = n_members / len(pydis.members) +        if percentage < constants.Verification.kick_confirmation_threshold: +            log.debug(f"Kicking {percentage:.2%} of the guild's population is seen as safe") +            return True + +        # Since `n_members` is a suspiciously large number, we will ask for confirmation +        log.debug("Amount of users is too large, requesting staff confirmation") + +        core_dev_channel = pydis.get_channel(constants.Channels.dev_core) +        core_dev_ping = f"<@&{constants.Roles.core_developers}>" + +        confirmation_msg = await core_dev_channel.send( +            f"{core_dev_ping} Verification determined that `{n_members}` members should be kicked as they haven't " +            f"verified in `{constants.Verification.kicked_after}` days. This is `{percentage:.2%}` of the guild's " +            f"population. Proceed?", +            allowed_mentions=mention_role(constants.Roles.core_developers), +        ) + +        options = (constants.Emojis.incident_actioned, constants.Emojis.incident_unactioned) +        for option in options: +            await confirmation_msg.add_reaction(option) + +        core_dev_ids = [member.id for member in pydis.get_role(constants.Roles.core_developers).members] + +        def check(reaction: discord.Reaction, user: discord.User) -> bool: +            """Check whether `reaction` is a valid reaction to `confirmation_msg`.""" +            return ( +                reaction.message.id == confirmation_msg.id  # Reacted to `confirmation_msg` +                and str(reaction.emoji) in options  # With one of `options` +                and user.id in core_dev_ids  # By a core developer +            ) + +        timeout = 60 * 5  # Seconds, i.e. 5 minutes +        try: +            choice, _ = await self.bot.wait_for("reaction_add", check=check, timeout=timeout) +        except asyncio.TimeoutError: +            log.debug("Staff prompt not answered, aborting operation") +            return False +        finally: +            with suppress(discord.HTTPException): +                await confirmation_msg.clear_reactions() + +        result = str(choice) == constants.Emojis.incident_actioned +        log.debug(f"Received answer: {choice}, result: {result}") + +        # Edit the prompt message to reflect the final choice +        if result is True: +            result_msg = f":ok_hand: {core_dev_ping} Request to kick `{n_members}` members was authorized!" +        else: +            result_msg = f":warning: {core_dev_ping} Request to kick `{n_members}` members was denied!" + +        with suppress(discord.HTTPException): +            await confirmation_msg.edit(content=result_msg) + +        return result + +    async def _alert_admins(self, exception: discord.HTTPException) -> None: +        """ +        Ping @Admins with information about `exception`. + +        This is used when a critical `exception` caused a verification task to abort. +        """ +        await self.bot.wait_until_guild_available() +        log.info(f"Sending admin alert regarding exception: {exception}") + +        admins_channel = self.bot.get_guild(constants.Guild.id).get_channel(constants.Channels.admins) +        ping = f"<@&{constants.Roles.admins}>" + +        await admins_channel.send( +            f"{ping} Aborted updating unverified users due to the following exception:\n" +            f"```{exception}```\n" +            f"Internal tasks will be stopped.", +            allowed_mentions=mention_role(constants.Roles.admins), +        ) + +    async def _send_requests(self, members: t.Collection[discord.Member], request: Request, limit: Limit) -> int: +        """ +        Pass `members` one by one to `request` handling Discord exceptions. + +        This coroutine serves as a generic `request` executor for kicking members and adding +        roles, as it allows us to define the error handling logic in one place only. + +        Any `request` has the ability to completely abort the execution by raising `StopExecution`. +        In such a case, the @Admins will be alerted of the reason attribute. + +        To avoid rate-limits, pass a `limit` configuring the batch size and the amount of seconds +        to sleep between batches. + +        Returns the amount of successful requests. Failed requests are logged at info level. +        """ +        log.info(f"Sending {len(members)} requests") +        n_success, bad_statuses = 0, set() + +        for progress, member in enumerate(members, start=1): +            if is_verified(member):  # Member could have verified in the meantime +                continue +            try: +                await request(member) +            except StopExecution as stop_execution: +                await self._alert_admins(stop_execution.reason) +                await self.task_cache.set("tasks_running", 0) +                self._stop_tasks(gracefully=True)  # Gracefully finish current iteration, then stop +                break +            except discord.HTTPException as http_exc: +                bad_statuses.add(http_exc.status) +            else: +                n_success += 1 + +            if progress % limit.batch_size == 0: +                log.trace(f"Processed {progress} requests, pausing for {limit.sleep_secs} seconds") +                await asyncio.sleep(limit.sleep_secs) + +        if bad_statuses: +            log.info(f"Failed to send {len(members) - n_success} requests due to following statuses: {bad_statuses}") + +        return n_success + +    async def _kick_members(self, members: t.Collection[discord.Member]) -> int: +        """ +        Kick `members` from the PyDis guild. + +        Due to strict ratelimits on sending messages (120 requests / 60 secs), we sleep for a second +        after each 2 requests to allow breathing room for other features. + +        Note that this is a potentially destructive operation. Returns the amount of successful requests. +        """ +        log.info(f"Kicking {len(members)} members (not verified after {constants.Verification.kicked_after} days)") + +        async def kick_request(member: discord.Member) -> None: +            """Send `KICKED_MESSAGE` to `member` and kick them from the guild.""" +            try: +                await member.send(KICKED_MESSAGE) +            except discord.Forbidden as exc_403: +                log.trace(f"DM dispatch failed on 403 error with code: {exc_403.code}") +                if exc_403.code != 50_007:  # 403 raised for any other reason than disabled DMs +                    raise StopExecution(reason=exc_403) +            await member.kick(reason=f"User has not verified in {constants.Verification.kicked_after} days") + +        n_kicked = await self._send_requests(members, kick_request, Limit(batch_size=2, sleep_secs=1)) +        self.bot.stats.incr("verification.kicked", count=n_kicked) + +        return n_kicked + +    async def _give_role(self, members: t.Collection[discord.Member], role: discord.Role) -> int: +        """ +        Give `role` to all `members`. + +        We pause for a second after batches of 25 requests to ensure ratelimits aren't exceeded. + +        Returns the amount of successful requests. +        """ +        log.info( +            f"Assigning {role} role to {len(members)} members (not verified " +            f"after {constants.Verification.unverified_after} days)" +        ) + +        async def role_request(member: discord.Member) -> None: +            """Add `role` to `member`.""" +            await member.add_roles(role, reason=f"Not verified after {constants.Verification.unverified_after} days") + +        return await self._send_requests(members, role_request, Limit(batch_size=25, sleep_secs=1)) + +    async def _check_members(self) -> t.Tuple[t.Set[discord.Member], t.Set[discord.Member]]: +        """ +        Check in on the verification status of PyDis members. + +        This coroutine finds two sets of users: +        * Not verified after configured `unverified_after` days, should be given the @Unverified role +        * Not verified after configured `kicked_after` days, should be kicked from the guild + +        These sets are always disjoint, i.e. share no common members. +        """ +        await self.bot.wait_until_guild_available()  # Ensure cache is ready +        pydis = self.bot.get_guild(constants.Guild.id) + +        unverified = pydis.get_role(constants.Roles.unverified) +        current_dt = datetime.utcnow()  # Discord timestamps are UTC + +        # Users to be given the @Unverified role, and those to be kicked, these should be entirely disjoint +        for_role, for_kick = set(), set() + +        log.debug("Checking verification status of guild members") +        for member in pydis.members: + +            # Skip verified members, bots, and members for which we do not know their join date, +            # this should be extremely rare but docs mention that it can happen +            if is_verified(member) or member.bot or member.joined_at is None: +                continue + +            # At this point, we know that `member` is an unverified user, and we will decide what +            # to do with them based on time passed since their join date +            since_join = current_dt - member.joined_at + +            if since_join > timedelta(days=constants.Verification.kicked_after): +                for_kick.add(member)  # User should be removed from the guild + +            elif ( +                since_join > timedelta(days=constants.Verification.unverified_after) +                and unverified not in member.roles +            ): +                for_role.add(member)  # User should be given the @Unverified role + +        log.debug(f"Found {len(for_role)} users for {unverified} role, {len(for_kick)} users to be kicked") +        return for_role, for_kick + +    @tasks.loop(minutes=30) +    async def update_unverified_members(self) -> None: +        """ +        Periodically call `_check_members` and update unverified members accordingly. + +        After each run, a summary will be sent to the modlog channel. If a suspiciously high +        amount of members to be kicked is found, the operation is guarded by `_verify_kick`. +        """ +        log.info("Updating unverified guild members") + +        await self.bot.wait_until_guild_available() +        unverified = self.bot.get_guild(constants.Guild.id).get_role(constants.Roles.unverified) + +        for_role, for_kick = await self._check_members() + +        if not for_role: +            role_report = f"Found no users to be assigned the {unverified.mention} role." +        else: +            n_roles = await self._give_role(for_role, unverified) +            role_report = f"Assigned {unverified.mention} role to `{n_roles}`/`{len(for_role)}` members." + +        if not for_kick: +            kick_report = "Found no users to be kicked." +        elif not await self._verify_kick(len(for_kick)): +            kick_report = f"Not authorized to kick `{len(for_kick)}` members." +        else: +            n_kicks = await self._kick_members(for_kick) +            kick_report = f"Kicked `{n_kicks}`/`{len(for_kick)}` members from the guild." + +        await self.mod_log.send_log_message( +            icon_url=self.bot.user.avatar_url, +            colour=discord.Colour.blurple(), +            title="Verification system", +            text=f"{kick_report}\n{role_report}", +        ) + +    # endregion +    # region: periodically ping @Unverified + +    @tasks.loop(hours=constants.Verification.reminder_frequency) +    async def ping_unverified(self) -> None: +        """ +        Delete latest `REMINDER_MESSAGE` and send it again. + +        This utilizes RedisCache to persist the latest reminder message id. +        """ +        await self.bot.wait_until_guild_available() +        verification = self.bot.get_guild(constants.Guild.id).get_channel(constants.Channels.verification) + +        last_reminder: t.Optional[int] = await self.task_cache.get("last_reminder") + +        if last_reminder is not None: +            log.trace(f"Found verification reminder message in cache, deleting: {last_reminder}") + +            with suppress(discord.HTTPException):  # If something goes wrong, just ignore it +                await self.bot.http.delete_message(verification.id, last_reminder) + +        log.trace("Sending verification reminder") +        new_reminder = await verification.send( +            REMINDER_MESSAGE, allowed_mentions=mention_role(constants.Roles.unverified), +        ) + +        await self.task_cache.set("last_reminder", new_reminder.id) + +    @ping_unverified.before_loop +    async def _before_first_ping(self) -> None: +        """ +        Sleep until `REMINDER_MESSAGE` should be sent again. + +        If latest reminder is not cached, exit instantly. Otherwise, wait wait until the +        configured `reminder_frequency` has passed. +        """ +        last_reminder: t.Optional[int] = await self.task_cache.get("last_reminder") + +        if last_reminder is None: +            log.trace("Latest verification reminder message not cached, task will not wait") +            return + +        # Convert cached message id into a timestamp +        time_since = datetime.utcnow() - snowflake_time(last_reminder) +        log.trace(f"Time since latest verification reminder: {time_since}") + +        to_sleep = timedelta(hours=constants.Verification.reminder_frequency) - time_since +        log.trace(f"Time to sleep until next ping: {to_sleep}") + +        # Delta can be negative if `reminder_frequency` has already passed +        secs = max(to_sleep.total_seconds(), 0) +        await asyncio.sleep(secs) + +    # endregion +    # region: listeners +      @Cog.listener() -    async def on_message(self, message: Message) -> None: +    async def on_member_join(self, member: discord.Member) -> None: +        """Attempt to send initial direct message to each new member.""" +        if member.guild.id != constants.Guild.id: +            return  # Only listen for PyDis events + +        log.trace(f"Sending on join message to new member: {member.id}") +        with suppress(discord.Forbidden): +            await member.send(ON_JOIN_MESSAGE) + +    @Cog.listener() +    async def on_message(self, message: discord.Message) -> None:          """Check new message event for messages to the checkpoint channel & process."""          if message.channel.id != constants.Channels.verification:              return  # Only listen for #checkpoint messages +        if message.content == REMINDER_MESSAGE: +            return  # Ignore bots own verification reminder +          if message.author.bot:              # They're a bot, delete their message after the delay. -            await message.delete(delay=BOT_MESSAGE_DELETE_DELAY) +            await message.delete(delay=constants.Verification.bot_message_delete_delay)              return          # if a user mentions a role or guild member @@ -74,7 +533,7 @@ class Verification(Cog):              # Send pretty mod log embed to mod-alerts              await self.mod_log.send_log_message(                  icon_url=constants.Icons.filtering, -                colour=Colour(constants.Colours.soft_red), +                colour=discord.Colour(constants.Colours.soft_red),                  title=f"User/Role mentioned in {message.channel.name}",                  text=embed_text,                  thumbnail=message.author.avatar_url_as(static_format="png"), @@ -103,23 +562,117 @@ class Verification(Cog):          )          log.trace(f"Deleting the message posted by {ctx.author}") -        with suppress(NotFound): +        with suppress(discord.NotFound):              await ctx.message.delete() +    # endregion +    # region: task management commands + +    @with_role(*constants.MODERATION_ROLES) +    @group(name="verification") +    async def verification_group(self, ctx: Context) -> None: +        """Manage internal verification tasks.""" +        if ctx.invoked_subcommand is None: +            await ctx.send_help(ctx.command) + +    @verification_group.command(name="status") +    async def status_cmd(self, ctx: Context) -> None: +        """Check whether verification tasks are running.""" +        log.trace("Checking status of verification tasks") + +        if self.update_unverified_members.is_running(): +            update_status = f"{constants.Emojis.incident_actioned} Member update task is running." +        else: +            update_status = f"{constants.Emojis.incident_unactioned} Member update task is **not** running." + +        mention = f"<@&{constants.Roles.unverified}>" +        if self.ping_unverified.is_running(): +            ping_status = f"{constants.Emojis.incident_actioned} Ping {mention} task is running." +        else: +            ping_status = f"{constants.Emojis.incident_unactioned} Ping {mention} task is **not** running." + +        embed = discord.Embed( +            title="Verification system", +            description=f"{update_status}\n{ping_status}", +            colour=discord.Colour.blurple(), +        ) +        await ctx.send(embed=embed) + +    @verification_group.command(name="start") +    async def start_cmd(self, ctx: Context) -> None: +        """Start verification tasks if they are not already running.""" +        log.info("Starting verification tasks") + +        if not self.update_unverified_members.is_running(): +            self.update_unverified_members.start() + +        if not self.ping_unverified.is_running(): +            self.ping_unverified.start() + +        await self.task_cache.set("tasks_running", 1) + +        colour = discord.Colour.blurple() +        await ctx.send(embed=discord.Embed(title="Verification system", description="Done. :ok_hand:", colour=colour)) + +    @verification_group.command(name="stop", aliases=["kill"]) +    async def stop_cmd(self, ctx: Context) -> None: +        """Stop verification tasks.""" +        log.info("Stopping verification tasks") + +        self._stop_tasks(gracefully=False) +        await self.task_cache.set("tasks_running", 0) + +        colour = discord.Colour.blurple() +        await ctx.send(embed=discord.Embed(title="Verification system", description="Tasks canceled.", colour=colour)) + +    # endregion +    # region: accept and subscribe commands + +    def _bump_verified_stats(self, verified_member: discord.Member) -> None: +        """ +        Increment verification stats for `verified_member`. + +        Each member falls into one of the three categories: +            * Verified within 24 hours after joining +            * Does not have @Unverified role yet +            * Does have @Unverified role + +        Stats for member kicking are handled separately. +        """ +        if verified_member.joined_at is None:  # Docs mention this can happen +            return + +        if (datetime.utcnow() - verified_member.joined_at) < timedelta(hours=24): +            category = "accepted_on_day_one" +        elif constants.Roles.unverified not in [role.id for role in verified_member.roles]: +            category = "accepted_before_unverified" +        else: +            category = "accepted_after_unverified" + +        log.trace(f"Bumping verification stats in category: {category}") +        self.bot.stats.incr(f"verification.{category}") +      @command(name='accept', aliases=('verify', 'verified', 'accepted'), hidden=True)      @without_role(constants.Roles.verified)      @in_whitelist(channels=(constants.Channels.verification,))      async def accept_command(self, ctx: Context, *_) -> None:  # We don't actually care about the args          """Accept our rules and gain access to the rest of the server."""          log.debug(f"{ctx.author} called !accept. Assigning the 'Developer' role.") -        await ctx.author.add_roles(Object(constants.Roles.verified), reason="Accepted the rules") +        await ctx.author.add_roles(discord.Object(constants.Roles.verified), reason="Accepted the rules") + +        self._bump_verified_stats(ctx.author)  # This checks for @Unverified so make sure it's not yet removed + +        if constants.Roles.unverified in [role.id for role in ctx.author.roles]: +            log.debug(f"Removing Unverified role from: {ctx.author}") +            await ctx.author.remove_roles(discord.Object(constants.Roles.unverified)) +          try: -            await ctx.author.send(WELCOME_MESSAGE) -        except Forbidden: +            await ctx.author.send(VERIFIED_MESSAGE) +        except discord.Forbidden:              log.info(f"Sending welcome message failed for {ctx.author}.")          finally:              log.trace(f"Deleting accept message by {ctx.author}.") -            with suppress(NotFound): +            with suppress(discord.NotFound):                  self.mod_log.ignore(constants.Event.message_delete, ctx.message.id)                  await ctx.message.delete() @@ -139,7 +692,7 @@ class Verification(Cog):              return          log.debug(f"{ctx.author} called !subscribe. Assigning the 'Announcements' role.") -        await ctx.author.add_roles(Object(constants.Roles.announcements), reason="Subscribed to announcements") +        await ctx.author.add_roles(discord.Object(constants.Roles.announcements), reason="Subscribed to announcements")          log.trace(f"Deleting the message posted by {ctx.author}.") @@ -163,7 +716,9 @@ class Verification(Cog):              return          log.debug(f"{ctx.author} called !unsubscribe. Removing the 'Announcements' role.") -        await ctx.author.remove_roles(Object(constants.Roles.announcements), reason="Unsubscribed from announcements") +        await ctx.author.remove_roles( +            discord.Object(constants.Roles.announcements), reason="Unsubscribed from announcements" +        )          log.trace(f"Deleting the message posted by {ctx.author}.") @@ -171,6 +726,9 @@ class Verification(Cog):              f"{ctx.author.mention} Unsubscribed from <#{constants.Channels.announcements}> notifications."          ) +    # endregion +    # region: miscellaneous +      # This cannot be static (must have a __func__ attribute).      async def cog_command_error(self, ctx: Context, error: Exception) -> None:          """Check for & ignore any InWhitelistCheckFailure.""" @@ -185,6 +743,8 @@ class Verification(Cog):          else:              return True +    # endregion +  def setup(bot: Bot) -> None:      """Load the Verification cog.""" diff --git a/bot/cogs/watchchannels/bigbrother.py b/bot/cogs/watchchannels/bigbrother.py index 702d371f4..11ab8917a 100644 --- a/bot/cogs/watchchannels/bigbrother.py +++ b/bot/cogs/watchchannels/bigbrother.py @@ -35,16 +35,31 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"):      @bigbrother_group.command(name='watched', aliases=('all', 'list'))      @with_role(*MODERATION_ROLES) -    async def watched_command(self, ctx: Context, update_cache: bool = True) -> None: +    async def watched_command( +        self, ctx: Context, oldest_first: bool = False, update_cache: bool = True +    ) -> None:          """          Shows the users that are currently being monitored by Big Brother. +        The optional kwarg `oldest_first` can be used to order the list by oldest watched. + +        The optional kwarg `update_cache` can be used to update the user +        cache using the API before listing the users. +        """ +        await self.list_watched_users(ctx, oldest_first=oldest_first, update_cache=update_cache) + +    @bigbrother_group.command(name='oldest') +    @with_role(*MODERATION_ROLES) +    async def oldest_command(self, ctx: Context, update_cache: bool = True) -> None: +        """ +        Shows Big Brother monitored users ordered by oldest watched. +          The optional kwarg `update_cache` can be used to update the user          cache using the API before listing the users.          """ -        await self.list_watched_users(ctx, update_cache) +        await ctx.invoke(self.watched_command, oldest_first=True, update_cache=update_cache) -    @bigbrother_group.command(name='watch', aliases=('w',)) +    @bigbrother_group.command(name='watch', aliases=('w',), root_aliases=('watch',))      @with_role(*MODERATION_ROLES)      async def watch_command(self, ctx: Context, user: FetchedMember, *, reason: str) -> None:          """ @@ -55,7 +70,7 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"):          """          await self.apply_watch(ctx, user, reason) -    @bigbrother_group.command(name='unwatch', aliases=('uw',)) +    @bigbrother_group.command(name='unwatch', aliases=('uw',), root_aliases=('unwatch',))      @with_role(*MODERATION_ROLES)      async def unwatch_command(self, ctx: Context, user: FetchedMember, *, reason: str) -> None:          """Stop relaying messages by the given `user`.""" @@ -116,8 +131,8 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"):          active_watches = await self.bot.api_client.get(              self.api_endpoint,              params=ChainMap( +                {"user__id": str(user.id)},                  self.api_default_params, -                {"user__id": str(user.id)}              )          )          if active_watches: diff --git a/bot/cogs/watchchannels/talentpool.py b/bot/cogs/watchchannels/talentpool.py index 14547105f..76d6fe9bd 100644 --- a/bot/cogs/watchchannels/talentpool.py +++ b/bot/cogs/watchchannels/talentpool.py @@ -1,8 +1,9 @@  import logging  import textwrap  from collections import ChainMap +from typing import Union -from discord import Color, Embed, Member +from discord import Color, Embed, Member, User  from discord.ext.commands import Cog, Context, group  from bot.api import ResponseCodeError @@ -36,18 +37,33 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"):          """Highlights the activity of helper nominees by relaying their messages to the talent pool channel."""          await ctx.send_help(ctx.command) -    @nomination_group.command(name='watched', aliases=('all', 'list')) +    @nomination_group.command(name='watched', aliases=('all', 'list'), root_aliases=("nominees",))      @with_role(*MODERATION_ROLES) -    async def watched_command(self, ctx: Context, update_cache: bool = True) -> None: +    async def watched_command( +        self, ctx: Context, oldest_first: bool = False, update_cache: bool = True +    ) -> None:          """          Shows the users that are currently being monitored in the talent pool. +        The optional kwarg `oldest_first` can be used to order the list by oldest nomination. + +        The optional kwarg `update_cache` can be used to update the user +        cache using the API before listing the users. +        """ +        await self.list_watched_users(ctx, oldest_first=oldest_first, update_cache=update_cache) + +    @nomination_group.command(name='oldest') +    @with_role(*MODERATION_ROLES) +    async def oldest_command(self, ctx: Context, update_cache: bool = True) -> None: +        """ +        Shows talent pool monitored users ordered by oldest nomination. +          The optional kwarg `update_cache` can be used to update the user          cache using the API before listing the users.          """ -        await self.list_watched_users(ctx, update_cache) +        await ctx.invoke(self.watched_command, oldest_first=True, update_cache=update_cache) -    @nomination_group.command(name='watch', aliases=('w', 'add', 'a')) +    @nomination_group.command(name='watch', aliases=('w', 'add', 'a'), root_aliases=("nominate",))      @with_role(*STAFF_ROLES)      async def watch_command(self, ctx: Context, user: FetchedMember, *, reason: str) -> None:          """ @@ -141,7 +157,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"):              max_size=1000          ) -    @nomination_group.command(name='unwatch', aliases=('end', )) +    @nomination_group.command(name='unwatch', aliases=('end', ), root_aliases=("unnominate",))      @with_role(*MODERATION_ROLES)      async def unwatch_command(self, ctx: Context, user: FetchedMember, *, reason: str) -> None:          """ @@ -149,25 +165,10 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"):          Providing a `reason` is required.          """ -        active_nomination = await self.bot.api_client.get( -            self.api_endpoint, -            params=ChainMap( -                self.api_default_params, -                {"user__id": str(user.id)} -            ) -        ) - -        if not active_nomination: +        if await self.unwatch(user.id, reason): +            await ctx.send(f":white_check_mark: Messages sent by {user} will no longer be relayed") +        else:              await ctx.send(":x: The specified user does not have an active nomination") -            return - -        [nomination] = active_nomination -        await self.bot.api_client.patch( -            f"{self.api_endpoint}/{nomination['id']}", -            json={'end_reason': reason, 'active': False} -        ) -        await ctx.send(f":white_check_mark: Messages sent by {user} will no longer be relayed") -        self._remove_user(user.id)      @nomination_group.group(name='edit', aliases=('e',), invoke_without_command=True)      @with_role(*MODERATION_ROLES) @@ -205,6 +206,36 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"):          await ctx.send(f":white_check_mark: Updated the {field} of the nomination!") +    @Cog.listener() +    async def on_member_ban(self, guild: Guild, user: Union[User, Member]) -> None: +        """Remove `user` from the talent pool after they are banned.""" +        await self.unwatch(user.id, "User was banned.") + +    async def unwatch(self, user_id: int, reason: str) -> bool: +        """End the active nomination of a user with the given reason and return True on success.""" +        active_nomination = await self.bot.api_client.get( +            self.api_endpoint, +            params=ChainMap( +                {"user__id": str(user_id)}, +                self.api_default_params, +            ) +        ) + +        if not active_nomination: +            log.debug(f"No active nominate exists for {user_id=}") +            return False + +        log.info(f"Ending nomination: {user_id=} {reason=}") + +        nomination = active_nomination[0] +        await self.bot.api_client.patch( +            f"{self.api_endpoint}/{nomination['id']}", +            json={'end_reason': reason, 'active': False} +        ) +        self._remove_user(user_id) + +        return True +      def _nomination_to_string(self, nomination_object: dict) -> str:          """Creates a string representation of a nomination."""          guild = self.bot.get_guild(Guild.id) @@ -224,7 +255,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"):                  Status: **Active**                  Date: {start_date}                  Actor: {actor.mention if actor else actor_id} -                Reason: {textwrap.shorten(nomination_object["reason"], width=200, placeholder="...")} +                Reason: {nomination_object["reason"]}                  Nomination ID: `{nomination_object["id"]}`                  ===============                  """ @@ -237,10 +268,10 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"):                  Status: Inactive                  Date: {start_date}                  Actor: {actor.mention if actor else actor_id} -                Reason: {textwrap.shorten(nomination_object["reason"], width=200, placeholder="...")} +                Reason: {nomination_object["reason"]}                  End date: {end_date} -                Unwatch reason: {textwrap.shorten(nomination_object["end_reason"], width=200, placeholder="...")} +                Unwatch reason: {nomination_object["end_reason"]}                  Nomination ID: `{nomination_object["id"]}`                  ===============                  """ diff --git a/bot/cogs/watchchannels/watchchannel.py b/bot/cogs/watchchannels/watchchannel.py index 7c58a0fb5..a58b604c0 100644 --- a/bot/cogs/watchchannels/watchchannel.py +++ b/bot/cogs/watchchannels/watchchannel.py @@ -15,6 +15,8 @@ from discord.ext.commands import Cog, Context  from bot.api import ResponseCodeError  from bot.bot import Bot  from bot.cogs.moderation import ModLog +from bot.cogs.token_remover import TokenRemover +from bot.cogs.webhook_remover import WEBHOOK_URL_RE  from bot.constants import BigBrother as BigBrotherConfig, Guild as GuildConfig, Icons  from bot.pagination import LinePaginator  from bot.utils import CogABCMeta, messages @@ -226,14 +228,16 @@ class WatchChannel(metaclass=CogABCMeta):              await self.send_header(msg) -        cleaned_content = msg.clean_content - -        if cleaned_content: +        if TokenRemover.find_token_in_message(msg) or WEBHOOK_URL_RE.search(msg.content): +            cleaned_content = "Content is censored because it contains a bot or webhook token." +        elif cleaned_content := msg.clean_content:              # Put all non-media URLs in a code block to prevent embeds              media_urls = {embed.url for embed in msg.embeds if embed.type in ("image", "video")}              for url in URL_RE.findall(cleaned_content):                  if url not in media_urls:                      cleaned_content = cleaned_content.replace(url, f"`{url}`") + +        if cleaned_content:              await self.webhook_send(                  cleaned_content,                  username=msg.author.display_name, @@ -287,10 +291,14 @@ class WatchChannel(metaclass=CogABCMeta):          await self.webhook_send(embed=embed, username=msg.author.display_name, avatar_url=msg.author.avatar_url) -    async def list_watched_users(self, ctx: Context, update_cache: bool = True) -> None: +    async def list_watched_users( +        self, ctx: Context, oldest_first: bool = False, update_cache: bool = True +    ) -> None:          """          Gives an overview of the watched user list for this channel. +        The optional kwarg `oldest_first` orders the list by oldest entry. +          The optional kwarg `update_cache` specifies whether the cache should          be refreshed by polling the API.          """ @@ -305,7 +313,11 @@ class WatchChannel(metaclass=CogABCMeta):              time_delta = self._get_time_delta(inserted_at)              lines.append(f"• <@{user_id}> (added {time_delta})") +        if oldest_first: +            lines.reverse() +          lines = lines or ("There's nothing here yet.",) +          embed = Embed(              title=f"{self.__class__.__name__} watched users ({'updated' if update_cache else 'cached'})",              color=Color.blue() diff --git a/bot/cogs/webhook_remover.py b/bot/cogs/webhook_remover.py index 543869215..5812da87c 100644 --- a/bot/cogs/webhook_remover.py +++ b/bot/cogs/webhook_remover.py @@ -8,7 +8,7 @@ from bot.bot import Bot  from bot.cogs.moderation.modlog import ModLog  from bot.constants import Channels, Colours, Event, Icons -WEBHOOK_URL_RE = re.compile(r"((?:https?://)?discordapp\.com/api/webhooks/\d+/)\S+/?", re.I) +WEBHOOK_URL_RE = re.compile(r"((?:https?://)?discord(?:app)?\.com/api/webhooks/\d+/)\S+/?", re.IGNORECASE)  ALERT_MESSAGE_TEMPLATE = (      "{user}, looks like you posted a Discord webhook URL. Therefore, your " diff --git a/bot/cogs/wolfram.py b/bot/cogs/wolfram.py deleted file mode 100644 index e6cae3bb8..000000000 --- a/bot/cogs/wolfram.py +++ /dev/null @@ -1,280 +0,0 @@ -import logging -from io import BytesIO -from typing import Callable, List, Optional, Tuple -from urllib import parse - -import discord -from dateutil.relativedelta import relativedelta -from discord import Embed -from discord.ext import commands -from discord.ext.commands import BucketType, Cog, Context, check, group - -from bot.bot import Bot -from bot.constants import Colours, STAFF_ROLES, Wolfram -from bot.pagination import ImagePaginator -from bot.utils.time import humanize_delta - -log = logging.getLogger(__name__) - -APPID = Wolfram.key -DEFAULT_OUTPUT_FORMAT = "JSON" -QUERY = "http://api.wolframalpha.com/v2/{request}?{data}" -WOLF_IMAGE = "https://www.symbols.com/gi.php?type=1&id=2886&i=1" - -MAX_PODS = 20 - -# Allows for 10 wolfram calls pr user pr day -usercd = commands.CooldownMapping.from_cooldown(Wolfram.user_limit_day, 60*60*24, BucketType.user) - -# Allows for max api requests / days in month per day for the entire guild (Temporary) -guildcd = commands.CooldownMapping.from_cooldown(Wolfram.guild_limit_day, 60*60*24, BucketType.guild) - - -async def send_embed( -        ctx: Context, -        message_txt: str, -        colour: int = Colours.soft_red, -        footer: str = None, -        img_url: str = None, -        f: discord.File = None -) -> None: -    """Generate & send a response embed with Wolfram as the author.""" -    embed = Embed(colour=colour) -    embed.description = message_txt -    embed.set_author(name="Wolfram Alpha", -                     icon_url=WOLF_IMAGE, -                     url="https://www.wolframalpha.com/") -    if footer: -        embed.set_footer(text=footer) - -    if img_url: -        embed.set_image(url=img_url) - -    await ctx.send(embed=embed, file=f) - - -def custom_cooldown(*ignore: List[int]) -> Callable: -    """ -    Implement per-user and per-guild cooldowns for requests to the Wolfram API. - -    A list of roles may be provided to ignore the per-user cooldown -    """ -    async def predicate(ctx: Context) -> bool: -        if ctx.invoked_with == 'help': -            # if the invoked command is help we don't want to increase the ratelimits since it's not actually -            # invoking the command/making a request, so instead just check if the user/guild are on cooldown. -            guild_cooldown = not guildcd.get_bucket(ctx.message).get_tokens() == 0  # if guild is on cooldown -            if not any(r.id in ignore for r in ctx.author.roles):  # check user bucket if user is not ignored -                return guild_cooldown and not usercd.get_bucket(ctx.message).get_tokens() == 0 -            return guild_cooldown - -        user_bucket = usercd.get_bucket(ctx.message) - -        if all(role.id not in ignore for role in ctx.author.roles): -            user_rate = user_bucket.update_rate_limit() - -            if user_rate: -                # Can't use api; cause: member limit -                delta = relativedelta(seconds=int(user_rate)) -                cooldown = humanize_delta(delta) -                message = ( -                    "You've used up your limit for Wolfram|Alpha requests.\n" -                    f"Cooldown: {cooldown}" -                ) -                await send_embed(ctx, message) -                return False - -        guild_bucket = guildcd.get_bucket(ctx.message) -        guild_rate = guild_bucket.update_rate_limit() - -        # Repr has a token attribute to read requests left -        log.debug(guild_bucket) - -        if guild_rate: -            # Can't use api; cause: guild limit -            message = ( -                "The max limit of requests for the server has been reached for today.\n" -                f"Cooldown: {int(guild_rate)}" -            ) -            await send_embed(ctx, message) -            return False - -        return True -    return check(predicate) - - -async def get_pod_pages(ctx: Context, bot: Bot, query: str) -> Optional[List[Tuple]]: -    """Get the Wolfram API pod pages for the provided query.""" -    async with ctx.channel.typing(): -        url_str = parse.urlencode({ -            "input": query, -            "appid": APPID, -            "output": DEFAULT_OUTPUT_FORMAT, -            "format": "image,plaintext" -        }) -        request_url = QUERY.format(request="query", data=url_str) - -        async with bot.http_session.get(request_url) as response: -            json = await response.json(content_type='text/plain') - -        result = json["queryresult"] - -        if result["error"]: -            # API key not set up correctly -            if result["error"]["msg"] == "Invalid appid": -                message = "Wolfram API key is invalid or missing." -                log.warning( -                    "API key seems to be missing, or invalid when " -                    f"processing a wolfram request: {url_str}, Response: {json}" -                ) -                await send_embed(ctx, message) -                return - -            message = "Something went wrong internally with your request, please notify staff!" -            log.warning(f"Something went wrong getting a response from wolfram: {url_str}, Response: {json}") -            await send_embed(ctx, message) -            return - -        if not result["success"]: -            message = f"I couldn't find anything for {query}." -            await send_embed(ctx, message) -            return - -        if not result["numpods"]: -            message = "Could not find any results." -            await send_embed(ctx, message) -            return - -        pods = result["pods"] -        pages = [] -        for pod in pods[:MAX_PODS]: -            subs = pod.get("subpods") - -            for sub in subs: -                title = sub.get("title") or sub.get("plaintext") or sub.get("id", "") -                img = sub["img"]["src"] -                pages.append((title, img)) -        return pages - - -class Wolfram(Cog): -    """Commands for interacting with the Wolfram|Alpha API.""" - -    def __init__(self, bot: Bot): -        self.bot = bot - -    @group(name="wolfram", aliases=("wolf", "wa"), invoke_without_command=True) -    @custom_cooldown(*STAFF_ROLES) -    async def wolfram_command(self, ctx: Context, *, query: str) -> None: -        """Requests all answers on a single image, sends an image of all related pods.""" -        url_str = parse.urlencode({ -            "i": query, -            "appid": APPID, -        }) -        query = QUERY.format(request="simple", data=url_str) - -        # Give feedback that the bot is working. -        async with ctx.channel.typing(): -            async with self.bot.http_session.get(query) as response: -                status = response.status -                image_bytes = await response.read() - -            f = discord.File(BytesIO(image_bytes), filename="image.png") -            image_url = "attachment://image.png" - -            if status == 501: -                message = "Failed to get response" -                footer = "" -                color = Colours.soft_red -            elif status == 400: -                message = "No input found" -                footer = "" -                color = Colours.soft_red -            elif status == 403: -                message = "Wolfram API key is invalid or missing." -                footer = "" -                color = Colours.soft_red -            else: -                message = "" -                footer = "View original for a bigger picture." -                color = Colours.soft_orange - -            # Sends a "blank" embed if no request is received, unsure how to fix -            await send_embed(ctx, message, color, footer=footer, img_url=image_url, f=f) - -    @wolfram_command.command(name="page", aliases=("pa", "p")) -    @custom_cooldown(*STAFF_ROLES) -    async def wolfram_page_command(self, ctx: Context, *, query: str) -> None: -        """ -        Requests a drawn image of given query. - -        Keywords worth noting are, "like curve", "curve", "graph", "pokemon", etc. -        """ -        pages = await get_pod_pages(ctx, self.bot, query) - -        if not pages: -            return - -        embed = Embed() -        embed.set_author(name="Wolfram Alpha", -                         icon_url=WOLF_IMAGE, -                         url="https://www.wolframalpha.com/") -        embed.colour = Colours.soft_orange - -        await ImagePaginator.paginate(pages, ctx, embed) - -    @wolfram_command.command(name="cut", aliases=("c",)) -    @custom_cooldown(*STAFF_ROLES) -    async def wolfram_cut_command(self, ctx: Context, *, query: str) -> None: -        """ -        Requests a drawn image of given query. - -        Keywords worth noting are, "like curve", "curve", "graph", "pokemon", etc. -        """ -        pages = await get_pod_pages(ctx, self.bot, query) - -        if not pages: -            return - -        if len(pages) >= 2: -            page = pages[1] -        else: -            page = pages[0] - -        await send_embed(ctx, page[0], colour=Colours.soft_orange, img_url=page[1]) - -    @wolfram_command.command(name="short", aliases=("sh", "s")) -    @custom_cooldown(*STAFF_ROLES) -    async def wolfram_short_command(self, ctx: Context, *, query: str) -> None: -        """Requests an answer to a simple question.""" -        url_str = parse.urlencode({ -            "i": query, -            "appid": APPID, -        }) -        query = QUERY.format(request="result", data=url_str) - -        # Give feedback that the bot is working. -        async with ctx.channel.typing(): -            async with self.bot.http_session.get(query) as response: -                status = response.status -                response_text = await response.text() - -            if status == 501: -                message = "Failed to get response" -                color = Colours.soft_red -            elif status == 400: -                message = "No input found" -                color = Colours.soft_red -            elif response_text == "Error 1: Invalid appid": -                message = "Wolfram API key is invalid or missing." -                color = Colours.soft_red -            else: -                message = response_text -                color = Colours.soft_orange - -            await send_embed(ctx, message, color) - - -def setup(bot: Bot) -> None: -    """Load the Wolfram cog.""" -    bot.add_cog(Wolfram(bot)) diff --git a/bot/command.py b/bot/command.py new file mode 100644 index 000000000..0fb900f7b --- /dev/null +++ b/bot/command.py @@ -0,0 +1,18 @@ +from discord.ext import commands + + +class Command(commands.Command): +    """ +    A `discord.ext.commands.Command` subclass which supports root aliases. + +    A `root_aliases` keyword argument is added, which is a sequence of alias names that will act as +    top-level commands rather than being aliases of the command's group. It's stored as an attribute +    also named `root_aliases`. +    """ + +    def __init__(self, *args, **kwargs): +        super().__init__(*args, **kwargs) +        self.root_aliases = kwargs.get("root_aliases", []) + +        if not isinstance(self.root_aliases, (list, tuple)): +            raise TypeError("Root aliases of a command must be a list or a tuple of strings.") diff --git a/bot/constants.py b/bot/constants.py index a1b392c82..17f14fec0 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -227,10 +227,6 @@ class Filter(metaclass=YAMLGetter):      ping_everyone: bool      offensive_msg_delete_days: int -    guild_invite_whitelist: List[int] -    domain_blacklist: List[str] -    word_watchlist: List[str] -    token_watchlist: List[str]      channel_whitelist: List[int]      role_whitelist: List[int] @@ -272,6 +268,21 @@ class Emojis(metaclass=YAMLGetter):      status_idle: str      status_dnd: str +    badge_staff: str +    badge_partner: str +    badge_hypesquad: str +    badge_bug_hunter: str +    badge_hypesquad_bravery: str +    badge_hypesquad_brilliance: str +    badge_hypesquad_balance: str +    badge_early_supporter: str +    badge_bug_hunter_level_2: str +    badge_verified_bot_developer: str + +    incident_actioned: str +    incident_unactioned: str +    incident_investigating: str +      failmail: str      trashcan: str @@ -395,10 +406,12 @@ class Channels(metaclass=YAMLGetter):      dev_contrib: int      dev_core: int      dev_log: int +    dm_log: int      esoteric: int      helpers: int      how_to_get_help: int      incidents: int +    incidents_archive: int      message_log: int      meta: int      mod_alerts: int @@ -422,11 +435,13 @@ class Webhooks(metaclass=YAMLGetter):      section = "guild"      subsection = "webhooks" -    talent_pool: int      big_brother: int -    reddit: int -    duck_pond: int      dev_log: int +    dm_log: int +    duck_pond: int +    incidents_archive: int +    reddit: int +    talent_pool: int  class Roles(metaclass=YAMLGetter): @@ -446,6 +461,7 @@ class Roles(metaclass=YAMLGetter):      partners: int      python_community: int      team_leaders: int +    unverified: int      verified: int  # This is the Developers role on PyDis, here named verified for readability reasons. @@ -453,6 +469,7 @@ class Guild(metaclass=YAMLGetter):      section = "guild"      id: int +    invite: str  # Discord invite, gets embedded in chat      moderation_channels: List[int]      moderation_roles: List[int]      modlog_blacklist: List[int] @@ -460,6 +477,7 @@ class Guild(metaclass=YAMLGetter):      staff_channels: List[int]      staff_roles: List[int] +  class Keys(metaclass=YAMLGetter):      section = "keys" @@ -480,25 +498,13 @@ class URLs(metaclass=YAMLGetter):      bot_avatar: str      github_bot_repo: str -    # Site endpoints +    # Base site vars      site: str      site_api: str -    site_superstarify_api: str -    site_logs_api: str -    site_logs_view: str -    site_reminders_api: str -    site_reminders_user_api: str      site_schema: str -    site_settings_api: str -    site_tags_api: str -    site_user_api: str -    site_user_complete_api: str -    site_infractions: str -    site_infractions_user: str -    site_infractions_type: str -    site_infractions_by_id: str -    site_infractions_user_type_current: str -    site_infractions_user_type: str + +    # Site endpoints +    site_logs_view: str      paste_service: str @@ -510,14 +516,6 @@ class Reddit(metaclass=YAMLGetter):      secret: Optional[str] -class Wolfram(metaclass=YAMLGetter): -    section = "wolfram" - -    user_limit_day: int -    guild_limit_day: int -    key: Optional[str] - -  class AntiSpam(metaclass=YAMLGetter):      section = 'anti_spam' @@ -528,12 +526,6 @@ class AntiSpam(metaclass=YAMLGetter):      rules: Dict[str, Dict[str, int]] -class AntiMalware(metaclass=YAMLGetter): -    section = "anti_malware" - -    whitelist: list - -  class BigBrother(metaclass=YAMLGetter):      section = 'big_brother' @@ -588,6 +580,16 @@ class PythonNews(metaclass=YAMLGetter):      webhook: int +class Verification(metaclass=YAMLGetter): +    section = "verification" + +    unverified_after: int +    kicked_after: int +    reminder_frequency: int +    bot_message_delete_delay: int +    kick_confirmation_threshold: float + +  class Event(Enum):      """      Event names. This does not include every event (for example, raw diff --git a/bot/converters.py b/bot/converters.py index 4deb59f87..1358cbf1e 100644 --- a/bot/converters.py +++ b/bot/converters.py @@ -9,8 +9,11 @@ import dateutil.tz  import discord  from aiohttp import ClientConnectorError  from dateutil.relativedelta import relativedelta -from discord.ext.commands import BadArgument, Context, Converter, UserConverter +from discord.ext.commands import BadArgument, Bot, Context, Converter, IDConverter, UserConverter +from bot.api import ResponseCodeError +from bot.constants import URLs +from bot.utils.regex import INVITE_RE  log = logging.getLogger(__name__) @@ -34,6 +37,90 @@ def allowed_strings(*values, preserve_case: bool = False) -> t.Callable[[str], s      return converter +class ValidDiscordServerInvite(Converter): +    """ +    A converter that validates whether a given string is a valid Discord server invite. + +    Raises 'BadArgument' if: +    - The string is not a valid Discord server invite. +    - The string is valid, but is an invite for a group DM. +    - The string is valid, but is expired. + +    Returns a (partial) guild object if: +    - The string is a valid vanity +    - The string is a full invite URI +    - The string contains the invite code (the stuff after discord.gg/) + +    See the Discord API docs for documentation on the guild object: +    https://discord.com/developers/docs/resources/guild#guild-object +    """ + +    async def convert(self, ctx: Context, server_invite: str) -> dict: +        """Check whether the string is a valid Discord server invite.""" +        invite_code = INVITE_RE.search(server_invite) +        if invite_code: +            response = await ctx.bot.http_session.get( +                f"{URLs.discord_invite_api}/{invite_code[1]}" +            ) +            if response.status != 404: +                invite_data = await response.json() +                return invite_data.get("guild") + +        id_converter = IDConverter() +        if id_converter._get_id_match(server_invite): +            raise BadArgument("Guild IDs are not supported, only invites.") + +        raise BadArgument("This does not appear to be a valid Discord server invite.") + + +class ValidFilterListType(Converter): +    """ +    A converter that checks whether the given string is a valid FilterList type. + +    Raises `BadArgument` if the argument is not a valid FilterList type, and simply +    passes through the given argument otherwise. +    """ + +    @staticmethod +    async def get_valid_types(bot: Bot) -> list: +        """ +        Try to get a list of valid filter list types. + +        Raise a BadArgument if the API can't respond. +        """ +        try: +            valid_types = await bot.api_client.get('bot/filter-lists/get-types') +        except ResponseCodeError: +            raise BadArgument("Cannot validate list_type: Unable to fetch valid types from API.") + +        return [enum for enum, classname in valid_types] + +    async def convert(self, ctx: Context, list_type: str) -> str: +        """Checks whether the given string is a valid FilterList type.""" +        valid_types = await self.get_valid_types(ctx.bot) +        list_type = list_type.upper() + +        if list_type not in valid_types: + +            # Maybe the user is using the plural form of this type, +            # e.g. "guild_invites" instead of "guild_invite". +            # +            # This code will support the simple plural form (a single 's' at the end), +            # which works for all current list types, but if a list type is added in the future +            # which has an irregular plural form (like 'ies'), this code will need to be +            # refactored to support this. +            if list_type.endswith("S") and list_type[:-1] in valid_types: +                list_type = list_type[:-1] + +            else: +                valid_types_list = '\n'.join([f"• {type_.lower()}" for type_ in valid_types]) +                raise BadArgument( +                    f"You have provided an invalid list type!\n\n" +                    f"Please provide one of the following: \n{valid_types_list}" +                ) +        return list_type + +  class ValidPythonIdentifier(Converter):      """      A converter that checks whether the given string is a valid Python identifier. @@ -181,8 +268,8 @@ class TagContentConverter(Converter):          return tag_content -class Duration(Converter): -    """Convert duration strings into UTC datetime.datetime objects.""" +class DurationDelta(Converter): +    """Convert duration strings into dateutil.relativedelta.relativedelta objects."""      duration_parser = re.compile(          r"((?P<years>\d+?) ?(years|year|Y|y) ?)?" @@ -194,9 +281,9 @@ class Duration(Converter):          r"((?P<seconds>\d+?) ?(seconds|second|S|s))?"      ) -    async def convert(self, ctx: Context, duration: str) -> datetime: +    async def convert(self, ctx: Context, duration: str) -> relativedelta:          """ -        Converts a `duration` string to a datetime object that's `duration` in the future. +        Converts a `duration` string to a relativedelta object.          The converter supports the following symbols for each unit of time:          - years: `Y`, `y`, `year`, `years` @@ -215,6 +302,20 @@ class Duration(Converter):          duration_dict = {unit: int(amount) for unit, amount in match.groupdict(default=0).items()}          delta = relativedelta(**duration_dict) + +        return delta + + +class Duration(DurationDelta): +    """Convert duration strings into UTC datetime.datetime objects.""" + +    async def convert(self, ctx: Context, duration: str) -> datetime: +        """ +        Converts a `duration` string to a datetime object that's `duration` in the future. + +        The converter supports the same symbols for each unit of time as its parent class. +        """ +        delta = await super().convert(ctx, duration)          now = datetime.utcnow()          try: @@ -223,6 +324,32 @@ class Duration(Converter):              raise BadArgument(f"`{duration}` results in a datetime outside the supported range.") +class OffTopicName(Converter): +    """A converter that ensures an added off-topic name is valid.""" + +    async def convert(self, ctx: Context, argument: str) -> str: +        """Attempt to replace any invalid characters with their approximate Unicode equivalent.""" +        allowed_characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ!?'`-" + +        # Chain multiple words to a single one +        argument = "-".join(argument.split()) + +        if not (2 <= len(argument) <= 96): +            raise BadArgument("Channel name must be between 2 and 96 chars long") + +        elif not all(c.isalnum() or c in allowed_characters for c in argument): +            raise BadArgument( +                "Channel name must only consist of " +                "alphanumeric characters, minus signs or apostrophes." +            ) + +        # Replace invalid characters with unicode alternatives. +        table = str.maketrans( +            allowed_characters, '𝖠𝖡𝖢𝖣𝖤𝖥𝖦𝖧𝖨𝖩𝖪𝖫𝖬𝖭𝖮𝖯𝖰𝖱𝖲𝖳𝖴𝖵𝖶𝖷𝖸𝖹ǃ?’’-' +        ) +        return argument.translate(table) + +  class ISODateTime(Converter):      """Converts an ISO-8601 datetime string into a datetime.datetime.""" @@ -316,6 +443,25 @@ def proxy_user(user_id: str) -> discord.Object:      return user +class UserMentionOrID(UserConverter): +    """ +    Converts to a `discord.User`, but only if a mention or userID is provided. + +    Unlike the default `UserConverter`, it does allow conversion from name, or name#descrim. + +    This is useful in cases where that lookup strategy would lead to ambiguity. +    """ + +    async def convert(self, ctx: Context, argument: str) -> discord.User: +        """Convert the `arg` to a `discord.User`.""" +        match = self._get_id_match(argument) or re.match(r'<@!?([0-9]+)>$', argument) + +        if match is not None: +            return await super().convert(ctx, argument) +        else: +            raise BadArgument(f"`{argument}` is not a User mention or a User ID.") + +  class FetchedUser(UserConverter):      """      Converts to a `discord.User` or, if it fails, a `discord.Object`. diff --git a/bot/pagination.py b/bot/pagination.py index 2aa3590ba..182b2fa76 100644 --- a/bot/pagination.py +++ b/bot/pagination.py @@ -37,12 +37,19 @@ class LinePaginator(Paginator):          The suffix appended at the end of every page. e.g. three backticks.      * max_size: `int`          The maximum amount of codepoints allowed in a page. +    * scale_to_size: `int` +        The maximum amount of characters a single line can scale up to.      * max_lines: `int`          The maximum amount of lines allowed in a page.      """      def __init__( -        self, prefix: str = '```', suffix: str = '```', max_size: int = 2000, max_lines: int = None +        self, +        prefix: str = '```', +        suffix: str = '```', +        max_size: int = 2000, +        scale_to_size: int = 2000, +        max_lines: t.Optional[int] = None      ) -> None:          """          This function overrides the Paginator.__init__ from inside discord.ext.commands. @@ -51,7 +58,21 @@ class LinePaginator(Paginator):          """          self.prefix = prefix          self.suffix = suffix + +        # Embeds that exceed 2048 characters will result in an HTTPException +        # (Discord API limit), so we've set a limit of 2000 +        if max_size > 2000: +            raise ValueError(f"max_size must be <= 2,000 characters. ({max_size} > 2000)") +          self.max_size = max_size - len(suffix) + +        if scale_to_size < max_size: +            raise ValueError(f"scale_to_size must be >= max_size. ({scale_to_size} < {max_size})") + +        if scale_to_size > 2000: +            raise ValueError(f"scale_to_size must be <= 2,000 characters. ({scale_to_size} > 2000)") + +        self.scale_to_size = scale_to_size - len(suffix)          self.max_lines = max_lines          self._current_page = [prefix]          self._linecount = 0 @@ -62,23 +83,38 @@ class LinePaginator(Paginator):          """          Adds a line to the current page. -        If the line exceeds the `self.max_size` then an exception is raised. +        If a line on a page exceeds `max_size` characters, then `max_size` will go up to +        `scale_to_size` for a single line before creating a new page for the overflow words. If it +        is still exceeded, the excess characters are stored and placed on the next pages unti +        there are none remaining (by word boundary). The line is truncated if `scale_to_size` is +        still exceeded after attempting to continue onto the next page. + +        In the case that the page already contains one or more lines and the new lines would cause +        `max_size` to be exceeded, a new page is created. This is done in order to make a best +        effort to avoid breaking up single lines across pages, while keeping the total length of the +        page at a reasonable size.          This function overrides the `Paginator.add_line` from inside `discord.ext.commands`.          It overrides in order to allow us to configure the maximum number of lines per page.          """ -        if len(line) > self.max_size - len(self.prefix) - 2: -            raise RuntimeError('Line exceeds maximum page size %s' % (self.max_size - len(self.prefix) - 2)) - -        if self.max_lines is not None: -            if self._linecount >= self.max_lines: -                self._linecount = 0 -                self.close_page() - -            self._linecount += 1 -        if self._count + len(line) + 1 > self.max_size: -            self.close_page() +        remaining_words = None +        if len(line) > (max_chars := self.max_size - len(self.prefix) - 2): +            if len(line) > self.scale_to_size: +                line, remaining_words = self._split_remaining_words(line, max_chars) +                if len(line) > self.scale_to_size: +                    log.debug("Could not continue to next page, truncating line.") +                    line = line[:self.scale_to_size] + +        # Check if we should start a new page or continue the line on the current one +        if self.max_lines is not None and self._linecount >= self.max_lines: +            log.debug("max_lines exceeded, creating new page.") +            self._new_page() +        elif self._count + len(line) + 1 > self.max_size and self._linecount > 0: +            log.debug("max_size exceeded on page with lines, creating new page.") +            self._new_page() + +        self._linecount += 1          self._count += len(line) + 1          self._current_page.append(line) @@ -87,6 +123,65 @@ class LinePaginator(Paginator):              self._current_page.append('')              self._count += 1 +        # Start a new page if there were any overflow words +        if remaining_words: +            self._new_page() +            self.add_line(remaining_words) + +    def _new_page(self) -> None: +        """ +        Internal: start a new page for the paginator. + +        This closes the current page and resets the counters for the new page's line count and +        character count. +        """ +        self._linecount = 0 +        self._count = len(self.prefix) + 1 +        self.close_page() + +    def _split_remaining_words(self, line: str, max_chars: int) -> t.Tuple[str, t.Optional[str]]: +        """ +        Internal: split a line into two strings -- reduced_words and remaining_words. + +        reduced_words: the remaining words in `line`, after attempting to remove all words that +            exceed `max_chars` (rounding down to the nearest word boundary). + +        remaining_words: the words in `line` which exceed `max_chars`. This value is None if +            no words could be split from `line`. + +        If there are any remaining_words, an ellipses is appended to reduced_words and a +        continuation header is inserted before remaining_words to visually communicate the line +        continuation. + +        Return a tuple in the format (reduced_words, remaining_words). +        """ +        reduced_words = [] +        remaining_words = [] + +        # "(Continued)" is used on a line by itself to indicate the continuation of last page +        continuation_header = "(Continued)\n-----------\n" +        reduced_char_count = 0 +        is_full = False + +        for word in line.split(" "): +            if not is_full: +                if len(word) + reduced_char_count <= max_chars: +                    reduced_words.append(word) +                    reduced_char_count += len(word) + 1 +                else: +                    # If reduced_words is empty, we were unable to split the words across pages +                    if not reduced_words: +                        return line, None +                    is_full = True +                    remaining_words.append(word) +            else: +                remaining_words.append(word) + +        return ( +            " ".join(reduced_words) + "..." if remaining_words else "", +            continuation_header + " ".join(remaining_words) if remaining_words else None +        ) +      @classmethod      async def paginate(          cls, @@ -97,6 +192,7 @@ class LinePaginator(Paginator):          suffix: str = "",          max_lines: t.Optional[int] = None,          max_size: int = 500, +        scale_to_size: int = 2000,          empty: bool = True,          restrict_to_user: User = None,          timeout: int = 300, @@ -142,7 +238,8 @@ class LinePaginator(Paginator):                  ))              ) -        paginator = cls(prefix=prefix, suffix=suffix, max_size=max_size, max_lines=max_lines) +        paginator = cls(prefix=prefix, suffix=suffix, max_size=max_size, max_lines=max_lines, +                        scale_to_size=scale_to_size)          current_page = 0          if not lines: @@ -216,8 +313,6 @@ class LinePaginator(Paginator):                  log.debug(f"Got first page reaction - changing to page 1/{len(paginator.pages)}") -                embed.description = "" -                await message.edit(embed=embed)                  embed.description = paginator.pages[current_page]                  if footer_text:                      embed.set_footer(text=f"{footer_text} (Page {current_page + 1}/{len(paginator.pages)})") @@ -231,8 +326,6 @@ class LinePaginator(Paginator):                  log.debug(f"Got last page reaction - changing to page {current_page + 1}/{len(paginator.pages)}") -                embed.description = "" -                await message.edit(embed=embed)                  embed.description = paginator.pages[current_page]                  if footer_text:                      embed.set_footer(text=f"{footer_text} (Page {current_page + 1}/{len(paginator.pages)})") @@ -250,8 +343,6 @@ class LinePaginator(Paginator):                  current_page -= 1                  log.debug(f"Got previous page reaction - changing to page {current_page + 1}/{len(paginator.pages)}") -                embed.description = "" -                await message.edit(embed=embed)                  embed.description = paginator.pages[current_page]                  if footer_text: @@ -271,8 +362,6 @@ class LinePaginator(Paginator):                  current_page += 1                  log.debug(f"Got next page reaction - changing to page {current_page + 1}/{len(paginator.pages)}") -                embed.description = "" -                await message.edit(embed=embed)                  embed.description = paginator.pages[current_page]                  if footer_text: @@ -285,169 +374,3 @@ class LinePaginator(Paginator):          log.debug("Ending pagination and clearing reactions.")          with suppress(discord.NotFound):              await message.clear_reactions() - - -class ImagePaginator(Paginator): -    """ -    Helper class that paginates images for embeds in messages. - -    Close resemblance to LinePaginator, except focuses on images over text. - -    Refer to ImagePaginator.paginate for documentation on how to use. -    """ - -    def __init__(self, prefix: str = "", suffix: str = ""): -        super().__init__(prefix, suffix) -        self._current_page = [prefix] -        self.images = [] -        self._pages = [] -        self._count = 0 - -    def add_line(self, line: str = '', *, empty: bool = False) -> None: -        """Adds a line to each page.""" -        if line: -            self._count = len(line) -        else: -            self._count = 0 -        self._current_page.append(line) -        self.close_page() - -    def add_image(self, image: str = None) -> None: -        """Adds an image to a page.""" -        self.images.append(image) - -    @classmethod -    async def paginate( -        cls, -        pages: t.List[t.Tuple[str, str]], -        ctx: Context, embed: discord.Embed, -        prefix: str = "", -        suffix: str = "", -        timeout: int = 300, -        exception_on_empty_embed: bool = False -    ) -> t.Optional[discord.Message]: -        """ -        Use a paginator and set of reactions to provide pagination over a set of title/image pairs. - -        The reactions are used to switch page, or to finish with pagination. - -        When used, this will send a message using `ctx.send()` and apply a set of reactions to it. These reactions may -        be used to change page, or to remove pagination from the message. - -        Note: Pagination will be removed automatically if no reaction is added for five minutes (300 seconds). - -        Example: -        >>> embed = discord.Embed() -        >>> embed.set_author(name="Some Operation", url=url, icon_url=icon) -        >>> await ImagePaginator.paginate(pages, ctx, embed) -        """ -        def check_event(reaction_: discord.Reaction, member: discord.Member) -> bool: -            """Checks each reaction added, if it matches our conditions pass the wait_for.""" -            return all(( -                # Reaction is on the same message sent -                reaction_.message.id == message.id, -                # The reaction is part of the navigation menu -                str(reaction_.emoji) in PAGINATION_EMOJI, -                # The reactor is not a bot -                not member.bot -            )) - -        paginator = cls(prefix=prefix, suffix=suffix) -        current_page = 0 - -        if not pages: -            if exception_on_empty_embed: -                log.exception("Pagination asked for empty image list") -                raise EmptyPaginatorEmbed("No images to paginate") - -            log.debug("No images to add to paginator, adding '(no images to display)' message") -            pages.append(("(no images to display)", "")) - -        for text, image_url in pages: -            paginator.add_line(text) -            paginator.add_image(image_url) - -        embed.description = paginator.pages[current_page] -        image = paginator.images[current_page] - -        if image: -            embed.set_image(url=image) - -        if len(paginator.pages) <= 1: -            return await ctx.send(embed=embed) - -        embed.set_footer(text=f"Page {current_page + 1}/{len(paginator.pages)}") -        message = await ctx.send(embed=embed) - -        for emoji in PAGINATION_EMOJI: -            await message.add_reaction(emoji) - -        while True: -            # Start waiting for reactions -            try: -                reaction, user = await ctx.bot.wait_for("reaction_add", timeout=timeout, check=check_event) -            except asyncio.TimeoutError: -                log.debug("Timed out waiting for a reaction") -                break  # We're done, no reactions for the last 5 minutes - -            # Deletes the users reaction -            await message.remove_reaction(reaction.emoji, user) - -            # Delete reaction press - [:trashcan:] -            if str(reaction.emoji) == DELETE_EMOJI: -                log.debug("Got delete reaction") -                return await message.delete() - -            # First reaction press - [:track_previous:] -            if reaction.emoji == FIRST_EMOJI: -                if current_page == 0: -                    log.debug("Got first page reaction, but we're on the first page - ignoring") -                    continue - -                current_page = 0 -                reaction_type = "first" - -            # Last reaction press - [:track_next:] -            if reaction.emoji == LAST_EMOJI: -                if current_page >= len(paginator.pages) - 1: -                    log.debug("Got last page reaction, but we're on the last page - ignoring") -                    continue - -                current_page = len(paginator.pages) - 1 -                reaction_type = "last" - -            # Previous reaction press - [:arrow_left: ] -            if reaction.emoji == LEFT_EMOJI: -                if current_page <= 0: -                    log.debug("Got previous page reaction, but we're on the first page - ignoring") -                    continue - -                current_page -= 1 -                reaction_type = "previous" - -            # Next reaction press - [:arrow_right:] -            if reaction.emoji == RIGHT_EMOJI: -                if current_page >= len(paginator.pages) - 1: -                    log.debug("Got next page reaction, but we're on the last page - ignoring") -                    continue - -                current_page += 1 -                reaction_type = "next" - -            # Magic happens here, after page and reaction_type is set -            embed.description = "" -            await message.edit(embed=embed) -            embed.description = paginator.pages[current_page] - -            image = paginator.images[current_page] -            if image: -                embed.set_image(url=image) - -            embed.set_footer(text=f"Page {current_page + 1}/{len(paginator.pages)}") -            log.debug(f"Got {reaction_type} page reaction - changing to page {current_page + 1}/{len(paginator.pages)}") - -            await message.edit(embed=embed) - -        log.debug("Ending pagination and clearing reactions.") -        with suppress(discord.NotFound): -            await message.clear_reactions() diff --git a/bot/resources/tags/ask.md b/bot/resources/tags/ask.md deleted file mode 100644 index e2c2a88f6..000000000 --- a/bot/resources/tags/ask.md +++ /dev/null @@ -1,9 +0,0 @@ -Asking good questions will yield a much higher chance of a quick response: - -• Don't ask to ask your question, just go ahead and tell us your problem.   -• Don't ask if anyone is knowledgeable in some area, filtering serves no purpose.   -• Try to solve the problem on your own first, we're not going to write code for you.   -• Show us the code you've tried and any errors or unexpected results it's giving.   -• Be patient while we're helping you. - -You can find a much more detailed explanation [on our website](https://pythondiscord.com/pages/asking-good-questions/). diff --git a/bot/resources/tags/kindling-projects.md b/bot/resources/tags/kindling-projects.md new file mode 100644 index 000000000..54ed8c961 --- /dev/null +++ b/bot/resources/tags/kindling-projects.md @@ -0,0 +1,3 @@ +**Kindling Projects** + +The [Kindling projects page](https://nedbatchelder.com/text/kindling.html) on Ned Batchelder's website contains a list of projects and ideas programmers can tackle to build their skills and knowledge. diff --git a/bot/resources/tags/or-gotcha.md b/bot/resources/tags/or-gotcha.md index 00c2db1f8..d75a73d78 100644 --- a/bot/resources/tags/or-gotcha.md +++ b/bot/resources/tags/or-gotcha.md @@ -3,7 +3,7 @@ When checking if something is equal to one thing or another, you might think tha  if favorite_fruit == 'grapefruit' or 'lemon':      print("That's a weird favorite fruit to have.")  ``` -After all, that's how you would normally phrase it in plain English. In Python, however, you have to have _complete instructions on both sides of the logical operator_. +While this makes sense in English, it may not behave the way you would expect. In Python, you should have _[complete instructions on both sides of the logical operator](https://docs.python.org/3/reference/expressions.html#boolean-operations)_.  So, if you want to check if something is equal to one thing or another, there are two common ways:  ```py diff --git a/bot/resources/tags/range-len.md b/bot/resources/tags/range-len.md new file mode 100644 index 000000000..65665eccf --- /dev/null +++ b/bot/resources/tags/range-len.md @@ -0,0 +1,11 @@ +Iterating over `range(len(...))` is a common approach to accessing each item in an ordered collection. +```py +for i in range(len(my_list)): +    do_something(my_list[i]) +``` +The pythonic syntax is much simpler, and is guaranteed to produce elements in the same order: +```py +for item in my_list: +    do_something(item) +``` +Python has other solutions for cases when the index itself might be needed. To get the element at the same index from two or more lists, use [zip](https://docs.python.org/3/library/functions.html#zip). To get both the index and the element at that index, use [enumerate](https://docs.python.org/3/library/functions.html#enumerate). diff --git a/bot/resources/tags/traceback.md b/bot/resources/tags/traceback.md index 46ef40aa1..e770fa86d 100644 --- a/bot/resources/tags/traceback.md +++ b/bot/resources/tags/traceback.md @@ -11,7 +11,7 @@ ZeroDivisionError: integer division or modulo by zero  ```  The best way to read your traceback is bottom to top. -• Identify the exception raised (e.g. ZeroDivisonError)   +• Identify the exception raised (e.g. ZeroDivisionError)    • Make note of the line number, and navigate there in your program.    • Try to understand why the error occurred.   diff --git a/bot/rules/__init__.py b/bot/rules/__init__.py index a01ceae73..8a69cadee 100644 --- a/bot/rules/__init__.py +++ b/bot/rules/__init__.py @@ -10,3 +10,4 @@ from .links import apply as apply_links  from .mentions import apply as apply_mentions  from .newlines import apply as apply_newlines  from .role_mentions import apply as apply_role_mentions +from .everyone_ping import apply as apply_everyone_ping diff --git a/bot/rules/burst_shared.py b/bot/rules/burst_shared.py index bbe9271b3..0e66df69c 100644 --- a/bot/rules/burst_shared.py +++ b/bot/rules/burst_shared.py @@ -2,11 +2,20 @@ from typing import Dict, Iterable, List, Optional, Tuple  from discord import Member, Message +from bot.constants import Channels +  async def apply(      last_message: Message, recent_messages: List[Message], config: Dict[str, int]  ) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: -    """Detects repeated messages sent by multiple users.""" +    """ +    Detects repeated messages sent by multiple users. + +    This filter never triggers in the verification channel. +    """ +    if last_message.channel.id == Channels.verification: +        return +      total_recent = len(recent_messages)      if total_recent > config['max']: diff --git a/bot/rules/discord_emojis.py b/bot/rules/discord_emojis.py index 5bab514f2..6e47f0197 100644 --- a/bot/rules/discord_emojis.py +++ b/bot/rules/discord_emojis.py @@ -5,6 +5,7 @@ from discord import Member, Message  DISCORD_EMOJI_RE = re.compile(r"<:\w+:\d+>") +CODE_BLOCK_RE = re.compile(r"```.*?```", flags=re.DOTALL)  async def apply( @@ -17,8 +18,9 @@ async def apply(          if msg.author == last_message.author      ) +    # Get rid of code blocks in the message before searching for emojis.      total_emojis = sum( -        len(DISCORD_EMOJI_RE.findall(msg.content)) +        len(DISCORD_EMOJI_RE.findall(CODE_BLOCK_RE.sub("", msg.content)))          for msg in relevant_messages      ) diff --git a/bot/rules/everyone_ping.py b/bot/rules/everyone_ping.py new file mode 100644 index 000000000..89d9fe570 --- /dev/null +++ b/bot/rules/everyone_ping.py @@ -0,0 +1,41 @@ +import random +import re +from typing import Dict, Iterable, List, Optional, Tuple + +from discord import Embed, Member, Message + +from bot.constants import Colours, Guild, NEGATIVE_REPLIES + +# Generate regex for checking for pings: +guild_id = Guild.id +EVERYONE_RE_INLINE_CODE = re.compile(rf"^(?!`).*@everyone.*(?!`)$|^(?!`).*<@&{guild_id}>.*(?!`)$") +EVERYONE_RE_MULTILINE_CODE = re.compile(rf"^(?!```).*@everyone.*(?!```)$|^(?!```).*<@&{guild_id}>.*(?!```)$") + + +async def apply( +    last_message: Message, +    recent_messages: List[Message], +    config: Dict[str, int], +) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: +    """Detects if a user has sent an '@everyone' ping.""" +    relevant_messages = tuple(msg for msg in recent_messages if msg.author == last_message.author) + +    everyone_messages_count = 0 +    for msg in relevant_messages: +        num_everyone_pings_inline = len(re.findall(EVERYONE_RE_INLINE_CODE, msg.content)) +        num_everyone_pings_multiline = len(re.findall(EVERYONE_RE_MULTILINE_CODE, msg.content)) +        if num_everyone_pings_inline and num_everyone_pings_multiline: +            everyone_messages_count += 1 + +    if everyone_messages_count > config["max"]: +        # Send the channel an embed giving the user more info: +        embed_text = f"Please don't try to ping {last_message.guild.member_count:,} people." +        embed = Embed(title=random.choice(NEGATIVE_REPLIES), description=embed_text, colour=Colours.soft_red) +        await last_message.channel.send(embed=embed) + +        return ( +            "pinged the everyone role", +            (last_message.author,), +            relevant_messages, +        ) +    return None diff --git a/bot/utils/messages.py b/bot/utils/messages.py index a40a12e98..aa8f17f75 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -1,15 +1,17 @@  import asyncio  import contextlib  import logging +import random  import re  from io import BytesIO  from typing import List, Optional, Sequence, Union -from discord import Client, Embed, File, Member, Message, Reaction, TextChannel, Webhook +from discord import Client, Colour, Embed, File, Member, Message, Reaction, TextChannel, Webhook  from discord.abc import Snowflake  from discord.errors import HTTPException +from discord.ext.commands import Context -from bot.constants import Emojis +from bot.constants import Emojis, NEGATIVE_REPLIES  log = logging.getLogger(__name__) @@ -17,25 +19,20 @@ log = logging.getLogger(__name__)  async def wait_for_deletion(      message: Message,      user_ids: Sequence[Snowflake], +    client: Client,      deletion_emojis: Sequence[str] = (Emojis.trashcan,),      timeout: float = 60 * 5,      attach_emojis: bool = True, -    client: Optional[Client] = None  ) -> None:      """      Wait for up to `timeout` seconds for a reaction by any of the specified `user_ids` to delete the message.      An `attach_emojis` bool may be specified to determine whether to attach the given -    `deletion_emojis` to the message in the given `context` - -    A `client` instance may be optionally specified, otherwise client will be taken from the -    guild of the message. +    `deletion_emojis` to the message in the given `context`.      """ -    if message.guild is None and client is None: +    if message.guild is None:          raise ValueError("Message must be sent on a guild") -    bot = client or message.guild.me -      if attach_emojis:          for emoji in deletion_emojis:              await message.add_reaction(emoji) @@ -49,7 +46,7 @@ async def wait_for_deletion(          )      with contextlib.suppress(asyncio.TimeoutError): -        await bot.wait_for('reaction_add', check=check, timeout=timeout) +        await client.wait_for('reaction_add', check=check, timeout=timeout)          await message.delete() @@ -132,3 +129,13 @@ def sub_clyde(username: Optional[str]) -> Optional[str]:          return re.sub(r"(clyd)(e)", replace_e, username, flags=re.I)      else:          return username  # Empty string or None + + +async def send_denial(ctx: Context, reason: str) -> None: +    """Send an embed denying the user with the given reason.""" +    embed = Embed() +    embed.colour = Colour.red() +    embed.title = random.choice(NEGATIVE_REPLIES) +    embed.description = reason + +    await ctx.send(embed=embed) diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 58cfe1df5..52b689b49 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -226,7 +226,6 @@ class RedisCache:          for attribute in vars(instance).values():              if isinstance(attribute, Bot):                  self.bot = attribute -                self._redis = self.bot.redis_session                  return self          else:              error_message = ( @@ -251,7 +250,7 @@ class RedisCache:          value = self._value_to_typestring(value)          log.trace(f"Setting {key} to {value}.") -        await self._redis.hset(self._namespace, key, value) +        await self.bot.redis_session.hset(self._namespace, key, value)      async def get(self, key: RedisKeyType, default: Optional[RedisValueType] = None) -> Optional[RedisValueType]:          """Get an item from the Redis cache.""" @@ -259,7 +258,7 @@ class RedisCache:          key = self._key_to_typestring(key)          log.trace(f"Attempting to retrieve {key}.") -        value = await self._redis.hget(self._namespace, key) +        value = await self.bot.redis_session.hget(self._namespace, key)          if value is None:              log.trace(f"Value not found, returning default value {default}") @@ -281,7 +280,7 @@ class RedisCache:          key = self._key_to_typestring(key)          log.trace(f"Attempting to delete {key}.") -        return await self._redis.hdel(self._namespace, key) +        return await self.bot.redis_session.hdel(self._namespace, key)      async def contains(self, key: RedisKeyType) -> bool:          """ @@ -291,7 +290,7 @@ class RedisCache:          """          await self._validate_cache()          key = self._key_to_typestring(key) -        exists = await self._redis.hexists(self._namespace, key) +        exists = await self.bot.redis_session.hexists(self._namespace, key)          log.trace(f"Testing if {key} exists in the RedisCache - Result is {exists}")          return exists @@ -314,7 +313,7 @@ class RedisCache:          """          await self._validate_cache()          items = self._dict_from_typestring( -            await self._redis.hgetall(self._namespace) +            await self.bot.redis_session.hgetall(self._namespace)          ).items()          log.trace(f"Retrieving all key/value pairs from cache, total of {len(items)} items.") @@ -323,7 +322,7 @@ class RedisCache:      async def length(self) -> int:          """Return the number of items in the Redis cache."""          await self._validate_cache() -        number_of_items = await self._redis.hlen(self._namespace) +        number_of_items = await self.bot.redis_session.hlen(self._namespace)          log.trace(f"Returning length. Result is {number_of_items}.")          return number_of_items @@ -335,7 +334,7 @@ class RedisCache:          """Deletes the entire hash from the Redis cache."""          await self._validate_cache()          log.trace("Clearing the cache of all key/value pairs.") -        await self._redis.delete(self._namespace) +        await self.bot.redis_session.delete(self._namespace)      async def pop(self, key: RedisKeyType, default: Optional[RedisValueType] = None) -> RedisValueType:          """Get the item, remove it from the cache, and provide a default if not found.""" @@ -364,7 +363,7 @@ class RedisCache:          """          await self._validate_cache()          log.trace(f"Updating the cache with the following items:\n{items}") -        await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) +        await self.bot.redis_session.hmset_dict(self._namespace, self._dict_to_typestring(items))      async def increment(self, key: RedisKeyType, amount: Optional[int, float] = 1) -> None:          """ diff --git a/bot/utils/regex.py b/bot/utils/regex.py new file mode 100644 index 000000000..0d2068f90 --- /dev/null +++ b/bot/utils/regex.py @@ -0,0 +1,12 @@ +import re + +INVITE_RE = re.compile( +    r"(?:discord(?:[\.,]|dot)gg|"                     # Could be discord.gg/ +    r"discord(?:[\.,]|dot)com(?:\/|slash)invite|"     # or discord.com/invite/ +    r"discordapp(?:[\.,]|dot)com(?:\/|slash)invite|"  # or discordapp.com/invite/ +    r"discord(?:[\.,]|dot)me|"                        # or discord.me +    r"discord(?:[\.,]|dot)io"                         # or discord.io. +    r")(?:[\/]|slash)"                                # / or 'slash' +    r"([a-zA-Z0-9\-]+)",                              # the invite code itself +    flags=re.IGNORECASE +) diff --git a/bot/utils/scheduling.py b/bot/utils/scheduling.py index 8b778a093..03f31d78f 100644 --- a/bot/utils/scheduling.py +++ b/bot/utils/scheduling.py @@ -1,81 +1,126 @@  import asyncio  import contextlib +import inspect  import logging  import typing as t -from abc import abstractmethod +from datetime import datetime  from functools import partial -from bot.utils import CogABCMeta -log = logging.getLogger(__name__) +class Scheduler: +    """ +    Schedule the execution of coroutines and keep track of them. +    When instantiating a Scheduler, a name must be provided. This name is used to distinguish the +    instance's log messages from other instances. Using the name of the class or module containing +    the instance is suggested. -class Scheduler(metaclass=CogABCMeta): -    """Task scheduler.""" +    Coroutines can be scheduled immediately with `schedule` or in the future with `schedule_at` +    or `schedule_later`. A unique ID is required to be given in order to keep track of the +    resulting Tasks. Any scheduled task can be cancelled prematurely using `cancel` by providing +    the same ID used to schedule it.  The `in` operator is supported for checking if a task with a +    given ID is currently scheduled. -    def __init__(self): -        # Keep track of the child cog's name so the logs are clear. -        self.cog_name = self.__class__.__name__ +    Any exception raised in a scheduled task is logged when the task is done. +    """ -        self._scheduled_tasks: t.Dict[t.Hashable, asyncio.Task] = {} +    def __init__(self, name: str): +        self.name = name -    @abstractmethod -    async def _scheduled_task(self, task_object: t.Any) -> None: -        """ -        A coroutine which handles the scheduling. +        self._log = logging.getLogger(f"{__name__}.{name}") +        self._scheduled_tasks: t.Dict[t.Hashable, asyncio.Task] = {} -        This is added to the scheduled tasks, and should wait the task duration, execute the desired -        code, then clean up the task. +    def __contains__(self, task_id: t.Hashable) -> bool: +        """Return True if a task with the given `task_id` is currently scheduled.""" +        return task_id in self._scheduled_tasks -        For example, in Reminders this will wait for the reminder duration, send the reminder, -        then make a site API request to delete the reminder from the database. +    def schedule(self, task_id: t.Hashable, coroutine: t.Coroutine) -> None:          """ +        Schedule the execution of a `coroutine`. -    def schedule_task(self, task_id: t.Hashable, task_data: t.Any) -> None: +        If a task with `task_id` already exists, close `coroutine` instead of scheduling it. This +        prevents unawaited coroutine warnings. Don't pass a coroutine that'll be re-used elsewhere.          """ -        Schedules a task. +        self._log.trace(f"Scheduling task #{task_id}...") -        `task_data` is passed to the `Scheduler._scheduled_task()` coroutine. -        """ -        log.trace(f"{self.cog_name}: scheduling task #{task_id}...") +        msg = f"Cannot schedule an already started coroutine for #{task_id}" +        assert inspect.getcoroutinestate(coroutine) == "CORO_CREATED", msg          if task_id in self._scheduled_tasks: -            log.debug( -                f"{self.cog_name}: did not schedule task #{task_id}; task was already scheduled." -            ) +            self._log.debug(f"Did not schedule task #{task_id}; task was already scheduled.") +            coroutine.close()              return -        task = asyncio.create_task(self._scheduled_task(task_data)) +        task = asyncio.create_task(coroutine, name=f"{self.name}_{task_id}")          task.add_done_callback(partial(self._task_done_callback, task_id))          self._scheduled_tasks[task_id] = task -        log.debug(f"{self.cog_name}: scheduled task #{task_id} {id(task)}.") +        self._log.debug(f"Scheduled task #{task_id} {id(task)}.") + +    def schedule_at(self, time: datetime, task_id: t.Hashable, coroutine: t.Coroutine) -> None: +        """ +        Schedule `coroutine` to be executed at the given naïve UTC `time`. + +        If `time` is in the past, schedule `coroutine` immediately. + +        If a task with `task_id` already exists, close `coroutine` instead of scheduling it. This +        prevents unawaited coroutine warnings. Don't pass a coroutine that'll be re-used elsewhere. +        """ +        delay = (time - datetime.utcnow()).total_seconds() +        if delay > 0: +            coroutine = self._await_later(delay, task_id, coroutine) + +        self.schedule(task_id, coroutine) -    def cancel_task(self, task_id: t.Hashable, ignore_missing: bool = False) -> None: +    def schedule_later(self, delay: t.Union[int, float], task_id: t.Hashable, coroutine: t.Coroutine) -> None:          """ -        Unschedule the task identified by `task_id`. +        Schedule `coroutine` to be executed after the given `delay` number of seconds. -        If `ignore_missing` is True, a warning will not be sent if a task isn't found. +        If a task with `task_id` already exists, close `coroutine` instead of scheduling it. This +        prevents unawaited coroutine warnings. Don't pass a coroutine that'll be re-used elsewhere.          """ -        log.trace(f"{self.cog_name}: cancelling task #{task_id}...") -        task = self._scheduled_tasks.get(task_id) +        self.schedule(task_id, self._await_later(delay, task_id, coroutine)) -        if not task: -            if not ignore_missing: -                log.warning(f"{self.cog_name}: failed to unschedule {task_id} (no task found).") -            return +    def cancel(self, task_id: t.Hashable) -> None: +        """Unschedule the task identified by `task_id`. Log a warning if the task doesn't exist.""" +        self._log.trace(f"Cancelling task #{task_id}...") -        del self._scheduled_tasks[task_id] -        task.cancel() +        try: +            task = self._scheduled_tasks.pop(task_id) +        except KeyError: +            self._log.warning(f"Failed to unschedule {task_id} (no task found).") +        else: +            task.cancel() -        log.debug(f"{self.cog_name}: unscheduled task #{task_id} {id(task)}.") +            self._log.debug(f"Unscheduled task #{task_id} {id(task)}.")      def cancel_all(self) -> None:          """Unschedule all known tasks.""" -        log.debug(f"{self.cog_name}: unscheduling all tasks") +        self._log.debug("Unscheduling all tasks")          for task_id in self._scheduled_tasks.copy(): -            self.cancel_task(task_id, ignore_missing=True) +            self.cancel(task_id) + +    async def _await_later(self, delay: t.Union[int, float], task_id: t.Hashable, coroutine: t.Coroutine) -> None: +        """Await `coroutine` after the given `delay` number of seconds.""" +        try: +            self._log.trace(f"Waiting {delay} seconds before awaiting coroutine for #{task_id}.") +            await asyncio.sleep(delay) + +            # Use asyncio.shield to prevent the coroutine from cancelling itself. +            self._log.trace(f"Done waiting for #{task_id}; now awaiting the coroutine.") +            await asyncio.shield(coroutine) +        finally: +            # Close it to prevent unawaited coroutine warnings, +            # which would happen if the task was cancelled during the sleep. +            # Only close it if it's not been awaited yet. This check is important because the +            # coroutine may cancel this task, which would also trigger the finally block. +            state = inspect.getcoroutinestate(coroutine) +            if state == "CORO_CREATED": +                self._log.debug(f"Explicitly closing the coroutine for #{task_id}.") +                coroutine.close() +            else: +                self._log.debug(f"Finally block reached for #{task_id}; {state=}")      def _task_done_callback(self, task_id: t.Hashable, done_task: asyncio.Task) -> None:          """ @@ -84,24 +129,24 @@ class Scheduler(metaclass=CogABCMeta):          If `done_task` and the task associated with `task_id` are different, then the latter          will not be deleted. In this case, a new task was likely rescheduled with the same ID.          """ -        log.trace(f"{self.cog_name}: performing done callback for task #{task_id} {id(done_task)}.") +        self._log.trace(f"Performing done callback for task #{task_id} {id(done_task)}.")          scheduled_task = self._scheduled_tasks.get(task_id)          if scheduled_task and done_task is scheduled_task: -            # A task for the ID exists and its the same as the done task. +            # A task for the ID exists and is the same as the done task.              # Since this is the done callback, the task is already done so no need to cancel it. -            log.trace(f"{self.cog_name}: deleting task #{task_id} {id(done_task)}.") +            self._log.trace(f"Deleting task #{task_id} {id(done_task)}.")              del self._scheduled_tasks[task_id]          elif scheduled_task:              # A new task was likely rescheduled with the same ID. -            log.debug( -                f"{self.cog_name}: the scheduled task #{task_id} {id(scheduled_task)} " +            self._log.debug( +                f"The scheduled task #{task_id} {id(scheduled_task)} "                  f"and the done task {id(done_task)} differ."              )          elif not done_task.cancelled(): -            log.warning( -                f"{self.cog_name}: task #{task_id} not found while handling task {id(done_task)}! " +            self._log.warning( +                f"Task #{task_id} not found while handling task {id(done_task)}! "                  f"A task somehow got unscheduled improperly (i.e. deleted but not cancelled)."              ) @@ -109,7 +154,4 @@ class Scheduler(metaclass=CogABCMeta):              exception = done_task.exception()              # Log the exception if one exists.              if exception: -                log.error( -                    f"{self.cog_name}: error in task #{task_id} {id(done_task)}!", -                    exc_info=exception -                ) +                self._log.error(f"Error in task #{task_id} {id(done_task)}!", exc_info=exception) diff --git a/bot/utils/time.py b/bot/utils/time.py index 77060143c..47e49904b 100644 --- a/bot/utils/time.py +++ b/bot/utils/time.py @@ -20,7 +20,9 @@ def _stringify_time_unit(value: int, unit: str) -> str:      >>> _stringify_time_unit(0, "minutes")      "less than a minute"      """ -    if value == 1: +    if unit == "seconds" and value == 0: +        return "0 seconds" +    elif value == 1:          return f"{value} {unit[:-1]}"      elif value == 0:          return f"less than a {unit[:-1]}" diff --git a/bot/utils/webhooks.py b/bot/utils/webhooks.py new file mode 100644 index 000000000..66f82ec66 --- /dev/null +++ b/bot/utils/webhooks.py @@ -0,0 +1,34 @@ +import logging +from typing import Optional + +import discord +from discord import Embed + +from bot.utils.messages import sub_clyde + +log = logging.getLogger(__name__) + + +async def send_webhook( +        webhook: discord.Webhook, +        content: Optional[str] = None, +        username: Optional[str] = None, +        avatar_url: Optional[str] = None, +        embed: Optional[Embed] = None, +        wait: Optional[bool] = False +) -> discord.Message: +    """ +    Send a message using the provided webhook. + +    This uses sub_clyde() and tries for an HTTPException to ensure it doesn't crash. +    """ +    try: +        return await webhook.send( +            content=content, +            username=sub_clyde(username), +            avatar_url=avatar_url, +            embed=embed, +            wait=wait, +        ) +    except discord.HTTPException: +        log.exception("Failed to send a message to the webhook!") diff --git a/config-default.yml b/config-default.yml index 64c4e715b..20254d584 100644 --- a/config-default.yml +++ b/config-default.yml @@ -38,6 +38,21 @@ style:          status_dnd:     "<:status_dnd:470326272082313216>"          status_offline: "<:status_offline:470326266537705472>" +        badge_staff: "<:discord_staff:743882896498098226>" +        badge_partner: "<:partner:748666453242413136>" +        badge_hypesquad: "<:hypesquad_events:743882896892362873>" +        badge_bug_hunter: "<:bug_hunter_lvl1:743882896372269137>" +        badge_hypesquad_bravery: "<:hypesquad_bravery:743882896745693335>" +        badge_hypesquad_brilliance: "<:hypesquad_brilliance:743882896938631248>" +        badge_hypesquad_balance: "<:hypesquad_balance:743882896460480625>" +        badge_early_supporter: "<:early_supporter:743882896909140058>" +        badge_bug_hunter_level_2: "<:bug_hunter_lvl2:743882896611344505>" +        badge_verified_bot_developer: "<:verified_bot_dev:743882897299210310>" + +        incident_actioned:      "<:incident_actioned:719645530128646266>" +        incident_unactioned:    "<:incident_unactioned:719645583245180960>" +        incident_investigating: "<:incident_investigating:719645658671480924>" +          failmail: "<:failmail:633660039931887616>"          trashcan: "<:trashcan:637136429717389331>" @@ -119,6 +134,7 @@ style:  guild:      id: 267624335836053506 +    invite: "https://discord.gg/python"      categories:          help_available:                     691405807388196926 @@ -150,6 +166,7 @@ guild:          mod_log:            &MOD_LOG        282638479504965634          user_log:                           528976905546760203          voice_log:                          640292421988646961 +        dm_log:                             653713721625018428          # Off-topic          off_topic_0:    291284109232308226 @@ -167,12 +184,13 @@ guild:          admin_spam:         &ADMIN_SPAM     563594791770914816          defcon:             &DEFCON         464469101889454091          helpers:            &HELPERS        385474242440986624 +        incidents:                          714214212200562749 +        incidents_archive:                  720668923636351037          mods:               &MODS           305126844661760000          mod_alerts:         &MOD_ALERTS     473092532147060736          mod_spam:           &MOD_SPAM       620607373828030464          organisation:       &ORGANISATION   551789653284356126          staff_lounge:       &STAFF_LOUNGE   464905259261755392 -        incidents:                          714214212200562749          # Voice          admins_voice:       &ADMINS_VOICE   500734494840717332 @@ -219,8 +237,8 @@ guild:          partners:                               323426753857191936          python_community:   &PY_COMMUNITY_ROLE  458226413825294336 -        # This is the Developers role on PyDis, here named verified for readability reasons -        verified:                               352427296948486144 +        unverified:                             739794855945044069 +        verified:                               352427296948486144  # @Developers on PyDis          # Staff          admins:             &ADMINS_ROLE    267628507062992896 @@ -230,8 +248,8 @@ guild:          owners:             &OWNERS_ROLE    267627879762755584          # Code Jam -        jammers:        591786436651646989 -        team_leaders:   501324292341104650 +        jammers:        737249140966162473 +        team_leaders:   737250302834638889      moderation_roles:          - *OWNERS_ROLE @@ -245,16 +263,16 @@ guild:          - *HELPERS_ROLE      webhooks: -        talent_pool:                    569145364800602132 -        big_brother:                    569133704568373283 -        reddit:                         635408384794951680 -        duck_pond:                      637821475327311927 -        dev_log:                        680501655111729222 -        python_news:    &PYNEWS_WEBHOOK 704381182279942324 - +        big_brother:                        569133704568373283 +        dev_log:                            680501655111729222 +        dm_log:                             654567640664244225 +        duck_pond:                          637821475327311927 +        incidents_archive:                  720671599790915702 +        python_news:        &PYNEWS_WEBHOOK 704381182279942324 +        reddit:                             635408384794951680 +        talent_pool:                        569145364800602132  filter: -      # What do we filter?      filter_zalgo:       false      filter_invites:     true @@ -269,100 +287,9 @@ filter:      notify_user_domains:     false      # Filter configuration -    ping_everyone:             true  # Ping @everyone when we send a mod-alert? +    ping_everyone:             true      offensive_msg_delete_days: 7     # How many days before deleting an offensive message? -    guild_invite_whitelist: -        - 280033776820813825  # Functional Programming -        - 267624335836053506  # Python Discord -        - 440186186024222721  # Python Discord: Emojis 1 -        - 578587418123304970  # Python Discord: Emojis 2 -        - 273944235143593984  # STEM -        - 348658686962696195  # RLBot -        - 531221516914917387  # Pallets -        - 249111029668249601  # Gentoo -        - 327254708534116352  # Adafruit -        - 544525886180032552  # kennethreitz.org -        - 590806733924859943  # Discord Hack Week -        - 423249981340778496  # Kivy -        - 197038439483310086  # Discord Testers -        - 286633898581164032  # Ren'Py -        - 349505959032389632  # PyGame -        - 438622377094414346  # Pyglet -        - 524691714909274162  # Panda3D -        - 336642139381301249  # discord.py -        - 405403391410438165  # Sentdex -        - 172018499005317120  # The Coding Den -        - 666560367173828639  # PyWeek -        - 702724176489873509  # Microsoft Python -        - 81384788765712384   # Discord API -        - 613425648685547541  # Discord Developers -        - 185590609631903755  # Blender Hub -        - 420324994703163402  # /r/FlutterDev -        - 488751051629920277  # Python Atlanta -        - 143867839282020352  # C# - -    domain_blacklist: -        - pornhub.com -        - liveleak.com -        - grabify.link -        - bmwforum.co -        - leancoding.co -        - spottyfly.com -        - stopify.co -        - yoütu.be -        - discörd.com -        - minecräft.com -        - freegiftcards.co -        - disçordapp.com -        - fortnight.space -        - fortnitechat.site -        - joinmy.site -        - curiouscat.club -        - catsnthings.fun -        - yourtube.site -        - youtubeshort.watch -        - catsnthing.com -        - youtubeshort.pro -        - canadianlumberjacks.online -        - poweredbydialup.club -        - poweredbydialup.online -        - poweredbysecurity.org -        - poweredbysecurity.online -        - ssteam.site -        - steamwalletgift.com -        - discord.gift -        - lmgtfy.com - -    word_watchlist: -        - goo+ks* -        - ky+s+ -        - ki+ke+s* -        - beaner+s? -        - coo+ns* -        - nig+lets* -        - slant-eyes* -        - towe?l-?head+s* -        - chi*n+k+s* -        - spick*s* -        - kill* +(?:yo)?urself+ -        - jew+s* -        - suicide -        - rape -        - (re+)tar+(d+|t+)(ed)? -        - ta+r+d+ -        - cunts* -        - trann*y -        - shemale - -    token_watchlist: -        - fa+g+s* -        - 卐 -        - 卍 -        - cuck(?!oo+) -        - nigg+(?:e*r+|a+h*?|u+h+)s? -        - fag+o+t+s* -      # Censor doesn't apply to these      channel_whitelist:          - *ADMINS @@ -394,24 +321,7 @@ urls:      site_staff:  &STAFF  !JOIN ["staff.", *DOMAIN]      site_schema: &SCHEMA       "https://" -    site_bigbrother_api:                !JOIN [*SCHEMA, *API, "/bot/bigbrother"] -    site_docs_api:                      !JOIN [*SCHEMA, *API, "/bot/docs"] -    site_superstarify_api:              !JOIN [*SCHEMA, *API, "/bot/superstarify"] -    site_infractions:                   !JOIN [*SCHEMA, *API, "/bot/infractions"] -    site_infractions_user:              !JOIN [*SCHEMA, *API, "/bot/infractions/user/{user_id}"] -    site_infractions_type:              !JOIN [*SCHEMA, *API, "/bot/infractions/type/{infraction_type}"] -    site_infractions_by_id:             !JOIN [*SCHEMA, *API, "/bot/infractions/id/{infraction_id}"] -    site_infractions_user_type_current: !JOIN [*SCHEMA, *API, "/bot/infractions/user/{user_id}/{infraction_type}/current"] -    site_infractions_user_type:         !JOIN [*SCHEMA, *API, "/bot/infractions/user/{user_id}/{infraction_type}"] -    site_logs_api:                      !JOIN [*SCHEMA, *API, "/bot/logs"]      site_logs_view:                     !JOIN [*SCHEMA, *STAFF, "/bot/logs"] -    site_off_topic_names_api:           !JOIN [*SCHEMA, *API, "/bot/off-topic-names"] -    site_reminders_api:                 !JOIN [*SCHEMA, *API, "/bot/reminders"] -    site_reminders_user_api:            !JOIN [*SCHEMA, *API, "/bot/reminders/user"] -    site_settings_api:                  !JOIN [*SCHEMA, *API, "/bot/settings"] -    site_tags_api:                      !JOIN [*SCHEMA, *API, "/bot/tags"] -    site_user_api:                      !JOIN [*SCHEMA, *API, "/bot/users"] -    site_user_complete_api:             !JOIN [*SCHEMA, *API, "/bot/users/complete"]      paste_service:                      !JOIN [*SCHEMA, *PASTE, "/{key}"]      # Snekbox @@ -443,9 +353,13 @@ anti_spam:              interval: 10              max: 7 -        burst_shared: -            interval: 10 -            max: 20 +        # Burst shared it (temporarily) disabled to prevent +        # the bug that triggers multiple infractions/DMs per +        # user. It also tends to catch a lot of innocent users +        # now that we're so big. +        # burst_shared: +        #    interval: 10 +        #    max: 20          chars:              interval: 5 @@ -476,34 +390,11 @@ anti_spam:              interval: 10              max: 3 - -anti_malware: -    whitelist: -        - '.3gp' -        - '.3g2' -        - '.avi' -        - '.bmp' -        - '.gif' -        - '.h264' -        - '.jpg' -        - '.jpeg' -        - '.m4v' -        - '.mkv' -        - '.mov' -        - '.mp4' -        - '.mpeg' -        - '.mpg' -        - '.png' -        - '.tiff' -        - '.wmv' -        - '.svg' -        - '.psd'  # Photoshop -        - '.ai'   # Illustrator -        - '.aep'  # After Effects -        - '.xcf'  # GIMP -        - '.mp3' -        - '.wav' -        - '.ogg' +        # The everyone ping filter is temporarily disabled +        # until we've fixed a couple of bugs. +        # everyone_ping: +        #    interval: 10 +        #    max: 0  reddit: @@ -513,13 +404,6 @@ reddit:      secret:    !ENV "REDDIT_SECRET" -wolfram: -    # Max requests per day. -    user_limit_day: 10 -    guild_limit_day: 67 -    key: !ENV "WOLFRAM_API_KEY" - -  big_brother:      log_delay: 15      header_message_limit: 15 @@ -546,8 +430,8 @@ help_channels:      # Allowed duration of inactivity before making a channel dormant      idle_minutes: 30 -    # Allowed duration of inactivity when question message deleted -    # and no one other sent before message making channel dormant. +    # Allowed duration of inactivity when channel is empty (due to deleted messages) +    # before message making a channel dormant      deleted_idle_minutes: 5      # Maximum number of channels to put in the available category @@ -606,5 +490,18 @@ python_news:      channel: *PYNEWS_CHANNEL      webhook: *PYNEWS_WEBHOOK + +verification: +    unverified_after: 3  # Days after which non-Developers receive the @Unverified role +    kicked_after: 30  # Days after which non-Developers get kicked from the guild +    reminder_frequency: 28  # Hours between @Unverified pings +    bot_message_delete_delay: 10  # Seconds before deleting bots response in #verification + +    # Number in range [0, 1] determining the percentage of unverified users that are safe +    # to be kicked from the guild in one batch, any larger amount will require staff confirmation, +    # set this to 0 to require explicit approval for batches of any size +    kick_confirmation_threshold: 0.01  # 1% + +  config:      required_keys: ['bot.token'] diff --git a/tests/bot/cogs/moderation/test_incidents.py b/tests/bot/cogs/moderation/test_incidents.py new file mode 100644 index 000000000..435a1cd51 --- /dev/null +++ b/tests/bot/cogs/moderation/test_incidents.py @@ -0,0 +1,770 @@ +import asyncio +import enum +import logging +import typing as t +import unittest +from unittest.mock import AsyncMock, MagicMock, call, patch + +import aiohttp +import discord + +from bot.cogs.moderation import Incidents, incidents +from bot.constants import Colours +from tests.helpers import ( +    MockAsyncWebhook, +    MockAttachment, +    MockBot, +    MockMember, +    MockMessage, +    MockReaction, +    MockRole, +    MockTextChannel, +    MockUser, +) + + +class MockAsyncIterable: +    """ +    Helper for mocking asynchronous for loops. + +    It does not appear that the `unittest` library currently provides anything that would +    allow us to simply mock an async iterator, such as `discord.TextChannel.history`. + +    We therefore write our own helper to wrap a regular synchronous iterable, and feed +    its values via `__anext__` rather than `__next__`. + +    This class was written for the purposes of testing the `Incidents` cog - it may not +    be generic enough to be placed in the `tests.helpers` module. +    """ + +    def __init__(self, messages: t.Iterable): +        """Take a sync iterable to be wrapped.""" +        self.iter_messages = iter(messages) + +    def __aiter__(self): +        """Return `self` as we provide the `__anext__` method.""" +        return self + +    async def __anext__(self): +        """ +        Feed the next item, or raise `StopAsyncIteration`. + +        Since we're wrapping a sync iterator, it will communicate that it has been depleted +        by raising a `StopIteration`. The `async for` construct does not expect it, and we +        therefore need to substitute it for the appropriate exception type. +        """ +        try: +            return next(self.iter_messages) +        except StopIteration: +            raise StopAsyncIteration + + +class MockSignal(enum.Enum): +    A = "A" +    B = "B" + + +mock_404 = discord.NotFound( +    response=MagicMock(aiohttp.ClientResponse),  # Mock the erroneous response +    message="Not found", +) + + +class TestDownloadFile(unittest.IsolatedAsyncioTestCase): +    """Collection of tests for the `download_file` helper function.""" + +    async def test_download_file_success(self): +        """If `to_file` succeeds, function returns the acquired `discord.File`.""" +        file = MagicMock(discord.File, filename="bigbadlemon.jpg") +        attachment = MockAttachment(to_file=AsyncMock(return_value=file)) + +        acquired_file = await incidents.download_file(attachment) +        self.assertIs(file, acquired_file) + +    async def test_download_file_404(self): +        """If `to_file` encounters a 404, function handles the exception & returns None.""" +        attachment = MockAttachment(to_file=AsyncMock(side_effect=mock_404)) + +        acquired_file = await incidents.download_file(attachment) +        self.assertIsNone(acquired_file) + +    async def test_download_file_fail(self): +        """If `to_file` fails on a non-404 error, function logs the exception & returns None.""" +        arbitrary_error = discord.HTTPException(MagicMock(aiohttp.ClientResponse), "Arbitrary API error") +        attachment = MockAttachment(to_file=AsyncMock(side_effect=arbitrary_error)) + +        with self.assertLogs(logger=incidents.log, level=logging.ERROR): +            acquired_file = await incidents.download_file(attachment) + +        self.assertIsNone(acquired_file) + + +class TestMakeEmbed(unittest.IsolatedAsyncioTestCase): +    """Collection of tests for the `make_embed` helper function.""" + +    async def test_make_embed_actioned(self): +        """Embed is coloured green and footer contains 'Actioned' when `outcome=Signal.ACTIONED`.""" +        embed, file = await incidents.make_embed(MockMessage(), incidents.Signal.ACTIONED, MockMember()) + +        self.assertEqual(embed.colour.value, Colours.soft_green) +        self.assertIn("Actioned", embed.footer.text) + +    async def test_make_embed_not_actioned(self): +        """Embed is coloured red and footer contains 'Rejected' when `outcome=Signal.NOT_ACTIONED`.""" +        embed, file = await incidents.make_embed(MockMessage(), incidents.Signal.NOT_ACTIONED, MockMember()) + +        self.assertEqual(embed.colour.value, Colours.soft_red) +        self.assertIn("Rejected", embed.footer.text) + +    async def test_make_embed_content(self): +        """Incident content appears as embed description.""" +        incident = MockMessage(content="this is an incident") +        embed, file = await incidents.make_embed(incident, incidents.Signal.ACTIONED, MockMember()) + +        self.assertEqual(incident.content, embed.description) + +    async def test_make_embed_with_attachment_succeeds(self): +        """Incident's attachment is downloaded and displayed in the embed's image field.""" +        file = MagicMock(discord.File, filename="bigbadjoe.jpg") +        attachment = MockAttachment(filename="bigbadjoe.jpg") +        incident = MockMessage(content="this is an incident", attachments=[attachment]) + +        # Patch `download_file` to return our `file` +        with patch("bot.cogs.moderation.incidents.download_file", AsyncMock(return_value=file)): +            embed, returned_file = await incidents.make_embed(incident, incidents.Signal.ACTIONED, MockMember()) + +        self.assertIs(file, returned_file) +        self.assertEqual("attachment://bigbadjoe.jpg", embed.image.url) + +    async def test_make_embed_with_attachment_fails(self): +        """Incident's attachment fails to download, proxy url is linked instead.""" +        attachment = MockAttachment(proxy_url="discord.com/bigbadjoe.jpg") +        incident = MockMessage(content="this is an incident", attachments=[attachment]) + +        # Patch `download_file` to return None as if the download failed +        with patch("bot.cogs.moderation.incidents.download_file", AsyncMock(return_value=None)): +            embed, returned_file = await incidents.make_embed(incident, incidents.Signal.ACTIONED, MockMember()) + +        self.assertIsNone(returned_file) + +        # The author name field is simply expected to have something in it, we do not assert the message +        self.assertGreater(len(embed.author.name), 0) +        self.assertEqual(embed.author.url, "discord.com/bigbadjoe.jpg")  # However, it should link the exact url + + +@patch("bot.constants.Channels.incidents", 123) +class TestIsIncident(unittest.TestCase): +    """ +    Collection of tests for the `is_incident` helper function. + +    In `setUp`, we will create a mock message which should qualify as an incident. Each +    test case will then mutate this instance to make it **not** qualify, in various ways. + +    Notice that we patch the #incidents channel id globally for this class. +    """ + +    def setUp(self) -> None: +        """Prepare a mock message which should qualify as an incident.""" +        self.incident = MockMessage( +            channel=MockTextChannel(id=123), +            content="this is an incident", +            author=MockUser(bot=False), +            pinned=False, +        ) + +    def test_is_incident_true(self): +        """Message qualifies as an incident if unchanged.""" +        self.assertTrue(incidents.is_incident(self.incident)) + +    def check_false(self): +        """Assert that `self.incident` does **not** qualify as an incident.""" +        self.assertFalse(incidents.is_incident(self.incident)) + +    def test_is_incident_false_channel(self): +        """Message doesn't qualify if sent outside of #incidents.""" +        self.incident.channel = MockTextChannel(id=456) +        self.check_false() + +    def test_is_incident_false_content(self): +        """Message doesn't qualify if content begins with hash symbol.""" +        self.incident.content = "# this is a comment message" +        self.check_false() + +    def test_is_incident_false_author(self): +        """Message doesn't qualify if author is a bot.""" +        self.incident.author = MockUser(bot=True) +        self.check_false() + +    def test_is_incident_false_pinned(self): +        """Message doesn't qualify if it is pinned.""" +        self.incident.pinned = True +        self.check_false() + + +class TestOwnReactions(unittest.TestCase): +    """Assertions for the `own_reactions` function.""" + +    def test_own_reactions(self): +        """Only bot's own emoji are extracted from the input incident.""" +        reactions = ( +            MockReaction(emoji="A", me=True), +            MockReaction(emoji="B", me=True), +            MockReaction(emoji="C", me=False), +        ) +        message = MockMessage(reactions=reactions) +        self.assertSetEqual(incidents.own_reactions(message), {"A", "B"}) + + +@patch("bot.cogs.moderation.incidents.ALL_SIGNALS", {"A", "B"}) +class TestHasSignals(unittest.TestCase): +    """ +    Assertions for the `has_signals` function. + +    We patch `ALL_SIGNALS` globally. Each test function then patches `own_reactions` +    as appropriate. +    """ + +    def test_has_signals_true(self): +        """True when `own_reactions` returns all emoji in `ALL_SIGNALS`.""" +        message = MockMessage() +        own_reactions = MagicMock(return_value={"A", "B"}) + +        with patch("bot.cogs.moderation.incidents.own_reactions", own_reactions): +            self.assertTrue(incidents.has_signals(message)) + +    def test_has_signals_false(self): +        """False when `own_reactions` does not return all emoji in `ALL_SIGNALS`.""" +        message = MockMessage() +        own_reactions = MagicMock(return_value={"A", "C"}) + +        with patch("bot.cogs.moderation.incidents.own_reactions", own_reactions): +            self.assertFalse(incidents.has_signals(message)) + + +@patch("bot.cogs.moderation.incidents.Signal", MockSignal) +class TestAddSignals(unittest.IsolatedAsyncioTestCase): +    """ +    Assertions for the `add_signals` coroutine. + +    These are all fairly similar and could go into a single test function, but I found the +    patching & sub-testing fairly awkward in that case and decided to split them up +    to avoid unnecessary syntax noise. +    """ + +    def setUp(self): +        """Prepare a mock incident message for tests to use.""" +        self.incident = MockMessage() + +    @patch("bot.cogs.moderation.incidents.own_reactions", MagicMock(return_value=set())) +    async def test_add_signals_missing(self): +        """All emoji are added when none are present.""" +        await incidents.add_signals(self.incident) +        self.incident.add_reaction.assert_has_calls([call("A"), call("B")]) + +    @patch("bot.cogs.moderation.incidents.own_reactions", MagicMock(return_value={"A"})) +    async def test_add_signals_partial(self): +        """Only missing emoji are added when some are present.""" +        await incidents.add_signals(self.incident) +        self.incident.add_reaction.assert_has_calls([call("B")]) + +    @patch("bot.cogs.moderation.incidents.own_reactions", MagicMock(return_value={"A", "B"})) +    async def test_add_signals_present(self): +        """No emoji are added when all are present.""" +        await incidents.add_signals(self.incident) +        self.incident.add_reaction.assert_not_called() + + +class TestIncidents(unittest.IsolatedAsyncioTestCase): +    """ +    Tests for bound methods of the `Incidents` cog. + +    Use this as a base class for `Incidents` tests - it will prepare a fresh instance +    for each test function, but not make any assertions on its own. Tests can mutate +    the instance as they wish. +    """ + +    def setUp(self): +        """ +        Prepare a fresh `Incidents` instance for each test. + +        Note that this will not schedule `crawl_incidents` in the background, as everything +        is being mocked. The `crawl_task` attribute will end up being None. +        """ +        self.cog_instance = Incidents(MockBot()) + + +@patch("asyncio.sleep", AsyncMock())  # Prevent the coro from sleeping to speed up the test +class TestCrawlIncidents(TestIncidents): +    """ +    Tests for the `Incidents.crawl_incidents` coroutine. + +    Apart from `test_crawl_incidents_waits_until_cache_ready`, all tests in this class +    will patch the return values of `is_incident` and `has_signal` and then observe +    whether the `AsyncMock` for `add_signals` was awaited or not. + +    The `add_signals` mock is added by each test separately to ensure it is clean (has not +    been awaited by another test yet). The mock can be reset, but this appears to be the +    cleaner way. + +    For each test, we inject a mock channel with a history of 1 message only (see: `setUp`). +    """ + +    def setUp(self): +        """For each test, ensure `bot.get_channel` returns a channel with 1 arbitrary message.""" +        super().setUp()  # First ensure we get `cog_instance` from parent + +        incidents_history = MagicMock(return_value=MockAsyncIterable([MockMessage()])) +        self.cog_instance.bot.get_channel = MagicMock(return_value=MockTextChannel(history=incidents_history)) + +    async def test_crawl_incidents_waits_until_cache_ready(self): +        """ +        The coroutine will await the `wait_until_guild_available` event. + +        Since this task is schedule in the `__init__`, it is critical that it waits for the +        cache to be ready, so that it can safely get the #incidents channel. +        """ +        await self.cog_instance.crawl_incidents() +        self.cog_instance.bot.wait_until_guild_available.assert_awaited() + +    @patch("bot.cogs.moderation.incidents.add_signals", AsyncMock()) +    @patch("bot.cogs.moderation.incidents.is_incident", MagicMock(return_value=False))  # Message doesn't qualify +    @patch("bot.cogs.moderation.incidents.has_signals", MagicMock(return_value=False)) +    async def test_crawl_incidents_noop_if_is_not_incident(self): +        """Signals are not added for a non-incident message.""" +        await self.cog_instance.crawl_incidents() +        incidents.add_signals.assert_not_awaited() + +    @patch("bot.cogs.moderation.incidents.add_signals", AsyncMock()) +    @patch("bot.cogs.moderation.incidents.is_incident", MagicMock(return_value=True))  # Message qualifies +    @patch("bot.cogs.moderation.incidents.has_signals", MagicMock(return_value=True))  # But already has signals +    async def test_crawl_incidents_noop_if_message_already_has_signals(self): +        """Signals are not added for messages which already have them.""" +        await self.cog_instance.crawl_incidents() +        incidents.add_signals.assert_not_awaited() + +    @patch("bot.cogs.moderation.incidents.add_signals", AsyncMock()) +    @patch("bot.cogs.moderation.incidents.is_incident", MagicMock(return_value=True))  # Message qualifies +    @patch("bot.cogs.moderation.incidents.has_signals", MagicMock(return_value=False))  # And doesn't have signals +    async def test_crawl_incidents_add_signals_called(self): +        """Message has signals added as it does not have them yet and qualifies as an incident.""" +        await self.cog_instance.crawl_incidents() +        incidents.add_signals.assert_awaited_once() + + +class TestArchive(TestIncidents): +    """Tests for the `Incidents.archive` coroutine.""" + +    async def test_archive_webhook_not_found(self): +        """ +        Method recovers and returns False when the webhook is not found. + +        Implicitly, this also tests that the error is handled internally and doesn't +        propagate out of the method, which is just as important. +        """ +        self.cog_instance.bot.fetch_webhook = AsyncMock(side_effect=mock_404) +        self.assertFalse( +            await self.cog_instance.archive(incident=MockMessage(), outcome=MagicMock(), actioned_by=MockMember()) +        ) + +    async def test_archive_relays_incident(self): +        """ +        If webhook is found, method relays `incident` properly. + +        This test will assert that the fetched webhook's `send` method is fed the correct arguments, +        and that the `archive` method returns True. +        """ +        webhook = MockAsyncWebhook() +        self.cog_instance.bot.fetch_webhook = AsyncMock(return_value=webhook)  # Patch in our webhook + +        # Define our own `incident` to be archived +        incident = MockMessage( +            content="this is an incident", +            author=MockUser(name="author_name", avatar_url="author_avatar"), +            id=123, +        ) +        built_embed = MagicMock(discord.Embed, id=123)  # We patch `make_embed` to return this + +        with patch("bot.cogs.moderation.incidents.make_embed", AsyncMock(return_value=(built_embed, None))): +            archive_return = await self.cog_instance.archive(incident, MagicMock(value="A"), MockMember()) + +        # Now we check that the webhook was given the correct args, and that `archive` returned True +        webhook.send.assert_called_once_with( +            embed=built_embed, +            username="author_name", +            avatar_url="author_avatar", +            file=None, +        ) +        self.assertTrue(archive_return) + +    async def test_archive_clyde_username(self): +        """ +        The archive webhook username is cleansed using `sub_clyde`. + +        Discord will reject any webhook with "clyde" in the username field, as it impersonates +        the official Clyde bot. Since we do not control what the username will be (the incident +        author name is used), we must ensure the name is cleansed, otherwise the relay may fail. + +        This test assumes the username is passed as a kwarg. If this test fails, please review +        whether the passed argument is being retrieved correctly. +        """ +        webhook = MockAsyncWebhook() +        self.cog_instance.bot.fetch_webhook = AsyncMock(return_value=webhook) + +        message_from_clyde = MockMessage(author=MockUser(name="clyde the great")) +        await self.cog_instance.archive(message_from_clyde, MagicMock(incidents.Signal), MockMember()) + +        self.assertNotIn("clyde", webhook.send.call_args.kwargs["username"]) + + +class TestMakeConfirmationTask(TestIncidents): +    """ +    Tests for the `Incidents.make_confirmation_task` method. + +    Writing tests for this method is difficult, as it mostly just delegates the provided +    information elsewhere. There is very little internal logic. Whether our approach +    works conceptually is difficult to prove using unit tests. +    """ + +    def test_make_confirmation_task_check(self): +        """ +        The internal check will recognize the passed incident. + +        This is a little tricky - we first pass a message with a specific `id` in, and then +        retrieve the built check from the `call_args` of the `wait_for` method. This relies +        on the check being passed as a kwarg. + +        Once the check is retrieved, we assert that it gives True for our incident's `id`, +        and False for any other. + +        If this function begins to fail, first check that `created_check` is being retrieved +        correctly. It should be the function that is built locally in the tested method. +        """ +        self.cog_instance.make_confirmation_task(MockMessage(id=123)) + +        self.cog_instance.bot.wait_for.assert_called_once() +        created_check = self.cog_instance.bot.wait_for.call_args.kwargs["check"] + +        # The `message_id` matches the `id` of our incident +        self.assertTrue(created_check(payload=MagicMock(message_id=123))) + +        # This `message_id` does not match +        self.assertFalse(created_check(payload=MagicMock(message_id=0))) + + +@patch("bot.cogs.moderation.incidents.ALLOWED_ROLES", {1, 2}) +@patch("bot.cogs.moderation.incidents.Incidents.make_confirmation_task", AsyncMock())  # Generic awaitable +class TestProcessEvent(TestIncidents): +    """Tests for the `Incidents.process_event` coroutine.""" + +    async def test_process_event_bad_role(self): +        """The reaction is removed when the author lacks all allowed roles.""" +        incident = MockMessage() +        member = MockMember(roles=[MockRole(id=0)])  # Must have role 1 or 2 + +        await self.cog_instance.process_event("reaction", incident, member) +        incident.remove_reaction.assert_called_once_with("reaction", member) + +    async def test_process_event_bad_emoji(self): +        """ +        The reaction is removed when an invalid emoji is used. + +        This requires that we pass in a `member` with valid roles, as we need the role check +        to succeed. +        """ +        incident = MockMessage() +        member = MockMember(roles=[MockRole(id=1)])  # Member has allowed role + +        await self.cog_instance.process_event("invalid_signal", incident, member) +        incident.remove_reaction.assert_called_once_with("invalid_signal", member) + +    async def test_process_event_no_archive_on_investigating(self): +        """Message is not archived on `Signal.INVESTIGATING`.""" +        with patch("bot.cogs.moderation.incidents.Incidents.archive", AsyncMock()) as mocked_archive: +            await self.cog_instance.process_event( +                reaction=incidents.Signal.INVESTIGATING.value, +                incident=MockMessage(), +                member=MockMember(roles=[MockRole(id=1)]), +            ) + +        mocked_archive.assert_not_called() + +    async def test_process_event_no_delete_if_archive_fails(self): +        """ +        Original message is not deleted when `Incidents.archive` returns False. + +        This is the way of signaling that the relay failed, and we should not remove the original, +        as that would result in losing the incident record. +        """ +        incident = MockMessage() + +        with patch("bot.cogs.moderation.incidents.Incidents.archive", AsyncMock(return_value=False)): +            await self.cog_instance.process_event( +                reaction=incidents.Signal.ACTIONED.value, +                incident=incident, +                member=MockMember(roles=[MockRole(id=1)]) +            ) + +        incident.delete.assert_not_called() + +    async def test_process_event_confirmation_task_is_awaited(self): +        """Task given by `Incidents.make_confirmation_task` is awaited before method exits.""" +        mock_task = AsyncMock() + +        with patch("bot.cogs.moderation.incidents.Incidents.make_confirmation_task", mock_task): +            await self.cog_instance.process_event( +                reaction=incidents.Signal.ACTIONED.value, +                incident=MockMessage(), +                member=MockMember(roles=[MockRole(id=1)]) +            ) + +        mock_task.assert_awaited() + +    async def test_process_event_confirmation_task_timeout_is_handled(self): +        """ +        Confirmation task `asyncio.TimeoutError` is handled gracefully. + +        We have `make_confirmation_task` return a mock with a side effect, and then catch the +        exception should it propagate out of `process_event`. This is so that we can then manually +        fail the test with a more informative message than just the plain traceback. +        """ +        mock_task = AsyncMock(side_effect=asyncio.TimeoutError()) + +        try: +            with patch("bot.cogs.moderation.incidents.Incidents.make_confirmation_task", mock_task): +                await self.cog_instance.process_event( +                    reaction=incidents.Signal.ACTIONED.value, +                    incident=MockMessage(), +                    member=MockMember(roles=[MockRole(id=1)]) +                ) +        except asyncio.TimeoutError: +            self.fail("TimeoutError was not handled gracefully, and propagated out of `process_event`!") + + +class TestResolveMessage(TestIncidents): +    """Tests for the `Incidents.resolve_message` coroutine.""" + +    async def test_resolve_message_pass_message_id(self): +        """Method will call `_get_message` with the passed `message_id`.""" +        await self.cog_instance.resolve_message(123) +        self.cog_instance.bot._connection._get_message.assert_called_once_with(123) + +    async def test_resolve_message_in_cache(self): +        """ +        No API call is made if the queried message exists in the cache. + +        We mock the `_get_message` return value regardless of input. Whether it finds the message +        internally is considered d.py's responsibility, not ours. +        """ +        cached_message = MockMessage(id=123) +        self.cog_instance.bot._connection._get_message = MagicMock(return_value=cached_message) + +        return_value = await self.cog_instance.resolve_message(123) + +        self.assertIs(return_value, cached_message) +        self.cog_instance.bot.get_channel.assert_not_called()  # The `fetch_message` line was never hit + +    async def test_resolve_message_not_in_cache(self): +        """ +        The message is retrieved from the API if it isn't cached. + +        This is desired behaviour for messages which exist, but were sent before the bot's +        current session. +        """ +        self.cog_instance.bot._connection._get_message = MagicMock(return_value=None)  # Cache returns None + +        # API returns our message +        uncached_message = MockMessage() +        fetch_message = AsyncMock(return_value=uncached_message) +        self.cog_instance.bot.get_channel = MagicMock(return_value=MockTextChannel(fetch_message=fetch_message)) + +        retrieved_message = await self.cog_instance.resolve_message(123) +        self.assertIs(retrieved_message, uncached_message) + +    async def test_resolve_message_doesnt_exist(self): +        """ +        If the API returns a 404, the function handles it gracefully and returns None. + +        This is an edge-case happening with racing events - event A will relay the message +        to the archive and delete the original. Once event B acquires the `event_lock`, +        it will not find the message in the cache, and will ask the API. +        """ +        self.cog_instance.bot._connection._get_message = MagicMock(return_value=None)  # Cache returns None + +        fetch_message = AsyncMock(side_effect=mock_404) +        self.cog_instance.bot.get_channel = MagicMock(return_value=MockTextChannel(fetch_message=fetch_message)) + +        self.assertIsNone(await self.cog_instance.resolve_message(123)) + +    async def test_resolve_message_fetch_fails(self): +        """ +        Non-404 errors are handled, logged & None is returned. + +        In contrast with a 404, this should make an error-level log. We assert that at least +        one such log was made - we do not make any assertions about the log's message. +        """ +        self.cog_instance.bot._connection._get_message = MagicMock(return_value=None)  # Cache returns None + +        arbitrary_error = discord.HTTPException( +            response=MagicMock(aiohttp.ClientResponse), +            message="Arbitrary error", +        ) +        fetch_message = AsyncMock(side_effect=arbitrary_error) +        self.cog_instance.bot.get_channel = MagicMock(return_value=MockTextChannel(fetch_message=fetch_message)) + +        with self.assertLogs(logger=incidents.log, level=logging.ERROR): +            self.assertIsNone(await self.cog_instance.resolve_message(123)) + + +@patch("bot.constants.Channels.incidents", 123) +class TestOnRawReactionAdd(TestIncidents): +    """ +    Tests for the `Incidents.on_raw_reaction_add` listener. + +    Writing tests for this listener comes with additional complexity due to the listener +    awaiting the `crawl_task` task. See `asyncSetUp` for further details, which attempts +    to make unit testing this function possible. +    """ + +    def setUp(self): +        """ +        Prepare & assign `payload` attribute. + +        This attribute represents an *ideal* payload which will not be rejected by the +        listener. As each test will receive a fresh instance, it can be mutated to +        observe how the listener's behaviour changes with different attributes on +        the passed payload. +        """ +        super().setUp()  # Ensure `cog_instance` is assigned + +        self.payload = MagicMock( +            discord.RawReactionActionEvent, +            channel_id=123,  # Patched at class level +            message_id=456, +            member=MockMember(bot=False), +            emoji="reaction", +        ) + +    async def asyncSetUp(self):  # noqa: N802 +        """ +        Prepare an empty task and assign it as `crawl_task`. + +        It appears that the `unittest` framework does not provide anything for mocking +        asyncio tasks. An `AsyncMock` instance can be called and then awaited, however, +        it does not provide the `done` method or any other parts of the `asyncio.Task` +        interface. + +        Although we do not need to make any assertions about the task itself while +        testing the listener, the code will still await it and call the `done` method, +        and so we must inject something that will not fail on either action. + +        Note that this is done in an `asyncSetUp`, which runs after `setUp`. +        The justification is that creating an actual task requires the event +        loop to be ready, which is not the case in the `setUp`. +        """ +        mock_task = asyncio.create_task(AsyncMock()())  # Mock async func, then a coro +        self.cog_instance.crawl_task = mock_task + +    async def test_on_raw_reaction_add_wrong_channel(self): +        """ +        Events outside of #incidents will be ignored. + +        We check this by asserting that `resolve_message` was never queried. +        """ +        self.payload.channel_id = 0 +        self.cog_instance.resolve_message = AsyncMock() + +        await self.cog_instance.on_raw_reaction_add(self.payload) +        self.cog_instance.resolve_message.assert_not_called() + +    async def test_on_raw_reaction_add_user_is_bot(self): +        """ +        Events dispatched by bot accounts will be ignored. + +        We check this by asserting that `resolve_message` was never queried. +        """ +        self.payload.member = MockMember(bot=True) +        self.cog_instance.resolve_message = AsyncMock() + +        await self.cog_instance.on_raw_reaction_add(self.payload) +        self.cog_instance.resolve_message.assert_not_called() + +    async def test_on_raw_reaction_add_message_doesnt_exist(self): +        """ +        Listener gracefully handles the case where `resolve_message` gives None. + +        We check this by asserting that `process_event` was never called. +        """ +        self.cog_instance.process_event = AsyncMock() +        self.cog_instance.resolve_message = AsyncMock(return_value=None) + +        await self.cog_instance.on_raw_reaction_add(self.payload) +        self.cog_instance.process_event.assert_not_called() + +    async def test_on_raw_reaction_add_message_is_not_an_incident(self): +        """ +        The event won't be processed if the related message is not an incident. + +        This is an edge-case that can happen if someone manually leaves a reaction +        on a pinned message, or a comment. + +        We check this by asserting that `process_event` was never called. +        """ +        self.cog_instance.process_event = AsyncMock() +        self.cog_instance.resolve_message = AsyncMock(return_value=MockMessage()) + +        with patch("bot.cogs.moderation.incidents.is_incident", MagicMock(return_value=False)): +            await self.cog_instance.on_raw_reaction_add(self.payload) + +        self.cog_instance.process_event.assert_not_called() + +    async def test_on_raw_reaction_add_valid_event_is_processed(self): +        """ +        If the reaction event is valid, it is passed to `process_event`. + +        This is the case when everything goes right: +            * The reaction was placed in #incidents, and not by a bot +            * The message was found successfully +            * The message qualifies as an incident + +        Additionally, we check that all arguments were passed as expected. +        """ +        incident = MockMessage(id=1) + +        self.cog_instance.process_event = AsyncMock() +        self.cog_instance.resolve_message = AsyncMock(return_value=incident) + +        with patch("bot.cogs.moderation.incidents.is_incident", MagicMock(return_value=True)): +            await self.cog_instance.on_raw_reaction_add(self.payload) + +        self.cog_instance.process_event.assert_called_with( +            "reaction",  # Defined in `self.payload` +            incident, +            self.payload.member, +        ) + + +class TestOnMessage(TestIncidents): +    """ +    Tests for the `Incidents.on_message` listener. + +    Notice the decorators mocking the `is_incident` return value. The `is_incidents` +    function is tested in `TestIsIncident` - here we do not worry about it. +    """ + +    @patch("bot.cogs.moderation.incidents.is_incident", MagicMock(return_value=True)) +    async def test_on_message_incident(self): +        """Messages qualifying as incidents are passed to `add_signals`.""" +        incident = MockMessage() + +        with patch("bot.cogs.moderation.incidents.add_signals", AsyncMock()) as mock_add_signals: +            await self.cog_instance.on_message(incident) + +        mock_add_signals.assert_called_once_with(incident) + +    @patch("bot.cogs.moderation.incidents.is_incident", MagicMock(return_value=False)) +    async def test_on_message_non_incident(self): +        """Messages not qualifying as incidents are ignored.""" +        with patch("bot.cogs.moderation.incidents.add_signals", AsyncMock()) as mock_add_signals: +            await self.cog_instance.on_message(MockMessage()) + +        mock_add_signals.assert_not_called() diff --git a/tests/bot/cogs/sync/test_cog.py b/tests/bot/cogs/sync/test_cog.py index 14fd909c4..120bc991d 100644 --- a/tests/bot/cogs/sync/test_cog.py +++ b/tests/bot/cogs/sync/test_cog.py @@ -131,6 +131,15 @@ class SyncCogListenerTests(SyncCogTestCase):          super().setUp()          self.cog.patch_user = mock.AsyncMock(spec_set=self.cog.patch_user) +        self.guild_id_patcher = mock.patch("bot.cogs.sync.cog.constants.Guild.id", 5) +        self.guild_id = self.guild_id_patcher.start() + +        self.guild = helpers.MockGuild(id=self.guild_id) +        self.other_guild = helpers.MockGuild(id=0) + +    def tearDown(self): +        self.guild_id_patcher.stop() +      async def test_sync_cog_on_guild_role_create(self):          """A POST request should be sent with the new role's data."""          self.assertTrue(self.cog.on_guild_role_create.__cog_listener__) @@ -142,20 +151,32 @@ class SyncCogListenerTests(SyncCogTestCase):              "permissions": 8,              "position": 23,          } -        role = helpers.MockRole(**role_data) +        role = helpers.MockRole(**role_data, guild=self.guild)          await self.cog.on_guild_role_create(role)          self.bot.api_client.post.assert_called_once_with("bot/roles", json=role_data) +    async def test_sync_cog_on_guild_role_create_ignores_guilds(self): +        """Events from other guilds should be ignored.""" +        role = helpers.MockRole(guild=self.other_guild) +        await self.cog.on_guild_role_create(role) +        self.bot.api_client.post.assert_not_awaited() +      async def test_sync_cog_on_guild_role_delete(self):          """A DELETE request should be sent."""          self.assertTrue(self.cog.on_guild_role_delete.__cog_listener__) -        role = helpers.MockRole(id=99) +        role = helpers.MockRole(id=99, guild=self.guild)          await self.cog.on_guild_role_delete(role)          self.bot.api_client.delete.assert_called_once_with("bot/roles/99") +    async def test_sync_cog_on_guild_role_delete_ignores_guilds(self): +        """Events from other guilds should be ignored.""" +        role = helpers.MockRole(guild=self.other_guild) +        await self.cog.on_guild_role_delete(role) +        self.bot.api_client.delete.assert_not_awaited() +      async def test_sync_cog_on_guild_role_update(self):          """A PUT request should be sent if the colour, name, permissions, or position changes."""          self.assertTrue(self.cog.on_guild_role_update.__cog_listener__) @@ -180,8 +201,8 @@ class SyncCogListenerTests(SyncCogTestCase):                      after_role_data = role_data.copy()                      after_role_data[attribute] = 876 -                    before_role = helpers.MockRole(**role_data) -                    after_role = helpers.MockRole(**after_role_data) +                    before_role = helpers.MockRole(**role_data, guild=self.guild) +                    after_role = helpers.MockRole(**after_role_data, guild=self.guild)                      await self.cog.on_guild_role_update(before_role, after_role) @@ -193,31 +214,43 @@ class SyncCogListenerTests(SyncCogTestCase):                      else:                          self.bot.api_client.put.assert_not_called() +    async def test_sync_cog_on_guild_role_update_ignores_guilds(self): +        """Events from other guilds should be ignored.""" +        role = helpers.MockRole(guild=self.other_guild) +        await self.cog.on_guild_role_update(role, role) +        self.bot.api_client.put.assert_not_awaited() +      async def test_sync_cog_on_member_remove(self): -        """Member should patched to set in_guild as False.""" +        """Member should be patched to set in_guild as False."""          self.assertTrue(self.cog.on_member_remove.__cog_listener__) -        member = helpers.MockMember() +        member = helpers.MockMember(guild=self.guild)          await self.cog.on_member_remove(member)          self.cog.patch_user.assert_called_once_with(              member.id, -            updated_information={"in_guild": False} +            json={"in_guild": False}          ) +    async def test_sync_cog_on_member_remove_ignores_guilds(self): +        """Events from other guilds should be ignored.""" +        member = helpers.MockMember(guild=self.other_guild) +        await self.cog.on_member_remove(member) +        self.cog.patch_user.assert_not_awaited() +      async def test_sync_cog_on_member_update_roles(self):          """Members should be patched if their roles have changed."""          self.assertTrue(self.cog.on_member_update.__cog_listener__)          # Roles are intentionally unsorted.          before_roles = [helpers.MockRole(id=12), helpers.MockRole(id=30), helpers.MockRole(id=20)] -        before_member = helpers.MockMember(roles=before_roles) -        after_member = helpers.MockMember(roles=before_roles[1:]) +        before_member = helpers.MockMember(roles=before_roles, guild=self.guild) +        after_member = helpers.MockMember(roles=before_roles[1:], guild=self.guild)          await self.cog.on_member_update(before_member, after_member)          data = {"roles": sorted(role.id for role in after_member.roles)} -        self.cog.patch_user.assert_called_once_with(after_member.id, updated_information=data) +        self.cog.patch_user.assert_called_once_with(after_member.id, json=data)      async def test_sync_cog_on_member_update_other(self):          """Members should not be patched if other attributes have changed.""" @@ -233,13 +266,19 @@ class SyncCogListenerTests(SyncCogTestCase):              with self.subTest(attribute=attribute):                  self.cog.patch_user.reset_mock() -                before_member = helpers.MockMember(**{attribute: old_value}) -                after_member = helpers.MockMember(**{attribute: new_value}) +                before_member = helpers.MockMember(**{attribute: old_value}, guild=self.guild) +                after_member = helpers.MockMember(**{attribute: new_value}, guild=self.guild)                  await self.cog.on_member_update(before_member, after_member)                  self.cog.patch_user.assert_not_called() +    async def test_sync_cog_on_member_update_ignores_guilds(self): +        """Events from other guilds should be ignored.""" +        member = helpers.MockMember(guild=self.other_guild) +        await self.cog.on_member_update(member, member) +        self.cog.patch_user.assert_not_awaited() +      async def test_sync_cog_on_user_update(self):          """A user should be patched only if the name, discriminator, or avatar changes."""          self.assertTrue(self.cog.on_user_update.__cog_listener__) @@ -272,12 +311,15 @@ class SyncCogListenerTests(SyncCogTestCase):                      # Don't care if *all* keys are present; only the changed one is required                      call_args = self.cog.patch_user.call_args -                    self.assertEqual(call_args[0][0], after_user.id) -                    self.assertIn("updated_information", call_args[1]) +                    self.assertEqual(call_args.args[0], after_user.id) +                    self.assertIn("json", call_args.kwargs) + +                    self.assertIn("ignore_404", call_args.kwargs) +                    self.assertTrue(call_args.kwargs["ignore_404"]) -                    updated_information = call_args[1]["updated_information"] -                    self.assertIn(api_field, updated_information) -                    self.assertEqual(updated_information[api_field], api_value) +                    json = call_args.kwargs["json"] +                    self.assertIn(api_field, json) +                    self.assertEqual(json[api_field], api_value)                  else:                      self.cog.patch_user.assert_not_called() @@ -290,6 +332,7 @@ class SyncCogListenerTests(SyncCogTestCase):          member = helpers.MockMember(              discriminator="1234",              roles=[helpers.MockRole(id=22), helpers.MockRole(id=12)], +            guild=self.guild,          )          data = { @@ -334,6 +377,13 @@ class SyncCogListenerTests(SyncCogTestCase):          self.bot.api_client.post.assert_not_called() +    async def test_sync_cog_on_member_join_ignores_guilds(self): +        """Events from other guilds should be ignored.""" +        member = helpers.MockMember(guild=self.other_guild) +        await self.cog.on_member_join(member) +        self.bot.api_client.post.assert_not_awaited() +        self.bot.api_client.put.assert_not_awaited() +  class SyncCogCommandTests(SyncCogTestCase, CommandTestCase):      """Tests for the commands in the Sync cog.""" diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index f219fc1ba..f50c0492d 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,28 +1,35 @@  import unittest -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import AsyncMock, Mock  from discord import NotFound  from bot.cogs import antimalware -from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES +from bot.constants import Channels, STAFF_ROLES  from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole -MODULE = "bot.cogs.antimalware" - -@patch(f"{MODULE}.AntiMalwareConfig.whitelist", new=[".first", ".second", ".third"])  class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase):      """Test the AntiMalware cog."""      def setUp(self):          """Sets up fresh objects for each test."""          self.bot = MockBot() +        self.bot.filter_list_cache = { +            "FILE_FORMAT.True": { +                ".first": {}, +                ".second": {}, +                ".third": {}, +            } +        }          self.cog = antimalware.AntiMalware(self.bot)          self.message = MockMessage() +        self.message.webhook_id = None +        self.message.author.bot = None +        self.whitelist = [".first", ".second", ".third"]      async def test_message_with_allowed_attachment(self):          """Messages with allowed extensions should not be deleted""" -        attachment = MockAttachment(filename=f"python{AntiMalwareConfig.whitelist[0]}") +        attachment = MockAttachment(filename="python.first")          self.message.attachments = [attachment]          await self.cog.on_message(self.message) @@ -43,6 +50,26 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase):          self.message.delete.assert_not_called() +    async def test_webhook_message_with_illegal_extension(self): +        """A webhook message containing an illegal extension should be ignored.""" +        attachment = MockAttachment(filename="python.disallowed") +        self.message.webhook_id = 697140105563078727 +        self.message.attachments = [attachment] + +        await self.cog.on_message(self.message) + +        self.message.delete.assert_not_called() + +    async def test_bot_message_with_illegal_extension(self): +        """A bot message containing an illegal extension should be ignored.""" +        attachment = MockAttachment(filename="python.disallowed") +        self.message.author.bot = 409107086526644234 +        self.message.attachments = [attachment] + +        await self.cog.on_message(self.message) + +        self.message.delete.assert_not_called() +      async def test_message_with_illegal_extension_gets_deleted(self):          """A message containing an illegal extension should send an embed."""          attachment = MockAttachment(filename="python.disallowed") @@ -93,7 +120,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase):          self.assertEqual(embed.description, antimalware.TXT_EMBED_DESCRIPTION.format.return_value)          antimalware.TXT_EMBED_DESCRIPTION.format.assert_called_with(cmd_channel_mention=cmd_channel.mention) -    async def test_other_disallowed_extention_embed_description(self): +    async def test_other_disallowed_extension_embed_description(self):          """Test the description for a non .py/.txt disallowed extension."""          attachment = MockAttachment(filename="python.disallowed")          self.message.attachments = [attachment] @@ -109,6 +136,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase):          self.assertEqual(embed.description, antimalware.DISALLOWED_EMBED_DESCRIPTION.format.return_value)          antimalware.DISALLOWED_EMBED_DESCRIPTION.format.assert_called_with( +            joined_whitelist=", ".join(self.whitelist),              blocked_extensions_str=".disallowed",              meta_channel_mention=meta_channel.mention          ) @@ -135,7 +163,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase):          """The return value should include all non-whitelisted extensions."""          test_values = (              ([], []), -            (AntiMalwareConfig.whitelist, []), +            (self.whitelist, []),              ([".first"], []),              ([".first", ".disallowed"], [".disallowed"]),              ([".disallowed"], [".disallowed"]), @@ -145,7 +173,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase):          for extensions, expected_disallowed_extensions in test_values:              with self.subTest(extensions=extensions, expected_disallowed_extensions=expected_disallowed_extensions):                  self.message.attachments = [MockAttachment(filename=f"filename{extension}") for extension in extensions] -                disallowed_extensions = self.cog.get_disallowed_extensions(self.message) +                disallowed_extensions = self.cog._get_disallowed_extensions(self.message)                  self.assertCountEqual(disallowed_extensions, expected_disallowed_extensions) diff --git a/tests/bot/cogs/test_cogs.py b/tests/bot/cogs/test_cogs.py index fdda59a8f..30a04422a 100644 --- a/tests/bot/cogs/test_cogs.py +++ b/tests/bot/cogs/test_cogs.py @@ -53,6 +53,7 @@ class CommandNameTests(unittest.TestCase):          """Return a list of all qualified names, including aliases, for the `command`."""          names = [f"{command.full_parent_name} {alias}".strip() for alias in command.aliases]          names.append(command.qualified_name) +        names += getattr(command, "root_aliases", [])          return names diff --git a/tests/bot/cogs/test_duck_pond.py b/tests/bot/cogs/test_duck_pond.py index a8c0107c6..cfe10aebf 100644 --- a/tests/bot/cogs/test_duck_pond.py +++ b/tests/bot/cogs/test_duck_pond.py @@ -129,38 +129,6 @@ class DuckPondTests(base.LoggingTestsMixin, unittest.IsolatedAsyncioTestCase):              ):                  self.assertEqual(expected_return, actual_return) -    def test_send_webhook_correctly_passes_on_arguments(self): -        """The `send_webhook` method should pass the arguments to the webhook correctly.""" -        self.cog.webhook = helpers.MockAsyncWebhook() - -        content = "fake content" -        username = "fake username" -        avatar_url = "fake avatar_url" -        embed = "fake embed" - -        asyncio.run(self.cog.send_webhook(content, username, avatar_url, embed)) - -        self.cog.webhook.send.assert_called_once_with( -            content=content, -            username=username, -            avatar_url=avatar_url, -            embed=embed -        ) - -    def test_send_webhook_logs_when_sending_message_fails(self): -        """The `send_webhook` method should catch a `discord.HTTPException` and log accordingly.""" -        self.cog.webhook = helpers.MockAsyncWebhook() -        self.cog.webhook.send.side_effect = discord.HTTPException(response=MagicMock(), message="Something failed.") - -        log = logging.getLogger('bot.cogs.duck_pond') -        with self.assertLogs(logger=log, level=logging.ERROR) as log_watcher: -            asyncio.run(self.cog.send_webhook()) - -        self.assertEqual(len(log_watcher.records), 1) - -        record = log_watcher.records[0] -        self.assertEqual(record.levelno, logging.ERROR) -      def _get_reaction(          self,          emoji: typing.Union[str, helpers.MockEmoji], @@ -280,16 +248,20 @@ class DuckPondTests(base.LoggingTestsMixin, unittest.IsolatedAsyncioTestCase):      async def test_relay_message_correctly_relays_content_and_attachments(self):          """The `relay_message` method should correctly relay message content and attachments.""" -        send_webhook_path = f"{MODULE_PATH}.DuckPond.send_webhook" +        send_webhook_path = f"{MODULE_PATH}.send_webhook"          send_attachments_path = f"{MODULE_PATH}.send_attachments" +        author = MagicMock( +            display_name="x", +            avatar_url="https://" +        )          self.cog.webhook = helpers.MockAsyncWebhook()          test_values = ( -            (helpers.MockMessage(clean_content="", attachments=[]), False, False), -            (helpers.MockMessage(clean_content="message", attachments=[]), True, False), -            (helpers.MockMessage(clean_content="", attachments=["attachment"]), False, True), -            (helpers.MockMessage(clean_content="message", attachments=["attachment"]), True, True), +            (helpers.MockMessage(author=author, clean_content="", attachments=[]), False, False), +            (helpers.MockMessage(author=author, clean_content="message", attachments=[]), True, False), +            (helpers.MockMessage(author=author, clean_content="", attachments=["attachment"]), False, True), +            (helpers.MockMessage(author=author, clean_content="message", attachments=["attachment"]), True, True),          )          for message, expect_webhook_call, expect_attachment_call in test_values: @@ -314,14 +286,14 @@ class DuckPondTests(base.LoggingTestsMixin, unittest.IsolatedAsyncioTestCase):          for side_effect in side_effects:  # pragma: no cover              send_attachments.side_effect = side_effect -            with patch(f"{MODULE_PATH}.DuckPond.send_webhook", new_callable=AsyncMock) as send_webhook: +            with patch(f"{MODULE_PATH}.send_webhook", new_callable=AsyncMock) as send_webhook:                  with self.subTest(side_effect=type(side_effect).__name__):                      with self.assertNotLogs(logger=log, level=logging.ERROR):                          await self.cog.relay_message(message)                      self.assertEqual(send_webhook.call_count, 2) -    @patch(f"{MODULE_PATH}.DuckPond.send_webhook", new_callable=AsyncMock) +    @patch(f"{MODULE_PATH}.send_webhook", new_callable=AsyncMock)      @patch(f"{MODULE_PATH}.send_attachments", new_callable=AsyncMock)      async def test_relay_message_handles_attachment_http_error(self, send_attachments, send_webhook):          """The `relay_message` method should handle irretrievable attachments.""" @@ -337,6 +309,7 @@ class DuckPondTests(base.LoggingTestsMixin, unittest.IsolatedAsyncioTestCase):                  await self.cog.relay_message(message)              send_webhook.assert_called_once_with( +                webhook=self.cog.webhook,                  content=message.clean_content,                  username=message.author.display_name,                  avatar_url=message.author.avatar_url diff --git a/tests/bot/cogs/test_information.py b/tests/bot/cogs/test_information.py index 79c0e0ad3..77b0ddf17 100644 --- a/tests/bot/cogs/test_information.py +++ b/tests/bot/cogs/test_information.py @@ -215,10 +215,10 @@ class UserInfractionHelperMethodTests(unittest.TestCase):              with self.subTest(method=method, api_response=api_response, expected_lines=expected_lines):                  self.bot.api_client.get.return_value = api_response -                expected_output = "\n".join(default_header + expected_lines) +                expected_output = "\n".join(expected_lines)                  actual_output = asyncio.run(method(self.member)) -                self.assertEqual(expected_output, actual_output) +                self.assertEqual((default_header, expected_output), actual_output)      def test_basic_user_infraction_counts_returns_correct_strings(self):          """The method should correctly list both the total and active number of non-hidden infractions.""" @@ -249,7 +249,7 @@ class UserInfractionHelperMethodTests(unittest.TestCase):              },          ) -        header = ["**Infractions**"] +        header = "Infractions"          self._method_subtests(self.cog.basic_user_infraction_counts, test_values, header) @@ -258,7 +258,7 @@ class UserInfractionHelperMethodTests(unittest.TestCase):          test_values = (              {                  "api response": [], -                "expected_lines": ["This user has never received an infraction."], +                "expected_lines": ["No infractions"],              },              # Shows non-hidden inactive infraction as expected              { @@ -304,7 +304,7 @@ class UserInfractionHelperMethodTests(unittest.TestCase):              },          ) -        header = ["**Infractions**"] +        header = "Infractions"          self._method_subtests(self.cog.expanded_user_infraction_counts, test_values, header) @@ -313,15 +313,15 @@ class UserInfractionHelperMethodTests(unittest.TestCase):          test_values = (              {                  "api response": [], -                "expected_lines": ["This user has never been nominated."], +                "expected_lines": ["No nominations"],              },              {                  "api response": [{'active': True}], -                "expected_lines": ["This user is **currently** nominated (1 nomination in total)."], +                "expected_lines": ["This user is **currently** nominated", "(1 nomination in total)"],              },              {                  "api response": [{'active': True}, {'active': False}], -                "expected_lines": ["This user is **currently** nominated (2 nominations in total)."], +                "expected_lines": ["This user is **currently** nominated", "(2 nominations in total)"],              },              {                  "api response": [{'active': False}], @@ -334,7 +334,7 @@ class UserInfractionHelperMethodTests(unittest.TestCase):          ) -        header = ["**Nominations**"] +        header = "Nominations"          self._method_subtests(self.cog.user_nomination_counts, test_values, header) @@ -350,7 +350,10 @@ class UserEmbedTests(unittest.TestCase):          self.bot.api_client.get = unittest.mock.AsyncMock()          self.cog = information.Information(self.bot) -    @unittest.mock.patch(f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value="")) +    @unittest.mock.patch( +        f"{COG_PATH}.basic_user_infraction_counts", +        new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) +    )      def test_create_user_embed_uses_string_representation_of_user_in_title_if_nick_is_not_available(self):          """The embed should use the string representation of the user if they don't have a nick."""          ctx = helpers.MockContext(channel=helpers.MockTextChannel(id=1)) @@ -362,7 +365,10 @@ class UserEmbedTests(unittest.TestCase):          self.assertEqual(embed.title, "Mr. Hemlock") -    @unittest.mock.patch(f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value="")) +    @unittest.mock.patch( +        f"{COG_PATH}.basic_user_infraction_counts", +        new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) +    )      def test_create_user_embed_uses_nick_in_title_if_available(self):          """The embed should use the nick if it's available."""          ctx = helpers.MockContext(channel=helpers.MockTextChannel(id=1)) @@ -374,7 +380,10 @@ class UserEmbedTests(unittest.TestCase):          self.assertEqual(embed.title, "Cat lover (Mr. Hemlock)") -    @unittest.mock.patch(f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value="")) +    @unittest.mock.patch( +        f"{COG_PATH}.basic_user_infraction_counts", +        new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) +    )      def test_create_user_embed_ignores_everyone_role(self):          """Created `!user` embeds should not contain mention of the @everyone-role."""          ctx = helpers.MockContext(channel=helpers.MockTextChannel(id=1)) @@ -386,8 +395,8 @@ class UserEmbedTests(unittest.TestCase):          embed = asyncio.run(self.cog.create_user_embed(ctx, user)) -        self.assertIn("&Admins", embed.description) -        self.assertNotIn("&Everyone", embed.description) +        self.assertIn("&Admins", embed.fields[1].value) +        self.assertNotIn("&Everyone", embed.fields[1].value)      @unittest.mock.patch(f"{COG_PATH}.expanded_user_infraction_counts", new_callable=unittest.mock.AsyncMock)      @unittest.mock.patch(f"{COG_PATH}.user_nomination_counts", new_callable=unittest.mock.AsyncMock) @@ -398,8 +407,8 @@ class UserEmbedTests(unittest.TestCase):          moderators_role = helpers.MockRole(name='Moderators')          moderators_role.colour = 100 -        infraction_counts.return_value = "expanded infractions info" -        nomination_counts.return_value = "nomination info" +        infraction_counts.return_value = ("Infractions", "expanded infractions info") +        nomination_counts.return_value = ("Nominations", "nomination info")          user = helpers.MockMember(id=314, roles=[moderators_role], top_role=moderators_role)          embed = asyncio.run(self.cog.create_user_embed(ctx, user)) @@ -409,20 +418,19 @@ class UserEmbedTests(unittest.TestCase):          self.assertEqual(              textwrap.dedent(f""" -                **User Information**                  Created: {"1 year ago"}                  Profile: {user.mention}                  ID: {user.id} +            """).strip(), +            embed.fields[0].value +        ) -                **Member Information** +        self.assertEqual( +            textwrap.dedent(f"""                  Joined: {"1 year ago"}                  Roles: &Moderators - -                expanded infractions info - -                nomination info              """).strip(), -            embed.description +            embed.fields[1].value          )      @unittest.mock.patch(f"{COG_PATH}.basic_user_infraction_counts", new_callable=unittest.mock.AsyncMock) @@ -433,7 +441,7 @@ class UserEmbedTests(unittest.TestCase):          moderators_role = helpers.MockRole(name='Moderators')          moderators_role.colour = 100 -        infraction_counts.return_value = "basic infractions info" +        infraction_counts.return_value = ("Infractions", "basic infractions info")          user = helpers.MockMember(id=314, roles=[moderators_role], top_role=moderators_role)          embed = asyncio.run(self.cog.create_user_embed(ctx, user)) @@ -442,21 +450,30 @@ class UserEmbedTests(unittest.TestCase):          self.assertEqual(              textwrap.dedent(f""" -                **User Information**                  Created: {"1 year ago"}                  Profile: {user.mention}                  ID: {user.id} +            """).strip(), +            embed.fields[0].value +        ) -                **Member Information** +        self.assertEqual( +            textwrap.dedent(f"""                  Joined: {"1 year ago"}                  Roles: &Moderators - -                basic infractions info              """).strip(), -            embed.description +            embed.fields[1].value +        ) + +        self.assertEqual( +            "basic infractions info", +            embed.fields[3].value          ) -    @unittest.mock.patch(f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value="")) +    @unittest.mock.patch( +        f"{COG_PATH}.basic_user_infraction_counts", +        new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) +    )      def test_create_user_embed_uses_top_role_colour_when_user_has_roles(self):          """The embed should be created with the colour of the top role, if a top role is available."""          ctx = helpers.MockContext() @@ -469,7 +486,10 @@ class UserEmbedTests(unittest.TestCase):          self.assertEqual(embed.colour, discord.Colour(moderators_role.colour)) -    @unittest.mock.patch(f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value="")) +    @unittest.mock.patch( +        f"{COG_PATH}.basic_user_infraction_counts", +        new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) +    )      def test_create_user_embed_uses_blurple_colour_when_user_has_no_roles(self):          """The embed should be created with a blurple colour if the user has no assigned roles."""          ctx = helpers.MockContext() @@ -479,7 +499,10 @@ class UserEmbedTests(unittest.TestCase):          self.assertEqual(embed.colour, discord.Colour.blurple()) -    @unittest.mock.patch(f"{COG_PATH}.basic_user_infraction_counts", new=unittest.mock.AsyncMock(return_value="")) +    @unittest.mock.patch( +        f"{COG_PATH}.basic_user_infraction_counts", +        new=unittest.mock.AsyncMock(return_value=("Infractions", "basic infractions")) +    )      def test_create_user_embed_uses_png_format_of_user_avatar_as_thumbnail(self):          """The embed thumbnail should be set to the user's avatar in `png` format."""          ctx = helpers.MockContext() diff --git a/tests/bot/cogs/test_jams.py b/tests/bot/cogs/test_jams.py new file mode 100644 index 000000000..b4ad8535f --- /dev/null +++ b/tests/bot/cogs/test_jams.py @@ -0,0 +1,173 @@ +import unittest +from unittest.mock import AsyncMock, MagicMock, create_autospec + +from discord import CategoryChannel + +from bot.cogs import jams +from bot.constants import Roles +from tests.helpers import MockBot, MockContext, MockGuild, MockMember, MockRole, MockTextChannel + + +def get_mock_category(channel_count: int, name: str) -> CategoryChannel: +    """Return a mocked code jam category.""" +    category = create_autospec(CategoryChannel, spec_set=True, instance=True) +    category.name = name +    category.channels = [MockTextChannel() for _ in range(channel_count)] + +    return category + + +class JamCreateTeamTests(unittest.IsolatedAsyncioTestCase): +    """Tests for `createteam` command.""" + +    def setUp(self): +        self.bot = MockBot() +        self.admin_role = MockRole(name="Admins", id=Roles.admins) +        self.command_user = MockMember([self.admin_role]) +        self.guild = MockGuild([self.admin_role]) +        self.ctx = MockContext(bot=self.bot, author=self.command_user, guild=self.guild) +        self.cog = jams.CodeJams(self.bot) + +    async def test_too_small_amount_of_team_members_passed(self): +        """Should `ctx.send` and exit early when too small amount of members.""" +        for case in (1, 2): +            with self.subTest(amount_of_members=case): +                self.cog.create_channels = AsyncMock() +                self.cog.add_roles = AsyncMock() + +                self.ctx.reset_mock() +                members = (MockMember() for _ in range(case)) +                await self.cog.createteam(self.cog, self.ctx, "foo", members) + +                self.ctx.send.assert_awaited_once() +                self.cog.create_channels.assert_not_awaited() +                self.cog.add_roles.assert_not_awaited() + +    async def test_duplicate_members_provided(self): +        """Should `ctx.send` and exit early because duplicate members provided and total there is only 1 member.""" +        self.cog.create_channels = AsyncMock() +        self.cog.add_roles = AsyncMock() + +        member = MockMember() +        await self.cog.createteam(self.cog, self.ctx, "foo", (member for _ in range(5))) + +        self.ctx.send.assert_awaited_once() +        self.cog.create_channels.assert_not_awaited() +        self.cog.add_roles.assert_not_awaited() + +    async def test_result_sending(self): +        """Should call `ctx.send` when everything goes right.""" +        self.cog.create_channels = AsyncMock() +        self.cog.add_roles = AsyncMock() + +        members = [MockMember() for _ in range(5)] +        await self.cog.createteam(self.cog, self.ctx, "foo", members) + +        self.cog.create_channels.assert_awaited_once() +        self.cog.add_roles.assert_awaited_once() +        self.ctx.send.assert_awaited_once() + +    async def test_category_doesnt_exist(self): +        """Should create a new code jam category.""" +        subtests = ( +            [], +            [get_mock_category(jams.MAX_CHANNELS - 1, jams.CATEGORY_NAME)], +            [get_mock_category(jams.MAX_CHANNELS - 2, "other")], +        ) + +        for categories in subtests: +            self.guild.reset_mock() +            self.guild.categories = categories + +            with self.subTest(categories=categories): +                actual_category = await self.cog.get_category(self.guild) + +                self.guild.create_category_channel.assert_awaited_once() +                category_overwrites = self.guild.create_category_channel.call_args[1]["overwrites"] + +                self.assertFalse(category_overwrites[self.guild.default_role].read_messages) +                self.assertTrue(category_overwrites[self.guild.me].read_messages) +                self.assertEqual(self.guild.create_category_channel.return_value, actual_category) + +    async def test_category_channel_exist(self): +        """Should not try to create category channel.""" +        expected_category = get_mock_category(jams.MAX_CHANNELS - 2, jams.CATEGORY_NAME) +        self.guild.categories = [ +            get_mock_category(jams.MAX_CHANNELS - 2, "other"), +            expected_category, +            get_mock_category(0, jams.CATEGORY_NAME), +        ] + +        actual_category = await self.cog.get_category(self.guild) +        self.assertEqual(expected_category, actual_category) + +    async def test_channel_overwrites(self): +        """Should have correct permission overwrites for users and roles.""" +        leader = MockMember() +        members = [leader] + [MockMember() for _ in range(4)] +        overwrites = self.cog.get_overwrites(members, self.guild) + +        # Leader permission overwrites +        self.assertTrue(overwrites[leader].manage_messages) +        self.assertTrue(overwrites[leader].read_messages) +        self.assertTrue(overwrites[leader].manage_webhooks) +        self.assertTrue(overwrites[leader].connect) + +        # Other members permission overwrites +        for member in members[1:]: +            self.assertTrue(overwrites[member].read_messages) +            self.assertTrue(overwrites[member].connect) + +        # Everyone and verified role overwrite +        self.assertFalse(overwrites[self.guild.default_role].read_messages) +        self.assertFalse(overwrites[self.guild.default_role].connect) +        self.assertFalse(overwrites[self.guild.get_role(Roles.verified)].read_messages) +        self.assertFalse(overwrites[self.guild.get_role(Roles.verified)].connect) + +    async def test_team_channels_creation(self): +        """Should create new voice and text channel for team.""" +        members = [MockMember() for _ in range(5)] + +        self.cog.get_overwrites = MagicMock() +        self.cog.get_category = AsyncMock() +        self.ctx.guild.create_text_channel.return_value = MockTextChannel(mention="foobar-channel") +        actual = await self.cog.create_channels(self.guild, "my-team", members) + +        self.assertEqual("foobar-channel", actual) +        self.cog.get_overwrites.assert_called_once_with(members, self.guild) +        self.cog.get_category.assert_awaited_once_with(self.guild) + +        self.guild.create_text_channel.assert_awaited_once_with( +            "my-team", +            overwrites=self.cog.get_overwrites.return_value, +            category=self.cog.get_category.return_value +        ) +        self.guild.create_voice_channel.assert_awaited_once_with( +            "My Team", +            overwrites=self.cog.get_overwrites.return_value, +            category=self.cog.get_category.return_value +        ) + +    async def test_jam_roles_adding(self): +        """Should add team leader role to leader and jam role to every team member.""" +        leader_role = MockRole(name="Team Leader") +        jam_role = MockRole(name="Jammer") +        self.guild.get_role.side_effect = [leader_role, jam_role] + +        leader = MockMember() +        members = [leader] + [MockMember() for _ in range(4)] +        await self.cog.add_roles(self.guild, members) + +        leader.add_roles.assert_any_await(leader_role) +        for member in members: +            member.add_roles.assert_any_await(jam_role) + + +class CodeJamSetup(unittest.TestCase): +    """Test for `setup` function of `CodeJam` cog.""" + +    def test_setup(self): +        """Should call `bot.add_cog`.""" +        bot = MockBot() +        jams.setup(bot) +        bot.add_cog.assert_called_once() diff --git a/tests/bot/cogs/test_logging.py b/tests/bot/cogs/test_logging.py new file mode 100644 index 000000000..8a18fdcd6 --- /dev/null +++ b/tests/bot/cogs/test_logging.py @@ -0,0 +1,32 @@ +import unittest +from unittest.mock import patch + +from bot import constants +from bot.cogs.logging import Logging +from tests.helpers import MockBot, MockTextChannel + + +class LoggingTests(unittest.IsolatedAsyncioTestCase): +    """Test cases for connected login.""" + +    def setUp(self): +        self.bot = MockBot() +        self.cog = Logging(self.bot) +        self.dev_log = MockTextChannel(id=1234, name="dev-log") + +    @patch("bot.cogs.logging.DEBUG_MODE", False) +    async def test_debug_mode_false(self): +        """Should send connected message to dev-log.""" +        self.bot.get_channel.return_value = self.dev_log + +        await self.cog.startup_greeting() +        self.bot.wait_until_guild_available.assert_awaited_once_with() +        self.bot.get_channel.assert_called_once_with(constants.Channels.dev_log) +        self.dev_log.send.assert_awaited_once() + +    @patch("bot.cogs.logging.DEBUG_MODE", True) +    async def test_debug_mode_true(self): +        """Should not send anything to dev-log.""" +        await self.cog.startup_greeting() +        self.bot.wait_until_guild_available.assert_awaited_once_with() +        self.bot.get_channel.assert_not_called() diff --git a/tests/bot/cogs/test_slowmode.py b/tests/bot/cogs/test_slowmode.py new file mode 100644 index 000000000..f442814c8 --- /dev/null +++ b/tests/bot/cogs/test_slowmode.py @@ -0,0 +1,111 @@ +import unittest +from unittest import mock + +from dateutil.relativedelta import relativedelta + +from bot.cogs.moderation.slowmode import Slowmode +from bot.constants import Emojis +from tests.helpers import MockBot, MockContext, MockTextChannel + + +class SlowmodeTests(unittest.IsolatedAsyncioTestCase): + +    def setUp(self) -> None: +        self.bot = MockBot() +        self.cog = Slowmode(self.bot) +        self.ctx = MockContext() + +    async def test_get_slowmode_no_channel(self) -> None: +        """Get slowmode without a given channel.""" +        self.ctx.channel = MockTextChannel(name='python-general', slowmode_delay=5) + +        await self.cog.get_slowmode(self.cog, self.ctx, None) +        self.ctx.send.assert_called_once_with("The slowmode delay for #python-general is 5 seconds.") + +    async def test_get_slowmode_with_channel(self) -> None: +        """Get slowmode with a given channel.""" +        text_channel = MockTextChannel(name='python-language', slowmode_delay=2) + +        await self.cog.get_slowmode(self.cog, self.ctx, text_channel) +        self.ctx.send.assert_called_once_with('The slowmode delay for #python-language is 2 seconds.') + +    async def test_set_slowmode_no_channel(self) -> None: +        """Set slowmode without a given channel.""" +        test_cases = ( +            ('helpers', 23, True, f'{Emojis.check_mark} The slowmode delay for #helpers is now 23 seconds.'), +            ('mods', 76526, False, f'{Emojis.cross_mark} The slowmode delay must be between 0 and 6 hours.'), +            ('admins', 97, True, f'{Emojis.check_mark} The slowmode delay for #admins is now 1 minute and 37 seconds.') +        ) + +        for channel_name, seconds, edited, result_msg in test_cases: +            with self.subTest( +                channel_mention=channel_name, +                seconds=seconds, +                edited=edited, +                result_msg=result_msg +            ): +                self.ctx.channel = MockTextChannel(name=channel_name) + +                await self.cog.set_slowmode(self.cog, self.ctx, None, relativedelta(seconds=seconds)) + +                if edited: +                    self.ctx.channel.edit.assert_awaited_once_with(slowmode_delay=float(seconds)) +                else: +                    self.ctx.channel.edit.assert_not_called() + +                self.ctx.send.assert_called_once_with(result_msg) + +            self.ctx.reset_mock() + +    async def test_set_slowmode_with_channel(self) -> None: +        """Set slowmode with a given channel.""" +        test_cases = ( +            ('bot-commands', 12, True, f'{Emojis.check_mark} The slowmode delay for #bot-commands is now 12 seconds.'), +            ('mod-spam', 21, True, f'{Emojis.check_mark} The slowmode delay for #mod-spam is now 21 seconds.'), +            ('admin-spam', 4323598, False, f'{Emojis.cross_mark} The slowmode delay must be between 0 and 6 hours.') +        ) + +        for channel_name, seconds, edited, result_msg in test_cases: +            with self.subTest( +                channel_mention=channel_name, +                seconds=seconds, +                edited=edited, +                result_msg=result_msg +            ): +                text_channel = MockTextChannel(name=channel_name) + +                await self.cog.set_slowmode(self.cog, self.ctx, text_channel, relativedelta(seconds=seconds)) + +                if edited: +                    text_channel.edit.assert_awaited_once_with(slowmode_delay=float(seconds)) +                else: +                    text_channel.edit.assert_not_called() + +                self.ctx.send.assert_called_once_with(result_msg) + +            self.ctx.reset_mock() + +    async def test_reset_slowmode_no_channel(self) -> None: +        """Reset slowmode without a given channel.""" +        self.ctx.channel = MockTextChannel(name='careers', slowmode_delay=6) + +        await self.cog.reset_slowmode(self.cog, self.ctx, None) +        self.ctx.send.assert_called_once_with( +            f'{Emojis.check_mark} The slowmode delay for #careers has been reset to 0 seconds.' +        ) + +    async def test_reset_slowmode_with_channel(self) -> None: +        """Reset slowmode with a given channel.""" +        text_channel = MockTextChannel(name='meta', slowmode_delay=1) + +        await self.cog.reset_slowmode(self.cog, self.ctx, text_channel) +        self.ctx.send.assert_called_once_with( +            f'{Emojis.check_mark} The slowmode delay for #meta has been reset to 0 seconds.' +        ) + +    @mock.patch("bot.cogs.moderation.slowmode.with_role_check") +    @mock.patch("bot.cogs.moderation.slowmode.MODERATION_ROLES", new=(1, 2, 3)) +    def test_cog_check(self, role_check): +        """Role check is called with `MODERATION_ROLES`""" +        self.cog.cog_check(self.ctx) +        role_check.assert_called_once_with(self.ctx, *(1, 2, 3)) diff --git a/tests/bot/cogs/test_snekbox.py b/tests/bot/cogs/test_snekbox.py index d37a9d07d..f22952931 100644 --- a/tests/bot/cogs/test_snekbox.py +++ b/tests/bot/cogs/test_snekbox.py @@ -203,9 +203,13 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase):          self.cog.get_status_emoji = MagicMock(return_value=':yay!:')          self.cog.format_output = AsyncMock(return_value=('[No output]', None)) +        mocked_filter_cog = MagicMock() +        mocked_filter_cog.filter_eval = AsyncMock(return_value=False) +        self.bot.get_cog.return_value = mocked_filter_cog +          await self.cog.send_eval(ctx, 'MyAwesomeCode')          ctx.send.assert_called_once_with( -            '@LemonLemonishBeard#0042 :yay!: Return code 0.\n\n```py\n[No output]\n```' +            '@LemonLemonishBeard#0042 :yay!: Return code 0.\n\n```\n[No output]\n```'          )          self.cog.post_eval.assert_called_once_with('MyAwesomeCode')          self.cog.get_status_emoji.assert_called_once_with({'stdout': '', 'returncode': 0}) @@ -224,10 +228,14 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase):          self.cog.get_status_emoji = MagicMock(return_value=':yay!:')          self.cog.format_output = AsyncMock(return_value=('Way too long beard', 'lookatmybeard.com')) +        mocked_filter_cog = MagicMock() +        mocked_filter_cog.filter_eval = AsyncMock(return_value=False) +        self.bot.get_cog.return_value = mocked_filter_cog +          await self.cog.send_eval(ctx, 'MyAwesomeCode')          ctx.send.assert_called_once_with(              '@LemonLemonishBeard#0042 :yay!: Return code 0.' -            '\n\n```py\nWay too long beard\n```\nFull output: lookatmybeard.com' +            '\n\n```\nWay too long beard\n```\nFull output: lookatmybeard.com'          )          self.cog.post_eval.assert_called_once_with('MyAwesomeCode')          self.cog.get_status_emoji.assert_called_once_with({'stdout': 'Way too long beard', 'returncode': 0}) @@ -245,9 +253,13 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase):          self.cog.get_status_emoji = MagicMock(return_value=':nope!:')          self.cog.format_output = AsyncMock()  # This function isn't called +        mocked_filter_cog = MagicMock() +        mocked_filter_cog.filter_eval = AsyncMock(return_value=False) +        self.bot.get_cog.return_value = mocked_filter_cog +          await self.cog.send_eval(ctx, 'MyAwesomeCode')          ctx.send.assert_called_once_with( -            '@LemonLemonishBeard#0042 :nope!: Return code 127.\n\n```py\nBeard got stuck in the eval\n```' +            '@LemonLemonishBeard#0042 :nope!: Return code 127.\n\n```\nBeard got stuck in the eval\n```'          )          self.cog.post_eval.assert_called_once_with('MyAwesomeCode')          self.cog.get_status_emoji.assert_called_once_with({'stdout': 'ERROR', 'returncode': 127}) diff --git a/tests/bot/test_pagination.py b/tests/bot/test_pagination.py index 0a734b505..630f2516d 100644 --- a/tests/bot/test_pagination.py +++ b/tests/bot/test_pagination.py @@ -8,29 +8,39 @@ class LinePaginatorTests(TestCase):      def setUp(self):          """Create a paginator for the test method.""" -        self.paginator = pagination.LinePaginator(prefix='', suffix='', max_size=30) - -    def test_add_line_raises_on_too_long_lines(self): -        """`add_line` should raise a `RuntimeError` for too long lines.""" -        message = f"Line exceeds maximum page size {self.paginator.max_size - 2}" -        with self.assertRaises(RuntimeError, msg=message): -            self.paginator.add_line('x' * self.paginator.max_size) +        self.paginator = pagination.LinePaginator(prefix='', suffix='', max_size=30, +                                                  scale_to_size=50)      def test_add_line_works_on_small_lines(self):          """`add_line` should allow small lines to be added."""          self.paginator.add_line('x' * (self.paginator.max_size - 3)) - - -class ImagePaginatorTests(TestCase): -    """Tests functionality of the `ImagePaginator`.""" - -    def setUp(self): -        """Create a paginator for the test method.""" -        self.paginator = pagination.ImagePaginator() - -    def test_add_image_appends_image(self): -        """`add_image` appends the image to the image list.""" -        image = 'lemon' -        self.paginator.add_image(image) - -        assert self.paginator.images == [image] +        # Note that the page isn't added to _pages until it's full. +        self.assertEqual(len(self.paginator._pages), 0) + +    def test_add_line_works_on_long_lines(self): +        """After additional lines after `max_size` is exceeded should go on the next page.""" +        self.paginator.add_line('x' * self.paginator.max_size) +        self.assertEqual(len(self.paginator._pages), 0) + +        # Any additional lines should start a new page after `max_size` is exceeded. +        self.paginator.add_line('x') +        self.assertEqual(len(self.paginator._pages), 1) + +    def test_add_line_continuation(self): +        """When `scale_to_size` is exceeded, remaining words should be split onto the next page.""" +        self.paginator.add_line('zyz ' * (self.paginator.scale_to_size//4 + 1)) +        self.assertEqual(len(self.paginator._pages), 1) + +    def test_add_line_no_continuation(self): +        """If adding a new line to an existing page would exceed `max_size`, it should start a new +        page rather than using continuation. +        """ +        self.paginator.add_line('z' * (self.paginator.max_size - 3)) +        self.paginator.add_line('z') +        self.assertEqual(len(self.paginator._pages), 1) + +    def test_add_line_truncates_very_long_words(self): +        """`add_line` should truncate if a single long word exceeds `scale_to_size`.""" +        self.paginator.add_line('x' * (self.paginator.scale_to_size + 1)) +        # Note: item at index 1 is the truncated line, index 0 is prefix +        self.assertEqual(self.paginator._current_page[1], 'x' * self.paginator.scale_to_size) | 
