diff options
Diffstat (limited to '')
| -rw-r--r-- | Pipfile | 2 | ||||
| -rw-r--r-- | Pipfile.lock | 362 | ||||
| -rw-r--r-- | bot/__main__.py | 8 | ||||
| -rw-r--r-- | bot/bot.py | 107 | ||||
| -rw-r--r-- | bot/cogs/antimalware.py | 24 | ||||
| -rw-r--r-- | bot/cogs/error_handler.py | 32 | ||||
| -rw-r--r-- | bot/cogs/filter_lists.py | 273 | ||||
| -rw-r--r-- | bot/cogs/filtering.py | 96 | ||||
| -rw-r--r-- | bot/cogs/moderation/scheduler.py | 4 | ||||
| -rw-r--r-- | bot/cogs/moderation/silence.py | 3 | ||||
| -rw-r--r-- | bot/cogs/off_topic_names.py | 31 | ||||
| -rw-r--r-- | bot/cogs/reminders.py | 4 | ||||
| -rw-r--r-- | bot/cogs/source.py | 22 | ||||
| -rw-r--r-- | bot/constants.py | 10 | ||||
| -rw-r--r-- | bot/converters.py | 115 | ||||
| -rw-r--r-- | bot/utils/redis_cache.py | 17 | ||||
| -rw-r--r-- | bot/utils/regex.py | 12 | ||||
| -rw-r--r-- | config-default.yml | 134 | ||||
| -rw-r--r-- | tests/bot/cogs/test_antimalware.py | 24 |
19 files changed, 803 insertions, 477 deletions
@@ -28,7 +28,7 @@ statsd = "~=3.3" [dev-packages] coverage = "~=5.0" -flake8 = "~=3.7" +flake8 = "~=3.8" flake8-annotations = "~=2.0" flake8-bugbear = "~=20.1" flake8-docstrings = "~=1.4" diff --git a/Pipfile.lock b/Pipfile.lock index 4b9d092d4..c8cd96d3d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8a53baefbbd2a0f3fbaf831f028b23d257a5e28b5efa1260661d74604f4113b8" + "sha256": "eab4852974d26bd2c10362540c3e01d34af62446cb4e1915ec9a0bf2bddf4d94" }, "pipfile-spec": 6, "requires": { @@ -115,36 +115,36 @@ }, "cffi": { "hashes": [ - "sha256:001bf3242a1bb04d985d63e138230802c6c8d4db3668fb545fb5005ddf5bb5ff", - "sha256:00789914be39dffba161cfc5be31b55775de5ba2235fe49aa28c148236c4e06b", - "sha256:028a579fc9aed3af38f4892bdcc7390508adabc30c6af4a6e4f611b0c680e6ac", - "sha256:14491a910663bf9f13ddf2bc8f60562d6bc5315c1f09c704937ef17293fb85b0", - "sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384", - "sha256:2089ed025da3919d2e75a4d963d008330c96751127dd6f73c8dc0c65041b4c26", - "sha256:2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6", - "sha256:337d448e5a725bba2d8293c48d9353fc68d0e9e4088d62a9571def317797522b", - "sha256:399aed636c7d3749bbed55bc907c3288cb43c65c4389964ad5ff849b6370603e", - "sha256:3b911c2dbd4f423b4c4fcca138cadde747abdb20d196c4a48708b8a2d32b16dd", - "sha256:3d311bcc4a41408cf5854f06ef2c5cab88f9fded37a3b95936c9879c1640d4c2", - "sha256:62ae9af2d069ea2698bf536dcfe1e4eed9090211dbaafeeedf5cb6c41b352f66", - "sha256:66e41db66b47d0d8672d8ed2708ba91b2f2524ece3dee48b5dfb36be8c2f21dc", - "sha256:675686925a9fb403edba0114db74e741d8181683dcf216be697d208857e04ca8", - "sha256:7e63cbcf2429a8dbfe48dcc2322d5f2220b77b2e17b7ba023d6166d84655da55", - "sha256:8a6c688fefb4e1cd56feb6c511984a6c4f7ec7d2a1ff31a10254f3c817054ae4", - "sha256:8c0ffc886aea5df6a1762d0019e9cb05f825d0eec1f520c51be9d198701daee5", - "sha256:95cd16d3dee553f882540c1ffe331d085c9e629499ceadfbda4d4fde635f4b7d", - "sha256:99f748a7e71ff382613b4e1acc0ac83bf7ad167fb3802e35e90d9763daba4d78", - "sha256:b8c78301cefcf5fd914aad35d3c04c2b21ce8629b5e4f4e45ae6812e461910fa", - "sha256:c420917b188a5582a56d8b93bdd8e0f6eca08c84ff623a4c16e809152cd35793", - "sha256:c43866529f2f06fe0edc6246eb4faa34f03fe88b64a0a9a942561c8e22f4b71f", - "sha256:cab50b8c2250b46fe738c77dbd25ce017d5e6fb35d3407606e7a4180656a5a6a", - "sha256:cef128cb4d5e0b3493f058f10ce32365972c554572ff821e175dbc6f8ff6924f", - "sha256:cf16e3cf6c0a5fdd9bc10c21687e19d29ad1fe863372b5543deaec1039581a30", - "sha256:e56c744aa6ff427a607763346e4170629caf7e48ead6921745986db3692f987f", - "sha256:e577934fc5f8779c554639376beeaa5657d54349096ef24abe8c74c5d9c117c3", - "sha256:f2b0fa0c01d8a0c7483afd9f31d7ecf2d71760ca24499c8697aeb5ca37dc090c" - ], - "version": "==1.14.0" + "sha256:267adcf6e68d77ba154334a3e4fc921b8e63cbb38ca00d33d40655d4228502bc", + "sha256:26f33e8f6a70c255767e3c3f957ccafc7f1f706b966e110b855bfe944511f1f9", + "sha256:3cd2c044517f38d1b577f05927fb9729d3396f1d44d0c659a445599e79519792", + "sha256:4a03416915b82b81af5502459a8a9dd62a3c299b295dcdf470877cb948d655f2", + "sha256:4ce1e995aeecf7cc32380bc11598bfdfa017d592259d5da00fc7ded11e61d022", + "sha256:4f53e4128c81ca3212ff4cf097c797ab44646a40b42ec02a891155cd7a2ba4d8", + "sha256:4fa72a52a906425416f41738728268072d5acfd48cbe7796af07a923236bcf96", + "sha256:66dd45eb9530e3dde8f7c009f84568bc7cac489b93d04ac86e3111fb46e470c2", + "sha256:6923d077d9ae9e8bacbdb1c07ae78405a9306c8fd1af13bfa06ca891095eb995", + "sha256:833401b15de1bb92791d7b6fb353d4af60dc688eaa521bd97203dcd2d124a7c1", + "sha256:8416ed88ddc057bab0526d4e4e9f3660f614ac2394b5e019a628cdfff3733849", + "sha256:892daa86384994fdf4856cb43c93f40cbe80f7f95bb5da94971b39c7f54b3a9c", + "sha256:98be759efdb5e5fa161e46d404f4e0ce388e72fbf7d9baf010aff16689e22abe", + "sha256:a6d28e7f14ecf3b2ad67c4f106841218c8ab12a0683b1528534a6c87d2307af3", + "sha256:b1d6ebc891607e71fd9da71688fcf332a6630b7f5b7f5549e6e631821c0e5d90", + "sha256:b2a2b0d276a136146e012154baefaea2758ef1f56ae9f4e01c612b0831e0bd2f", + "sha256:b87dfa9f10a470eee7f24234a37d1d5f51e5f5fa9eeffda7c282e2b8f5162eb1", + "sha256:bac0d6f7728a9cc3c1e06d4fcbac12aaa70e9379b3025b27ec1226f0e2d404cf", + "sha256:c991112622baee0ae4d55c008380c32ecfd0ad417bcd0417ba432e6ba7328caa", + "sha256:cda422d54ee7905bfc53ee6915ab68fe7b230cacf581110df4272ee10462aadc", + "sha256:d3148b6ba3923c5850ea197a91a42683f946dba7e8eb82dfa211ab7e708de939", + "sha256:d6033b4ffa34ef70f0b8086fd4c3df4bf801fee485a8a7d4519399818351aa8e", + "sha256:ddff0b2bd7edcc8c82d1adde6dbbf5e60d57ce985402541cd2985c27f7bec2a0", + "sha256:e23cb7f1d8e0f93addf0cae3c5b6f00324cccb4a7949ee558d7b6ca973ab8ae9", + "sha256:effd2ba52cee4ceff1a77f20d2a9f9bf8d50353c854a282b8760ac15b9833168", + "sha256:f90c2267101010de42f7273c94a1f026e56cbc043f9330acd8a80e64300aba33", + "sha256:f960375e9823ae6a07072ff7f8a85954e5a6434f97869f50d0e41649a1c8144f", + "sha256:fcf32bf76dc25e30ed793145a57426064520890d7c02866eb93d3e4abe516948" + ], + "version": "==1.14.1" }, "chardet": { "hashes": [ @@ -216,49 +216,55 @@ }, "hiredis": { "hashes": [ - "sha256:01b577f84c20ecc9c07fc4c184231b08e3c3942de096fa99978e053de231c423", - "sha256:01ff0900134166961c9e339df77c33b72f7edc5cb41739f0babcd9faa345926e", - "sha256:03ed34a13316d0c34213c4fd46e0fa3a5299073f4d4f08e93fed8c2108b399b3", - "sha256:040436e91df5143aff9e0debb49530d0b17a6bd52200ce568621c31ef581b10d", - "sha256:091eb38fbf968d1c5b703e412bbbd25f43a7967d8400842cee33a5a07b33c27b", - "sha256:102f9b9dc6ed57feb3a7c9bdf7e71cb7c278fe8df1edfcfe896bc3e0c2be9447", - "sha256:2b4b392c7e3082860c8371fab3ae762139090f9115819e12d9f56060f9ede05d", - "sha256:2c9cc0b986397b833073f466e6b9e9c70d1d4dc2c2c1b3e9cae3a23102ff296c", - "sha256:2fa65a9df683bca72073cd77709ddeb289ea2b114d3775d225fbbcc5faf808c5", - "sha256:38437a681f17c975fd22349e72c29bc643f8e7eb2d6dc5df419eac59afa4d7ce", - "sha256:3b3428fa3cf1ee178807b52c9bee8950ab94cd4eaa9bfae8c1bbae3c49501d34", - "sha256:3dd8c2fae7f5494978facb0e93297dd627b1a3f536f3b070cf0a7d9157a07dcb", - "sha256:4414a96c212e732723b5c3d7c04d386ebbb2ec359e1de646322cbc3f875cbd0d", - "sha256:48c627581ad4ef60adbac980981407939acf13a0e18f093502c7b542223c4f19", - "sha256:4a60e71625a2d78d8ab84dfb2fa2cfd9458c964b6e6c04fea76d9ade153fb371", - "sha256:585ace09f434e43d8a8dbeb366865b1a044d7c06319b3c7372a0a00e63b860f4", - "sha256:74b364b3f06c9cf0a53f7df611045bc9437ed972a283fa1f0b12537236d23ddc", - "sha256:75c65c3850e89e9daa68d1b9bedd5806f177d60aa5a7b0953b4829481cfc1f72", - "sha256:7f052de8bf744730a9120dbdc67bfeb7605a01f69fb8e7ba5c475af33c24e145", - "sha256:8113a7d5e87ecf57cd4ae263cc9e429adb9a3e59f5a7768da5d3312a8d0a051a", - "sha256:84857ce239eb8ed191ac78e77ff65d52902f00f30f4ee83bf80eb71da73b70e6", - "sha256:8644a48ddc4a40b3e3a6b9443f396c2ee353afb2d45656c4fc68d04a82e8e3f7", - "sha256:936aa565e673536e8a211e43ec43197406f24cd1f290138bd143765079c8ba00", - "sha256:9afeb88c67bbc663b9f27385c496da056d06ad87f55df6e393e1516cfecb0461", - "sha256:9d62cc7880110e4f83b0a51d218f465d3095e2751fbddd34e553dbd106a929ff", - "sha256:a1fadd062fc8d647ff39220c57ea2b48c99bb73f18223828ec97f88fc27e7898", - "sha256:a7754a783b1e5d6f627c19d099b178059c62f782ab62b4d8ba165b9fbc2ee34c", - "sha256:aa59dd63bb3f736de4fc2d080114429d5d369dfb3265f771778e8349d67a97a4", - "sha256:ae2ee0992f8de249715435942137843a93db204dd7db1e7cc9bdc5a8436443e8", - "sha256:b36842d7cf32929d568f37ec5b3173b72b2ec6572dec4d6be6ce774762215aee", - "sha256:bcbf9379c553b5facc6c04c1e5569b44b38ff16bcbf354676287698d61ee0c92", - "sha256:cbccbda6f1c62ab460449d9c85fdf24d0d32a6bf45176581151e53cc26a5d910", - "sha256:d0caf98dfb8af395d6732bd16561c0a2458851bea522e39f12f04802dbf6f502", - "sha256:d6456afeddba036def1a36d8a2758eca53202308d83db20ab5d0b66590919627", - "sha256:dbaef9a21a4f10bc281684ee4124f169e62bb533c2a92b55f8c06f64f9af7b8f", - "sha256:dce84916c09aaece006272b37234ae84a8ed13abb3a4d341a23933b8701abfb5", - "sha256:eb8c9c8b9869539d58d60ff4a28373a22514d40495911451343971cb4835b7a9", - "sha256:efc98b14ee3a8595e40b1425e8d42f5fd26f11a7b215a81ef9259068931754f4", - "sha256:fa2dc05b87d97acc1c6ae63f3e0f39eae5246565232484b08db6bf2dc1580678", - "sha256:fe7d6ce9f6a5fbe24f09d95ea93e9c7271abc4e1565da511e1449b107b4d7848" + "sha256:06a039208f83744a702279b894c8cf24c14fd63c59cd917dcde168b79eef0680", + "sha256:0a909bf501459062aa1552be1461456518f367379fdc9fdb1f2ca5e4a1fdd7c0", + "sha256:18402d9e54fb278cb9a8c638df6f1550aca36a009d47ecf5aa263a38600f35b0", + "sha256:1e4cbbc3858ec7e680006e5ca590d89a5e083235988f26a004acf7244389ac01", + "sha256:23344e3c2177baf6975fbfa361ed92eb7d36d08f454636e5054b3faa7c2aff8a", + "sha256:289b31885b4996ce04cadfd5fc03d034dce8e2a8234479f7c9e23b9e245db06b", + "sha256:2c1c570ae7bf1bab304f29427e2475fe1856814312c4a1cf1cd0ee133f07a3c6", + "sha256:2c227c0ed371771ffda256034427320870e8ea2e4fd0c0a618c766e7c49aad73", + "sha256:3bb9b63d319402cead8bbd9dd55dca3b667d2997e9a0d8a1f9b6cc274db4baee", + "sha256:3ef2183de67b59930d2db8b8e8d4d58e00a50fcc5e92f4f678f6eed7a1c72d55", + "sha256:43b8ed3dbfd9171e44c554cb4acf4ee4505caa84c5e341858b50ea27dd2b6e12", + "sha256:47bcf3c5e6c1e87ceb86cdda2ee983fa0fe56a999e6185099b3c93a223f2fa9b", + "sha256:5263db1e2e1e8ae30500cdd75a979ff99dcc184201e6b4b820d0de74834d2323", + "sha256:5b1451727f02e7acbdf6aae4e06d75f66ee82966ff9114550381c3271a90f56c", + "sha256:6996883a8a6ff9117cbb3d6f5b0dcbbae6fb9e31e1a3e4e2f95e0214d9a1c655", + "sha256:6c96f64a54f030366657a54bb90b3093afc9c16c8e0dfa29fc0d6dbe169103a5", + "sha256:7332d5c3e35154cd234fd79573736ddcf7a0ade7a986db35b6196b9171493e75", + "sha256:7885b6f32c4a898e825bb7f56f36a02781ac4a951c63e4169f0afcf9c8c30dfb", + "sha256:7b0f63f10a166583ab744a58baad04e0f52cfea1ac27bfa1b0c21a48d1003c23", + "sha256:819f95d4eba3f9e484dd115ab7ab72845cf766b84286a00d4ecf76d33f1edca1", + "sha256:8968eeaa4d37a38f8ca1f9dbe53526b69628edc9c42229a5b2f56d98bb828c1f", + "sha256:89ebf69cb19a33d625db72d2ac589d26e936b8f7628531269accf4a3196e7872", + "sha256:8daecd778c1da45b8bd54fd41ffcd471a86beed3d8e57a43acf7a8d63bba4058", + "sha256:955ba8ea73cf3ed8bd2f963b4cb9f8f0dcb27becd2f4b3dd536fd24c45533454", + "sha256:964f18a59f5a64c0170f684c417f4fe3e695a536612e13074c4dd5d1c6d7c882", + "sha256:969843fbdfbf56cdb71da6f0bdf50f9985b8b8aeb630102945306cf10a9c6af2", + "sha256:996021ef33e0f50b97ff2d6b5f422a0fe5577de21a8873b58a779a5ddd1c3132", + "sha256:9e9c9078a7ce07e6fce366bd818be89365a35d2e4b163268f0ca9ba7e13bb2f6", + "sha256:a04901757cb0fb0f5602ac11dda48f5510f94372144d06c2563ba56c480b467c", + "sha256:a7bf1492429f18d205f3a818da3ff1f242f60aa59006e53dee00b4ef592a3363", + "sha256:aa0af2deb166a5e26e0d554b824605e660039b161e37ed4f01b8d04beec184f3", + "sha256:abfb15a6a7822f0fae681785cb38860e7a2cb1616a708d53df557b3d76c5bfd4", + "sha256:b253fe4df2afea4dfa6b1fa8c5fef212aff8bcaaeb4207e81eed05cb5e4a7919", + "sha256:b27f082f47d23cffc4cf1388b84fdc45c4ef6015f906cd7e0d988d9e35d36349", + "sha256:b33aea449e7f46738811fbc6f0b3177c6777a572207412bbbf6f525ffed001ae", + "sha256:b44f9421c4505c548435244d74037618f452844c5d3c67719d8a55e2613549da", + "sha256:bcc371151d1512201d0214c36c0c150b1dc64f19c2b1a8c9cb1d7c7c15ebd93f", + "sha256:c2851deeabd96d3f6283e9c6b26e0bfed4de2dc6fb15edf913e78b79fc5909ed", + "sha256:cdfd501c7ac5b198c15df800a3a34c38345f5182e5f80770caf362bccca65628", + "sha256:d2c0caffa47606d6d7c8af94ba42547bd2a441f06c74fd90a1ffe328524a6c64", + "sha256:dcb2db95e629962db5a355047fb8aefb012df6c8ae608930d391619dbd96fd86", + "sha256:e0eeb9c112fec2031927a1745788a181d0eecbacbed941fc5c4f7bc3f7b273bf", + "sha256:e154891263306200260d7f3051982774d7b9ef35af3509d5adbbe539afd2610c", + "sha256:e2e023a42dcbab8ed31f97c2bcdb980b7fbe0ada34037d87ba9d799664b58ded", + "sha256:e64be68255234bb489a574c4f2f8df7029c98c81ec4d160d6cd836e7f0679390", + "sha256:e82d6b930e02e80e5109b678c663a9ed210680ded81c1abaf54635d88d1da298" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.0.1" + "version": "==1.1.0" }, "humanfriendly": { "hashes": [ @@ -294,36 +300,40 @@ }, "lxml": { "hashes": [ - "sha256:06748c7192eab0f48e3d35a7adae609a329c6257495d5e53878003660dc0fec6", - "sha256:0790ddca3f825dd914978c94c2545dbea5f56f008b050e835403714babe62a5f", - "sha256:1aa7a6197c1cdd65d974f3e4953764eee3d9c7b67e3966616b41fab7f8f516b7", - "sha256:22c6d34fdb0e65d5f782a4d1a1edb52e0a8365858dafb1c08cb1d16546cf0786", - "sha256:2754d4406438c83144f9ffd3628bbe2dcc6d62b20dbc5c1ec4bc4385e5d44b42", - "sha256:27ee0faf8077c7c1a589573b1450743011117f1aa1a91d5ae776bbc5ca6070f2", - "sha256:2b02c106709466a93ed424454ce4c970791c486d5fcdf52b0d822a7e29789626", - "sha256:2d1ddce96cf15f1254a68dba6935e6e0f1fe39247de631c115e84dd404a6f031", - "sha256:4f282737d187ae723b2633856085c31ae5d4d432968b7f3f478a48a54835f5c4", - "sha256:51bb4edeb36d24ec97eb3e6a6007be128b720114f9a875d6b370317d62ac80b9", - "sha256:7eee37c1b9815e6505847aa5e68f192e8a1b730c5c7ead39ff317fde9ce29448", - "sha256:7fd88cb91a470b383aafad554c3fe1ccf6dfb2456ff0e84b95335d582a799804", - "sha256:9144ce36ca0824b29ebc2e02ca186e54040ebb224292072250467190fb613b96", - "sha256:925baf6ff1ef2c45169f548cc85204433e061360bfa7d01e1be7ae38bef73194", - "sha256:a636346c6c0e1092ffc202d97ec1843a75937d8c98aaf6771348ad6422e44bb0", - "sha256:a87dbee7ad9dce3aaefada2081843caf08a44a8f52e03e0a4cc5819f8398f2f4", - "sha256:a9e3b8011388e7e373565daa5e92f6c9cb844790dc18e43073212bb3e76f7007", - "sha256:afb53edf1046599991fb4a7d03e601ab5f5422a5435c47ee6ba91ec3b61416a6", - "sha256:b26719890c79a1dae7d53acac5f089d66fd8cc68a81f4e4bd355e45470dc25e1", - "sha256:b7462cdab6fffcda853338e1741ce99706cdf880d921b5a769202ea7b94e8528", - "sha256:b77975465234ff49fdad871c08aa747aae06f5e5be62866595057c43f8d2f62c", - "sha256:c47a8a5d00060122ca5908909478abce7bbf62d812e3fc35c6c802df8fb01fe7", - "sha256:c79e5debbe092e3c93ca4aee44c9a7631bdd407b2871cb541b979fd350bbbc29", - "sha256:d8d40e0121ca1606aa9e78c28a3a7d88a05c06b3ca61630242cded87d8ce55fa", - "sha256:ee2be8b8f72a2772e72ab926a3bccebf47bb727bda41ae070dc91d1fb759b726", - "sha256:f95d28193c3863132b1f55c1056036bf580b5a488d908f7d22a04ace8935a3a9", - "sha256:fadd2a63a2bfd7fb604508e553d1cf68eca250b2fbdbd81213b5f6f2fbf23529" - ], - "index": "pypi", - "version": "==4.5.1" + "sha256:05a444b207901a68a6526948c7cc8f9fe6d6f24c70781488e32fd74ff5996e3f", + "sha256:08fc93257dcfe9542c0a6883a25ba4971d78297f63d7a5a26ffa34861ca78730", + "sha256:107781b213cf7201ec3806555657ccda67b1fccc4261fb889ef7fc56976db81f", + "sha256:121b665b04083a1e85ff1f5243d4a93aa1aaba281bc12ea334d5a187278ceaf1", + "sha256:1fa21263c3aba2b76fd7c45713d4428dbcc7644d73dcf0650e9d344e433741b3", + "sha256:2b30aa2bcff8e958cd85d907d5109820b01ac511eae5b460803430a7404e34d7", + "sha256:4b4a111bcf4b9c948e020fd207f915c24a6de3f1adc7682a2d92660eb4e84f1a", + "sha256:5591c4164755778e29e69b86e425880f852464a21c7bb53c7ea453bbe2633bbe", + "sha256:59daa84aef650b11bccd18f99f64bfe44b9f14a08a28259959d33676554065a1", + "sha256:5a9c8d11aa2c8f8b6043d845927a51eb9102eb558e3f936df494e96393f5fd3e", + "sha256:5dd20538a60c4cc9a077d3b715bb42307239fcd25ef1ca7286775f95e9e9a46d", + "sha256:74f48ec98430e06c1fa8949b49ebdd8d27ceb9df8d3d1c92e1fdc2773f003f20", + "sha256:786aad2aa20de3dbff21aab86b2fb6a7be68064cbbc0219bde414d3a30aa47ae", + "sha256:7ad7906e098ccd30d8f7068030a0b16668ab8aa5cda6fcd5146d8d20cbaa71b5", + "sha256:80a38b188d20c0524fe8959c8ce770a8fdf0e617c6912d23fc97c68301bb9aba", + "sha256:8f0ec6b9b3832e0bd1d57af41f9238ea7709bbd7271f639024f2fc9d3bb01293", + "sha256:92282c83547a9add85ad658143c76a64a8d339028926d7dc1998ca029c88ea6a", + "sha256:94150231f1e90c9595ccc80d7d2006c61f90a5995db82bccbca7944fd457f0f6", + "sha256:9dc9006dcc47e00a8a6a029eb035c8f696ad38e40a27d073a003d7d1443f5d88", + "sha256:a76979f728dd845655026ab991df25d26379a1a8fc1e9e68e25c7eda43004bed", + "sha256:aa8eba3db3d8761db161003e2d0586608092e217151d7458206e243be5a43843", + "sha256:bea760a63ce9bba566c23f726d72b3c0250e2fa2569909e2d83cda1534c79443", + "sha256:c3f511a3c58676147c277eff0224c061dd5a6a8e1373572ac817ac6324f1b1e0", + "sha256:c9d317efde4bafbc1561509bfa8a23c5cab66c44d49ab5b63ff690f5159b2304", + "sha256:cc411ad324a4486b142c41d9b2b6a722c534096963688d879ea6fa8a35028258", + "sha256:cdc13a1682b2a6241080745b1953719e7fe0850b40a5c71ca574f090a1391df6", + "sha256:cfd7c5dd3c35c19cec59c63df9571c67c6d6e5c92e0fe63517920e97f61106d1", + "sha256:e1cacf4796b20865789083252186ce9dc6cc59eca0c2e79cca332bdff24ac481", + "sha256:e70d4e467e243455492f5de463b72151cc400710ac03a0678206a5f27e79ddef", + "sha256:ecc930ae559ea8a43377e8b60ca6f8d61ac532fc57efb915d899de4a67928efd", + "sha256:f161af26f596131b63b236372e4ce40f3167c1b5b5d459b29d2514bd8c9dc9ee" + ], + "index": "pypi", + "version": "==4.5.2" }, "markdownify": { "hashes": [ @@ -532,11 +542,11 @@ }, "sentry-sdk": { "hashes": [ - "sha256:da06bc3641e81ec2c942f87a0676cd9180044fa3d1697524a0005345997542e2", - "sha256:e80d61af85d99a1222c1a3e2a24023618374cd50a99673aa7fa3cf920e7d813b" + "sha256:2de15b13836fa3522815a933bd9c887c77f4868071043349f94f1b896c1bcfb8", + "sha256:38bb09d0277117f76507c8728d9a5156f09a47ac5175bb8072513859d19a593b" ], "index": "pypi", - "version": "==0.16.0" + "version": "==0.16.2" }, "six": { "hashes": [ @@ -632,13 +642,21 @@ "index": "pypi", "version": "==3.3.0" }, + "typing-extensions": { + "hashes": [ + "sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5", + "sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae", + "sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392" + ], + "version": "==3.7.4.2" + }, "urllib3": { "hashes": [ - "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", - "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" + "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a", + "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.25.9" + "version": "==1.25.10" }, "websockets": { "hashes": [ @@ -670,26 +688,26 @@ }, "yarl": { "hashes": [ - "sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce", - "sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6", - "sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce", - "sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae", - "sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d", - "sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f", - "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b", - "sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b", - "sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb", - "sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462", - "sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea", - "sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70", - "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1", - "sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a", - "sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b", - "sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080", - "sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2" + "sha256:1707230e1ea48ea06a3e20acb4ce05a38d2465bd9566c21f48f6212a88e47536", + "sha256:1f269e8e6676193a94635399a77c9059e1826fb6265c9204c9e5a8ccd36006e1", + "sha256:2657716c1fc998f5f2675c0ee6ce91282e0da0ea9e4a94b584bb1917e11c1559", + "sha256:431faa6858f0ea323714d8b7b4a7da1db2eeb9403607f0eaa3800ab2c5a4b627", + "sha256:5bbcb195da7de57f4508b7508c33f7593e9516e27732d08b9aad8586c7b8c384", + "sha256:5c82f5b1499342339f22c83b97dbe2b8a09e47163fab86cd934a8dd46620e0fb", + "sha256:5d410f69b4f92c5e1e2a8ffb73337cd8a274388c6975091735795588a538e605", + "sha256:66b4f345e9573e004b1af184bc00431145cf5e089a4dcc1351505c1f5750192c", + "sha256:875b2a741ce0208f3b818008a859ab5d0f461e98a32bbdc6af82231a9e761c55", + "sha256:9a3266b047d15e78bba38c8455bf68b391c040231ca5965ef867f7cbbc60bde5", + "sha256:9a592c4aa642249e9bdaf76897d90feeb08118626b363a6be8788a9b300274b5", + "sha256:a1772068401d425e803999dada29a6babf041786e08be5e79ef63c9ecc4c9575", + "sha256:b065a5c3e050395ae563019253cc6c769a50fd82d7fa92d07476273521d56b7c", + "sha256:b325fefd574ebef50e391a1072d1712a60348ca29c183e1d546c9d87fec2cd32", + "sha256:cf5eb664910d759bbae0b76d060d6e21f8af5098242d66c448bbebaf2a7bfa70", + "sha256:f058b6541477022c7b54db37229f87dacf3b565de4f901ff5a0a78556a174fea", + "sha256:f5cfed0766837303f688196aa7002730d62c5cc802d98c6395ea1feb87252727" ], "markers": "python_version >= '3.5'", - "version": "==1.4.2" + "version": "==1.5.0" } }, "develop": { @@ -718,43 +736,43 @@ }, "coverage": { "hashes": [ - "sha256:0fc4e0d91350d6f43ef6a61f64a48e917637e1dcfcba4b4b7d543c628ef82c2d", - "sha256:10f2a618a6e75adf64329f828a6a5b40244c1c50f5ef4ce4109e904e69c71bd2", - "sha256:12eaccd86d9a373aea59869bc9cfa0ab6ba8b1477752110cb4c10d165474f703", - "sha256:1874bdc943654ba46d28f179c1846f5710eda3aeb265ff029e0ac2b52daae404", - "sha256:1dcebae667b73fd4aa69237e6afb39abc2f27520f2358590c1b13dd90e32abe7", - "sha256:1e58fca3d9ec1a423f1b7f2aa34af4f733cbfa9020c8fe39ca451b6071237405", - "sha256:214eb2110217f2636a9329bc766507ab71a3a06a8ea30cdeebb47c24dce5972d", - "sha256:25fe74b5b2f1b4abb11e103bb7984daca8f8292683957d0738cd692f6a7cc64c", - "sha256:32ecee61a43be509b91a526819717d5e5650e009a8d5eda8631a59c721d5f3b6", - "sha256:3740b796015b889e46c260ff18b84683fa2e30f0f75a171fb10d2bf9fb91fc70", - "sha256:3b2c34690f613525672697910894b60d15800ac7e779fbd0fccf532486c1ba40", - "sha256:41d88736c42f4a22c494c32cc48a05828236e37c991bd9760f8923415e3169e4", - "sha256:42fa45a29f1059eda4d3c7b509589cc0343cd6bbf083d6118216830cd1a51613", - "sha256:4bb385a747e6ae8a65290b3df60d6c8a692a5599dc66c9fa3520e667886f2e10", - "sha256:509294f3e76d3f26b35083973fbc952e01e1727656d979b11182f273f08aa80b", - "sha256:5c74c5b6045969b07c9fb36b665c9cac84d6c174a809fc1b21bdc06c7836d9a0", - "sha256:60a3d36297b65c7f78329b80120f72947140f45b5c7a017ea730f9112b40f2ec", - "sha256:6f91b4492c5cde83bfe462f5b2b997cdf96a138f7c58b1140f05de5751623cf1", - "sha256:7403675df5e27745571aba1c957c7da2dacb537c21e14007ec3a417bf31f7f3d", - "sha256:87bdc8135b8ee739840eee19b184804e5d57f518578ffc797f5afa2c3c297913", - "sha256:8a3decd12e7934d0254939e2bf434bf04a5890c5bf91a982685021786a08087e", - "sha256:9702e2cb1c6dec01fb8e1a64c015817c0800a6eca287552c47a5ee0ebddccf62", - "sha256:a4d511012beb967a39580ba7d2549edf1e6865a33e5fe51e4dce550522b3ac0e", - "sha256:bbb387811f7a18bdc61a2ea3d102be0c7e239b0db9c83be7bfa50f095db5b92a", - "sha256:bfcc811883699ed49afc58b1ed9f80428a18eb9166422bce3c31a53dba00fd1d", - "sha256:c32aa13cc3fe86b0f744dfe35a7f879ee33ac0a560684fef0f3e1580352b818f", - "sha256:ca63dae130a2e788f2b249200f01d7fa240f24da0596501d387a50e57aa7075e", - "sha256:d54d7ea74cc00482a2410d63bf10aa34ebe1c49ac50779652106c867f9986d6b", - "sha256:d67599521dff98ec8c34cd9652cbcfe16ed076a2209625fca9dc7419b6370e5c", - "sha256:d82db1b9a92cb5c67661ca6616bdca6ff931deceebb98eecbd328812dab52032", - "sha256:d9ad0a988ae20face62520785ec3595a5e64f35a21762a57d115dae0b8fb894a", - "sha256:ebf2431b2d457ae5217f3a1179533c456f3272ded16f8ed0b32961a6d90e38ee", - "sha256:ed9a21502e9223f563e071759f769c3d6a2e1ba5328c31e86830368e8d78bc9c", - "sha256:f50632ef2d749f541ca8e6c07c9928a37f87505ce3a9f20c8446ad310f1aa87b" - ], - "index": "pypi", - "version": "==5.2" + "sha256:098a703d913be6fbd146a8c50cc76513d726b022d170e5e98dc56d958fd592fb", + "sha256:16042dc7f8e632e0dcd5206a5095ebd18cb1d005f4c89694f7f8aafd96dd43a3", + "sha256:1adb6be0dcef0cf9434619d3b892772fdb48e793300f9d762e480e043bd8e716", + "sha256:27ca5a2bc04d68f0776f2cdcb8bbd508bbe430a7bf9c02315cd05fb1d86d0034", + "sha256:28f42dc5172ebdc32622a2c3f7ead1b836cdbf253569ae5673f499e35db0bac3", + "sha256:2fcc8b58953d74d199a1a4d633df8146f0ac36c4e720b4a1997e9b6327af43a8", + "sha256:304fbe451698373dc6653772c72c5d5e883a4aadaf20343592a7abb2e643dae0", + "sha256:30bc103587e0d3df9e52cd9da1dd915265a22fad0b72afe54daf840c984b564f", + "sha256:40f70f81be4d34f8d491e55936904db5c527b0711b2a46513641a5729783c2e4", + "sha256:4186fc95c9febeab5681bc3248553d5ec8c2999b8424d4fc3a39c9cba5796962", + "sha256:46794c815e56f1431c66d81943fa90721bb858375fb36e5903697d5eef88627d", + "sha256:4869ab1c1ed33953bb2433ce7b894a28d724b7aa76c19b11e2878034a4e4680b", + "sha256:4f6428b55d2916a69f8d6453e48a505c07b2245653b0aa9f0dee38785939f5e4", + "sha256:52f185ffd3291196dc1aae506b42e178a592b0b60a8610b108e6ad892cfc1bb3", + "sha256:538f2fd5eb64366f37c97fdb3077d665fa946d2b6d95447622292f38407f9258", + "sha256:64c4f340338c68c463f1b56e3f2f0423f7b17ba6c3febae80b81f0e093077f59", + "sha256:675192fca634f0df69af3493a48224f211f8db4e84452b08d5fcebb9167adb01", + "sha256:700997b77cfab016533b3e7dbc03b71d33ee4df1d79f2463a318ca0263fc29dd", + "sha256:8505e614c983834239f865da2dd336dcf9d72776b951d5dfa5ac36b987726e1b", + "sha256:962c44070c281d86398aeb8f64e1bf37816a4dfc6f4c0f114756b14fc575621d", + "sha256:9e536783a5acee79a9b308be97d3952b662748c4037b6a24cbb339dc7ed8eb89", + "sha256:9ea749fd447ce7fb1ac71f7616371f04054d969d412d37611716721931e36efd", + "sha256:a34cb28e0747ea15e82d13e14de606747e9e484fb28d63c999483f5d5188e89b", + "sha256:a3ee9c793ffefe2944d3a2bd928a0e436cd0ac2d9e3723152d6fd5398838ce7d", + "sha256:aab75d99f3f2874733946a7648ce87a50019eb90baef931698f96b76b6769a46", + "sha256:b1ed2bdb27b4c9fc87058a1cb751c4df8752002143ed393899edb82b131e0546", + "sha256:b360d8fd88d2bad01cb953d81fd2edd4be539df7bfec41e8753fe9f4456a5082", + "sha256:b8f58c7db64d8f27078cbf2a4391af6aa4e4767cc08b37555c4ae064b8558d9b", + "sha256:c1bbb628ed5192124889b51204de27c575b3ffc05a5a91307e7640eff1d48da4", + "sha256:c2ff24df02a125b7b346c4c9078c8936da06964cc2d276292c357d64378158f8", + "sha256:c890728a93fffd0407d7d37c1e6083ff3f9f211c83b4316fae3778417eab9811", + "sha256:c96472b8ca5dc135fb0aa62f79b033f02aa434fb03a8b190600a5ae4102df1fd", + "sha256:ce7866f29d3025b5b34c2e944e66ebef0d92e4a4f2463f7266daa03a1332a651", + "sha256:e26c993bd4b220429d4ec8c1468eca445a4064a61c74ca08da7429af9bc53bb0" + ], + "index": "pypi", + "version": "==5.2.1" }, "distlib": { "hashes": [ @@ -780,11 +798,11 @@ }, "flake8-annotations": { "hashes": [ - "sha256:babc81a17a5f1a63464195917e20d3e8663fb712b3633d4522dbfc407cff31b3", - "sha256:fcd833b415726a7a374922c95a5c47a7a4d8ea71cb4a586369c665e7476146e1" + "sha256:7816a5d8f65ffdf37b8e21e5b17e0fd1e492aa92638573276de066e889a22b26", + "sha256:8d18db74a750dd97f40b483cc3ef80d07d03f687525bad8fd83365dcd3bfd414" ], "index": "pypi", - "version": "==2.2.0" + "version": "==2.3.0" }, "flake8-bugbear": { "hashes": [ @@ -842,11 +860,11 @@ }, "identify": { "hashes": [ - "sha256:c4d07f2b979e3931894170a9e0d4b8281e6905ea6d018c326f7ffefaf20db680", - "sha256:dac33eff90d57164e289fb20bf4e131baef080947ee9bf45efcd0da8d19064bf" + "sha256:110ed090fec6bce1aabe3c72d9258a9de82207adeaa5a05cd75c635880312f9a", + "sha256:ccd88716b890ecbe10920659450a635d2d25de499b9a638525a48b48261d989b" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.4.21" + "version": "==1.4.25" }, "mccabe": { "hashes": [ @@ -950,11 +968,11 @@ }, "virtualenv": { "hashes": [ - "sha256:c11a475400e98450403c0364eb3a2d25d42f71cf1493da64390487b666de4324", - "sha256:e10cc66f40cbda459720dfe1d334c4dc15add0d80f09108224f171006a97a172" + "sha256:688a61d7976d82b92f7906c367e83bb4b3f0af96f8f75bfcd3da95608fe8ac6c", + "sha256:8f582a030156282a9ee9d319984b759a232b07f86048c1d6a9e394afa44e78c8" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.0.26" + "version": "==20.0.28" } } } diff --git a/bot/__main__.py b/bot/__main__.py index 5382f5502..f698b5662 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -34,21 +34,20 @@ bot = Bot( ) # Internal/debug +bot.load_extension("bot.cogs.config_verifier") bot.load_extension("bot.cogs.error_handler") bot.load_extension("bot.cogs.filtering") bot.load_extension("bot.cogs.logging") bot.load_extension("bot.cogs.security") -bot.load_extension("bot.cogs.config_verifier") # Commands, etc bot.load_extension("bot.cogs.antimalware") bot.load_extension("bot.cogs.antispam") bot.load_extension("bot.cogs.bot") bot.load_extension("bot.cogs.clean") +bot.load_extension("bot.cogs.doc") bot.load_extension("bot.cogs.extensions") bot.load_extension("bot.cogs.help") - -bot.load_extension("bot.cogs.doc") bot.load_extension("bot.cogs.verification") # Feature cogs @@ -57,11 +56,12 @@ bot.load_extension("bot.cogs.defcon") bot.load_extension("bot.cogs.dm_relay") bot.load_extension("bot.cogs.duck_pond") bot.load_extension("bot.cogs.eval") +bot.load_extension("bot.cogs.filter_lists") bot.load_extension("bot.cogs.information") bot.load_extension("bot.cogs.jams") bot.load_extension("bot.cogs.moderation") -bot.load_extension("bot.cogs.python_news") bot.load_extension("bot.cogs.off_topic_names") +bot.load_extension("bot.cogs.python_news") bot.load_extension("bot.cogs.reddit") bot.load_extension("bot.cogs.reminders") bot.load_extension("bot.cogs.site") diff --git a/bot/bot.py b/bot/bot.py index 313652d11..756449293 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -2,7 +2,8 @@ import asyncio import logging import socket import warnings -from typing import Optional +from collections import defaultdict +from typing import Dict, Optional import aiohttp import aioredis @@ -34,6 +35,7 @@ class Bot(commands.Bot): self.redis_ready = asyncio.Event() self.redis_closed = False self.api_client = api.APIClient(loop=self.loop) + self.filter_list_cache = defaultdict(dict) self._connector = None self._resolver = None @@ -49,6 +51,13 @@ class Bot(commands.Bot): self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") + async def cache_filter_list_data(self) -> None: + """Cache all the data in the FilterList on the site.""" + full_cache = await self.api_client.get('bot/filter-lists') + + for item in full_cache: + self.insert_item_into_filter_list_cache(item) + async def _create_redis_session(self) -> None: """ Create the Redis connection pool, and then open the redis event gate. @@ -73,6 +82,49 @@ class Bot(commands.Bot): self.redis_closed = False self.redis_ready.set() + def _recreate(self) -> None: + """Re-create the connector, aiohttp session, the APIClient and the Redis session.""" + # Use asyncio for DNS resolution instead of threads so threads aren't spammed. + # Doesn't seem to have any state with regards to being closed, so no need to worry? + self._resolver = aiohttp.AsyncResolver() + + # Its __del__ does send a warning but it doesn't always show up for some reason. + if self._connector and not self._connector._closed: + log.warning( + "The previous connector was not closed; it will remain open and be overwritten" + ) + + if self.redis_session and not self.redis_session.closed: + log.warning( + "The previous redis pool was not closed; it will remain open and be overwritten" + ) + + # Create the redis session + self.loop.create_task(self._create_redis_session()) + + # Use AF_INET as its socket family to prevent HTTPS related problems both locally + # and in production. + self._connector = aiohttp.TCPConnector( + resolver=self._resolver, + family=socket.AF_INET, + ) + + # Client.login() will call HTTPClient.static_login() which will create a session using + # this connector attribute. + self.http.connector = self._connector + + # Its __del__ does send a warning but it doesn't always show up for some reason. + if self.http_session and not self.http_session.closed: + log.warning( + "The previous session was not closed; it will remain open and be overwritten" + ) + + self.http_session = aiohttp.ClientSession(connector=self._connector) + self.api_client.recreate(force=True, connector=self._connector) + + # Build the FilterList cache + self.loop.create_task(self.cache_filter_list_data()) + def add_cog(self, cog: commands.Cog) -> None: """Adds a "cog" to the bot and logs the operation.""" super().add_cog(cog) @@ -113,52 +165,25 @@ class Bot(commands.Bot): self.redis_ready.clear() await self.redis_session.wait_closed() + def insert_item_into_filter_list_cache(self, item: Dict[str, str]) -> None: + """Add an item to the bots filter_list_cache.""" + type_ = item["type"] + allowed = item["allowed"] + content = item["content"] + + self.filter_list_cache[f"{type_}.{allowed}"][content] = { + "id": item["id"], + "comment": item["comment"], + "created_at": item["created_at"], + "updated_at": item["updated_at"], + } + async def login(self, *args, **kwargs) -> None: """Re-create the connector and set up sessions before logging into Discord.""" self._recreate() await self.stats.create_socket() await super().login(*args, **kwargs) - def _recreate(self) -> None: - """Re-create the connector, aiohttp session, the APIClient and the Redis session.""" - # Use asyncio for DNS resolution instead of threads so threads aren't spammed. - # Doesn't seem to have any state with regards to being closed, so no need to worry? - self._resolver = aiohttp.AsyncResolver() - - # Its __del__ does send a warning but it doesn't always show up for some reason. - if self._connector and not self._connector._closed: - log.warning( - "The previous connector was not closed; it will remain open and be overwritten" - ) - - if self.redis_session and not self.redis_session.closed: - log.warning( - "The previous redis pool was not closed; it will remain open and be overwritten" - ) - - # Create the redis session - self.loop.create_task(self._create_redis_session()) - - # Use AF_INET as its socket family to prevent HTTPS related problems both locally - # and in production. - self._connector = aiohttp.TCPConnector( - resolver=self._resolver, - family=socket.AF_INET, - ) - - # Client.login() will call HTTPClient.static_login() which will create a session using - # this connector attribute. - self.http.connector = self._connector - - # Its __del__ does send a warning but it doesn't always show up for some reason. - if self.http_session and not self.http_session.closed: - log.warning( - "The previous session was not closed; it will remain open and be overwritten" - ) - - self.http_session = aiohttp.ClientSession(connector=self._connector) - self.api_client.recreate(force=True, connector=self._connector) - async def on_guild_available(self, guild: discord.Guild) -> None: """ Set the internal guild available event when constants.Guild.id becomes available. diff --git a/bot/cogs/antimalware.py b/bot/cogs/antimalware.py index ea257442e..c76bd2c60 100644 --- a/bot/cogs/antimalware.py +++ b/bot/cogs/antimalware.py @@ -6,7 +6,7 @@ from discord import Embed, Message, NotFound from discord.ext.commands import Cog from bot.bot import Bot -from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES, URLs +from bot.constants import Channels, STAFF_ROLES, URLs log = logging.getLogger(__name__) @@ -27,7 +27,7 @@ TXT_EMBED_DESCRIPTION = ( DISALLOWED_EMBED_DESCRIPTION = ( "It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " - f"We currently allow the following file types: **{', '.join(AntiMalwareConfig.whitelist)}**.\n\n" + "We currently allow the following file types: **{joined_whitelist}**.\n\n" "Feel free to ask in {meta_channel_mention} if you think this is a mistake." ) @@ -38,6 +38,16 @@ class AntiMalware(Cog): def __init__(self, bot: Bot): self.bot = bot + def _get_whitelisted_file_formats(self) -> list: + """Get the file formats currently on the whitelist.""" + return self.bot.filter_list_cache['FILE_FORMAT.True'].keys() + + def _get_disallowed_extensions(self, message: Message) -> t.Iterable[str]: + """Get an iterable containing all the disallowed extensions of attachments.""" + file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} + extensions_blocked = file_extensions - set(self._get_whitelisted_file_formats()) + return extensions_blocked + @Cog.listener() async def on_message(self, message: Message) -> None: """Identify messages with prohibited attachments.""" @@ -51,7 +61,7 @@ class AntiMalware(Cog): return embed = Embed() - extensions_blocked = self.get_disallowed_extensions(message) + extensions_blocked = self._get_disallowed_extensions(message) blocked_extensions_str = ', '.join(extensions_blocked) if ".py" in extensions_blocked: # Short-circuit on *.py files to provide a pastebin link @@ -63,6 +73,7 @@ class AntiMalware(Cog): elif extensions_blocked: meta_channel = self.bot.get_channel(Channels.meta) embed.description = DISALLOWED_EMBED_DESCRIPTION.format( + joined_whitelist=', '.join(self._get_whitelisted_file_formats()), blocked_extensions_str=blocked_extensions_str, meta_channel_mention=meta_channel.mention, ) @@ -81,13 +92,6 @@ class AntiMalware(Cog): except NotFound: log.info(f"Tried to delete message `{message.id}`, but message could not be found.") - @classmethod - def get_disallowed_extensions(cls, message: Message) -> t.Iterable[str]: - """Get an iterable containing all the disallowed extensions of attachments.""" - file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} - extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) - return extensions_blocked - def setup(bot: Bot) -> None: """Load the AntiMalware cog.""" diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index 233851e41..f9d4de638 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -2,12 +2,13 @@ import contextlib import logging import typing as t +from discord import Embed from discord.ext.commands import Cog, Context, errors from sentry_sdk import push_scope from bot.api import ResponseCodeError from bot.bot import Bot -from bot.constants import Channels +from bot.constants import Channels, Colours from bot.converters import TagNameConverter from bot.utils.checks import InWhitelistCheckFailure @@ -20,6 +21,14 @@ class ErrorHandler(Cog): def __init__(self, bot: Bot): self.bot = bot + def _get_error_embed(self, title: str, body: str) -> Embed: + """Return an embed that contains the exception.""" + return Embed( + title=title, + colour=Colours.soft_red, + description=body + ) + @Cog.listener() async def on_command_error(self, ctx: Context, e: errors.CommandError) -> None: """ @@ -162,25 +171,34 @@ class ErrorHandler(Cog): prepared_help_command = self.get_help_command(ctx) if isinstance(e, errors.MissingRequiredArgument): - await ctx.send(f"Missing required argument `{e.param.name}`.") + embed = self._get_error_embed("Missing required argument", e.param.name) + await ctx.send(embed=embed) await prepared_help_command self.bot.stats.incr("errors.missing_required_argument") elif isinstance(e, errors.TooManyArguments): - await ctx.send("Too many arguments provided.") + embed = self._get_error_embed("Too many arguments", str(e)) + await ctx.send(embed=embed) await prepared_help_command self.bot.stats.incr("errors.too_many_arguments") elif isinstance(e, errors.BadArgument): - await ctx.send("Bad argument: Please double-check your input arguments and try again.\n") + embed = self._get_error_embed("Bad argument", str(e)) + await ctx.send(embed=embed) await prepared_help_command self.bot.stats.incr("errors.bad_argument") elif isinstance(e, errors.BadUnionArgument): - await ctx.send(f"Bad argument: {e}\n```{e.errors[-1]}```") + embed = self._get_error_embed("Bad argument", f"{e}\n{e.errors[-1]}") + await ctx.send(embed=embed) self.bot.stats.incr("errors.bad_union_argument") elif isinstance(e, errors.ArgumentParsingError): - await ctx.send(f"Argument parsing error: {e}") + embed = self._get_error_embed("Argument parsing error", str(e)) + await ctx.send(embed=embed) self.bot.stats.incr("errors.argument_parsing_error") else: - await ctx.send("Something about your input seems off. Check the arguments:") + embed = self._get_error_embed( + "Input error", + "Something about your input seems off. Check the arguments and try again." + ) + await ctx.send(embed=embed) await prepared_help_command self.bot.stats.incr("errors.other_user_input_error") diff --git a/bot/cogs/filter_lists.py b/bot/cogs/filter_lists.py new file mode 100644 index 000000000..c15adc461 --- /dev/null +++ b/bot/cogs/filter_lists.py @@ -0,0 +1,273 @@ +import logging +from typing import Optional + +from discord import Colour, Embed +from discord.ext.commands import BadArgument, Cog, Context, IDConverter, group + +from bot import constants +from bot.api import ResponseCodeError +from bot.bot import Bot +from bot.converters import ValidDiscordServerInvite, ValidFilterListType +from bot.pagination import LinePaginator +from bot.utils.checks import with_role_check + +log = logging.getLogger(__name__) + + +class FilterLists(Cog): + """Commands for blacklisting and whitelisting things.""" + + methods_with_filterlist_types = [ + "allow_add", + "allow_delete", + "allow_get", + "deny_add", + "deny_delete", + "deny_get", + ] + + def __init__(self, bot: Bot) -> None: + self.bot = bot + self.bot.loop.create_task(self._amend_docstrings()) + + async def _amend_docstrings(self) -> None: + """Add the valid FilterList types to the docstrings, so they'll appear in !help invocations.""" + await self.bot.wait_until_guild_available() + + # Add valid filterlist types to the docstrings + valid_types = await ValidFilterListType.get_valid_types(self.bot) + valid_types = [f"`{type_.lower()}`" for type_ in valid_types] + + for method_name in self.methods_with_filterlist_types: + command = getattr(self, method_name) + command.help = ( + f"{command.help}\n\nValid **list_type** values are {', '.join(valid_types)}." + ) + + async def _add_data( + self, + ctx: Context, + allowed: bool, + list_type: ValidFilterListType, + content: str, + comment: Optional[str] = None, + ) -> None: + """Add an item to a filterlist.""" + allow_type = "whitelist" if allowed else "blacklist" + + # If this is a server invite, we gotta validate it. + if list_type == "GUILD_INVITE": + guild_data = await self._validate_guild_invite(ctx, content) + content = guild_data.get("id") + + # Unless the user has specified another comment, let's + # use the server name as the comment so that the list + # of guild IDs will be more easily readable when we + # display it. + if not comment: + comment = guild_data.get("name") + + # If it's a file format, let's make sure it has a leading dot. + elif list_type == "FILE_FORMAT" and not content.startswith("."): + content = f".{content}" + + # Try to add the item to the database + log.trace(f"Trying to add the {content} item to the {list_type} {allow_type}") + payload = { + "allowed": allowed, + "type": list_type, + "content": content, + "comment": comment, + } + + try: + item = await self.bot.api_client.post( + "bot/filter-lists", + json=payload + ) + except ResponseCodeError as e: + if e.status == 400: + await ctx.message.add_reaction("❌") + log.debug( + f"{ctx.author} tried to add data to a {allow_type}, but the API returned 400, " + "probably because the request violated the UniqueConstraint." + ) + raise BadArgument( + f"Unable to add the item to the {allow_type}. " + "The item probably already exists. Keep in mind that a " + "blacklist and a whitelist for the same item cannot co-exist, " + "and we do not permit any duplicates." + ) + raise + + # Insert the item into the cache + self.bot.insert_item_into_filter_list_cache(item) + await ctx.message.add_reaction("✅") + + async def _delete_data(self, ctx: Context, allowed: bool, list_type: ValidFilterListType, content: str) -> None: + """Remove an item from a filterlist.""" + allow_type = "whitelist" if allowed else "blacklist" + + # If this is a server invite, we need to convert it. + if list_type == "GUILD_INVITE" and not IDConverter()._get_id_match(content): + guild_data = await self._validate_guild_invite(ctx, content) + content = guild_data.get("id") + + # If it's a file format, let's make sure it has a leading dot. + elif list_type == "FILE_FORMAT" and not content.startswith("."): + content = f".{content}" + + # Find the content and delete it. + log.trace(f"Trying to delete the {content} item from the {list_type} {allow_type}") + item = self.bot.filter_list_cache[f"{list_type}.{allowed}"].get(content) + + if item is not None: + try: + await self.bot.api_client.delete( + f"bot/filter-lists/{item['id']}" + ) + del self.bot.filter_list_cache[f"{list_type}.{allowed}"][content] + await ctx.message.add_reaction("✅") + except ResponseCodeError as e: + log.debug( + f"{ctx.author} tried to delete an item with the id {item['id']}, but " + f"the API raised an unexpected error: {e}" + ) + await ctx.message.add_reaction("❌") + else: + await ctx.message.add_reaction("❌") + + async def _list_all_data(self, ctx: Context, allowed: bool, list_type: ValidFilterListType) -> None: + """Paginate and display all items in a filterlist.""" + allow_type = "whitelist" if allowed else "blacklist" + result = self.bot.filter_list_cache[f"{list_type}.{allowed}"] + + # Build a list of lines we want to show in the paginator + lines = [] + for content, metadata in result.items(): + line = f"• `{content}`" + + if comment := metadata.get("comment"): + line += f" - {comment}" + + lines.append(line) + lines = sorted(lines) + + # Build the embed + list_type_plural = list_type.lower().replace("_", " ").title() + "s" + embed = Embed( + title=f"{allow_type.title()}ed {list_type_plural} ({len(result)} total)", + colour=Colour.blue() + ) + log.trace(f"Trying to list {len(result)} items from the {list_type.lower()} {allow_type}") + + if result: + await LinePaginator.paginate(lines, ctx, embed, max_lines=15, empty=False) + else: + embed.description = "Hmmm, seems like there's nothing here yet." + await ctx.send(embed=embed) + await ctx.message.add_reaction("❌") + + async def _sync_data(self, ctx: Context) -> None: + """Syncs the filterlists with the API.""" + try: + log.trace("Attempting to sync FilterList cache with data from the API.") + await self.bot.cache_filter_list_data() + await ctx.message.add_reaction("✅") + except ResponseCodeError as e: + log.debug( + f"{ctx.author} tried to sync FilterList cache data but " + f"the API raised an unexpected error: {e}" + ) + await ctx.message.add_reaction("❌") + + @staticmethod + async def _validate_guild_invite(ctx: Context, invite: str) -> dict: + """ + Validates a guild invite, and returns the guild info as a dict. + + Will raise a BadArgument if the guild invite is invalid. + """ + log.trace(f"Attempting to validate whether or not {invite} is a guild invite.") + validator = ValidDiscordServerInvite() + guild_data = await validator.convert(ctx, invite) + + # If we make it this far without raising a BadArgument, the invite is + # valid. Let's return a dict of guild information. + log.trace(f"{invite} validated as server invite. Converting to ID.") + return guild_data + + @group(aliases=("allowlist", "allow", "al", "wl")) + async def whitelist(self, ctx: Context) -> None: + """Group for whitelisting commands.""" + if not ctx.invoked_subcommand: + await ctx.send_help(ctx.command) + + @group(aliases=("denylist", "deny", "bl", "dl")) + async def blacklist(self, ctx: Context) -> None: + """Group for blacklisting commands.""" + if not ctx.invoked_subcommand: + await ctx.send_help(ctx.command) + + @whitelist.command(name="add", aliases=("a", "set")) + async def allow_add( + self, + ctx: Context, + list_type: ValidFilterListType, + content: str, + *, + comment: Optional[str] = None, + ) -> None: + """Add an item to the specified allowlist.""" + await self._add_data(ctx, True, list_type, content, comment) + + @blacklist.command(name="add", aliases=("a", "set")) + async def deny_add( + self, + ctx: Context, + list_type: ValidFilterListType, + content: str, + *, + comment: Optional[str] = None, + ) -> None: + """Add an item to the specified denylist.""" + await self._add_data(ctx, False, list_type, content, comment) + + @whitelist.command(name="remove", aliases=("delete", "rm",)) + async def allow_delete(self, ctx: Context, list_type: ValidFilterListType, content: str) -> None: + """Remove an item from the specified allowlist.""" + await self._delete_data(ctx, True, list_type, content) + + @blacklist.command(name="remove", aliases=("delete", "rm",)) + async def deny_delete(self, ctx: Context, list_type: ValidFilterListType, content: str) -> None: + """Remove an item from the specified denylist.""" + await self._delete_data(ctx, False, list_type, content) + + @whitelist.command(name="get", aliases=("list", "ls", "fetch", "show")) + async def allow_get(self, ctx: Context, list_type: ValidFilterListType) -> None: + """Get the contents of a specified allowlist.""" + await self._list_all_data(ctx, True, list_type) + + @blacklist.command(name="get", aliases=("list", "ls", "fetch", "show")) + async def deny_get(self, ctx: Context, list_type: ValidFilterListType) -> None: + """Get the contents of a specified denylist.""" + await self._list_all_data(ctx, False, list_type) + + @whitelist.command(name="sync", aliases=("s",)) + async def allow_sync(self, ctx: Context) -> None: + """Syncs both allowlists and denylists with the API.""" + await self._sync_data(ctx) + + @blacklist.command(name="sync", aliases=("s",)) + async def deny_sync(self, ctx: Context) -> None: + """Syncs both allowlists and denylists with the API.""" + await self._sync_data(ctx) + + def cog_check(self, ctx: Context) -> bool: + """Only allow moderators to invoke the commands in this cog.""" + return with_role_check(ctx, *constants.MODERATION_ROLES) + + +def setup(bot: Bot) -> None: + """Load the FilterLists cog.""" + bot.add_cog(FilterLists(bot)) diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 29aac812f..4ec95ad73 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -18,44 +18,18 @@ from bot.constants import ( Filter, Icons, URLs ) from bot.utils.redis_cache import RedisCache +from bot.utils.regex import INVITE_RE from bot.utils.scheduling import Scheduler log = logging.getLogger(__name__) -INVITE_RE = re.compile( - r"(?:discord(?:[\.,]|dot)gg|" # Could be discord.gg/ - r"discord(?:[\.,]|dot)com(?:\/|slash)invite|" # or discord.com/invite/ - r"discordapp(?:[\.,]|dot)com(?:\/|slash)invite|" # or discordapp.com/invite/ - r"discord(?:[\.,]|dot)me|" # or discord.me - r"discord(?:[\.,]|dot)io" # or discord.io. - r")(?:[\/]|slash)" # / or 'slash' - r"([a-zA-Z0-9]+)", # the invite code itself - flags=re.IGNORECASE -) - +# Regular expressions SPOILER_RE = re.compile(r"(\|\|.+?\|\|)", re.DOTALL) URL_RE = re.compile(r"(https?://[^\s]+)", flags=re.IGNORECASE) ZALGO_RE = re.compile(r"[\u0300-\u036F\u0489]") -WORD_WATCHLIST_PATTERNS = [ - re.compile(fr'\b{expression}\b', flags=re.IGNORECASE) for expression in Filter.word_watchlist -] -TOKEN_WATCHLIST_PATTERNS = [ - re.compile(fr'{expression}', flags=re.IGNORECASE) for expression in Filter.token_watchlist -] -WATCHLIST_PATTERNS = WORD_WATCHLIST_PATTERNS + TOKEN_WATCHLIST_PATTERNS - +# Other constants. DAYS_BETWEEN_ALERTS = 3 - - -def expand_spoilers(text: str) -> str: - """Return a string containing all interpretations of a spoilered message.""" - split_text = SPOILER_RE.split(text) - return ''.join( - split_text[0::2] + split_text[1::2] + split_text - ) - - OFFENSIVE_MSG_DELETE_TIME = timedelta(days=Filter.offensive_msg_delete_days) @@ -125,6 +99,22 @@ class Filtering(Cog): self.bot.loop.create_task(self.reschedule_offensive_msg_deletion()) + def cog_unload(self) -> None: + """Cancel scheduled tasks.""" + self.scheduler.cancel_all() + + def _get_filterlist_items(self, list_type: str, *, allowed: bool) -> list: + """Fetch items from the filter_list_cache.""" + return self.bot.filter_list_cache[f"{list_type.upper()}.{allowed}"].keys() + + @staticmethod + def _expand_spoilers(text: str) -> str: + """Return a string containing all interpretations of a spoilered message.""" + split_text = SPOILER_RE.split(text) + return ''.join( + split_text[0::2] + split_text[1::2] + split_text + ) + @property def mod_log(self) -> ModLog: """Get currently loaded ModLog cog instance.""" @@ -149,12 +139,12 @@ class Filtering(Cog): delta = relativedelta(after.edited_at, before.edited_at).microseconds await self._filter_message(after, delta) - @staticmethod - def get_name_matches(name: str) -> List[re.Match]: + def get_name_matches(self, name: str) -> List[re.Match]: """Check bad words from passed string (name). Return list of matches.""" matches = [] - for pattern in WATCHLIST_PATTERNS: - if match := pattern.search(name): + watchlist_patterns = self._get_filterlist_items('filter_token', allowed=False) + for pattern in watchlist_patterns: + if match := re.search(pattern, name, flags=re.IGNORECASE): matches.append(match) return matches @@ -370,14 +360,14 @@ class Filtering(Cog): # They have no data so additional embeds can't be created for them. if name == "filter_invites" and match is not True: additional_embeds = [] - for invite, data in match.items(): + for _, data in match.items(): embed = discord.Embed(description=( f"**Members:**\n{data['members']}\n" f"**Active:**\n{data['active']}" )) embed.set_author(name=data["name"]) embed.set_thumbnail(url=data["icon"]) - embed.set_footer(text=f"Guild Invite Code: {invite}") + embed.set_footer(text=f"Guild ID: {data['id']}") additional_embeds.append(embed) additional_embeds_msg = "For the following guild(s):" @@ -403,8 +393,7 @@ class Filtering(Cog): and not msg.author.bot # Author not a bot ) - @staticmethod - async def _has_watch_regex_match(text: str) -> Union[bool, re.Match]: + async def _has_watch_regex_match(self, text: str) -> Union[bool, re.Match]: """ Return True if `text` matches any regex from `word_watchlist` or `token_watchlist` configs. @@ -412,26 +401,27 @@ class Filtering(Cog): matched as-is. Spoilers are expanded, if any, and URLs are ignored. """ if SPOILER_RE.search(text): - text = expand_spoilers(text) + text = self._expand_spoilers(text) # Make sure it's not a URL if URL_RE.search(text): return False - for pattern in WATCHLIST_PATTERNS: - match = pattern.search(text) + watchlist_patterns = self._get_filterlist_items('filter_token', allowed=False) + for pattern in watchlist_patterns: + match = re.search(pattern, text, flags=re.IGNORECASE) if match: return match - @staticmethod - async def _has_urls(text: str) -> bool: + async def _has_urls(self, text: str) -> bool: """Returns True if the text contains one of the blacklisted URLs from the config file.""" if not URL_RE.search(text): return False text = text.lower() + domain_blacklist = self._get_filterlist_items("domain_name", allowed=False) - for url in Filter.domain_blacklist: + for url in domain_blacklist: if url.lower() in text: return True @@ -455,7 +445,7 @@ class Filtering(Cog): Attempts to catch some of common ways to try to cheat the system. """ - # Remove backslashes to prevent escape character around fuckery like + # Remove backslashes to prevent escape character aroundfuckery like # discord\.gg/gdudes-pony-farm text = text.replace("\\", "") @@ -476,9 +466,22 @@ class Filtering(Cog): # between invalid and expired invites return True - guild_id = int(guild.get("id")) + guild_id = guild.get("id") + guild_invite_whitelist = self._get_filterlist_items("guild_invite", allowed=True) + guild_invite_blacklist = self._get_filterlist_items("guild_invite", allowed=False) + + # Is this invite allowed? + guild_partnered_or_verified = ( + 'PARTNERED' in guild.get("features", []) + or 'VERIFIED' in guild.get("features", []) + ) + invite_not_allowed = ( + guild_id in guild_invite_blacklist # Blacklisted guilds are never permitted. + or guild_id not in guild_invite_whitelist # Whitelisted guilds are always permitted. + and not guild_partnered_or_verified # Otherwise guilds have to be Verified or Partnered. + ) - if guild_id not in Filter.guild_invite_whitelist: + if invite_not_allowed: guild_icon_hash = guild["icon"] guild_icon = ( "https://cdn.discordapp.com/icons/" @@ -487,6 +490,7 @@ class Filtering(Cog): invite_data[invite] = { "name": guild["name"], + "id": guild['id'], "icon": guild_icon, "members": response["approximate_member_count"], "active": response["approximate_presence_count"] diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 601e238c9..75028d851 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -31,6 +31,10 @@ class InfractionScheduler: self.bot.loop.create_task(self.reschedule_infractions(supported_infractions)) + def cog_unload(self) -> None: + """Cancel scheduled tasks.""" + self.scheduler.cancel_all() + @property def mod_log(self) -> ModLog: """Get the currently loaded ModLog cog instance.""" diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index ae4fb7b64..f8a6592bc 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -152,7 +152,8 @@ class Silence(commands.Cog): return False def cog_unload(self) -> None: - """Send alert with silenced channels on unload.""" + """Send alert with silenced channels and cancel scheduled tasks on unload.""" + self.scheduler.cancel_all() if self.muted_channels: channels_string = ''.join(channel.mention for channel in self.muted_channels) message = f"<@&{Roles.moderators}> channels left silenced on cog unload: {channels_string}" diff --git a/bot/cogs/off_topic_names.py b/bot/cogs/off_topic_names.py index 201579a0b..ce95450e0 100644 --- a/bot/cogs/off_topic_names.py +++ b/bot/cogs/off_topic_names.py @@ -4,46 +4,19 @@ import logging from datetime import datetime, timedelta from discord import Colour, Embed -from discord.ext.commands import BadArgument, Cog, Context, Converter, group +from discord.ext.commands import Cog, Context, group from bot.api import ResponseCodeError from bot.bot import Bot from bot.constants import Channels, MODERATION_ROLES +from bot.converters import OffTopicName from bot.decorators import with_role from bot.pagination import LinePaginator - CHANNELS = (Channels.off_topic_0, Channels.off_topic_1, Channels.off_topic_2) log = logging.getLogger(__name__) -class OffTopicName(Converter): - """A converter that ensures an added off-topic name is valid.""" - - @staticmethod - async def convert(ctx: Context, argument: str) -> str: - """Attempt to replace any invalid characters with their approximate Unicode equivalent.""" - allowed_characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ!?'`-" - - # Chain multiple words to a single one - argument = "-".join(argument.split()) - - if not (2 <= len(argument) <= 96): - raise BadArgument("Channel name must be between 2 and 96 chars long") - - elif not all(c.isalnum() or c in allowed_characters for c in argument): - raise BadArgument( - "Channel name must only consist of " - "alphanumeric characters, minus signs or apostrophes." - ) - - # Replace invalid characters with unicode alternatives. - table = str.maketrans( - allowed_characters, '𝖠𝖡𝖢𝖣𝖤𝖥𝖦𝖧𝖨𝖩𝖪𝖫𝖬𝖭𝖮𝖯𝖰𝖱𝖲𝖳𝖴𝖵𝖶𝖷𝖸𝖹ǃ？’’-' - ) - return argument.translate(table) - - async def update_names(bot: Bot) -> None: """Background updater task that performs the daily channel name update.""" while True: diff --git a/bot/cogs/reminders.py b/bot/cogs/reminders.py index b5998cc0e..670493bcf 100644 --- a/bot/cogs/reminders.py +++ b/bot/cogs/reminders.py @@ -37,6 +37,10 @@ class Reminders(Cog): self.bot.loop.create_task(self.reschedule_reminders()) + def cog_unload(self) -> None: + """Cancel scheduled tasks.""" + self.scheduler.cancel_all() + async def reschedule_reminders(self) -> None: """Get all current reminders from the API and reschedule them.""" await self.bot.wait_until_guild_available() diff --git a/bot/cogs/source.py b/bot/cogs/source.py index f1db745cd..205e0ba81 100644 --- a/bot/cogs/source.py +++ b/bot/cogs/source.py @@ -60,11 +60,12 @@ class BotSource(commands.Cog): await ctx.send(embed=embed) def get_source_link(self, source_item: SourceType) -> Tuple[str, str, Optional[int]]: - """Build GitHub link of source item, return this link, file location and first line number.""" - if isinstance(source_item, commands.HelpCommand): - src = type(source_item) - filename = inspect.getsourcefile(src) - elif isinstance(source_item, commands.Command): + """ + Build GitHub link of source item, return this link, file location and first line number. + + Raise BadArgument if `source_item` is a dynamically-created object (e.g. via internal eval). + """ + if isinstance(source_item, commands.Command): if source_item.cog_name == "Alias": cmd_name = source_item.callback.__name__.replace("_alias", "") cmd = self.bot.get_command(cmd_name.replace("_", " ")) @@ -78,10 +79,17 @@ class BotSource(commands.Cog): filename = tags_cog._cache[source_item]["location"] else: src = type(source_item) - filename = inspect.getsourcefile(src) + try: + filename = inspect.getsourcefile(src) + except TypeError: + raise commands.BadArgument("Cannot get source for a dynamically-created object.") if not isinstance(source_item, str): - lines, first_line_no = inspect.getsourcelines(src) + try: + lines, first_line_no = inspect.getsourcelines(src) + except OSError: + raise commands.BadArgument("Cannot get source for a dynamically-created object.") + lines_extension = f"#L{first_line_no}-L{first_line_no+len(lines)-1}" else: first_line_no = None diff --git a/bot/constants.py b/bot/constants.py index cce64a7c4..0902858ac 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -227,10 +227,6 @@ class Filter(metaclass=YAMLGetter): ping_everyone: bool offensive_msg_delete_days: int - guild_invite_whitelist: List[int] - domain_blacklist: List[str] - word_watchlist: List[str] - token_watchlist: List[str] channel_whitelist: List[int] role_whitelist: List[int] @@ -538,12 +534,6 @@ class AntiSpam(metaclass=YAMLGetter): rules: Dict[str, Dict[str, int]] -class AntiMalware(metaclass=YAMLGetter): - section = "anti_malware" - - whitelist: list - - class BigBrother(metaclass=YAMLGetter): section = 'big_brother' diff --git a/bot/converters.py b/bot/converters.py index 4a0633951..1358cbf1e 100644 --- a/bot/converters.py +++ b/bot/converters.py @@ -9,8 +9,11 @@ import dateutil.tz import discord from aiohttp import ClientConnectorError from dateutil.relativedelta import relativedelta -from discord.ext.commands import BadArgument, Context, Converter, UserConverter +from discord.ext.commands import BadArgument, Bot, Context, Converter, IDConverter, UserConverter +from bot.api import ResponseCodeError +from bot.constants import URLs +from bot.utils.regex import INVITE_RE log = logging.getLogger(__name__) @@ -34,6 +37,90 @@ def allowed_strings(*values, preserve_case: bool = False) -> t.Callable[[str], s return converter +class ValidDiscordServerInvite(Converter): + """ + A converter that validates whether a given string is a valid Discord server invite. + + Raises 'BadArgument' if: + - The string is not a valid Discord server invite. + - The string is valid, but is an invite for a group DM. + - The string is valid, but is expired. + + Returns a (partial) guild object if: + - The string is a valid vanity + - The string is a full invite URI + - The string contains the invite code (the stuff after discord.gg/) + + See the Discord API docs for documentation on the guild object: + https://discord.com/developers/docs/resources/guild#guild-object + """ + + async def convert(self, ctx: Context, server_invite: str) -> dict: + """Check whether the string is a valid Discord server invite.""" + invite_code = INVITE_RE.search(server_invite) + if invite_code: + response = await ctx.bot.http_session.get( + f"{URLs.discord_invite_api}/{invite_code[1]}" + ) + if response.status != 404: + invite_data = await response.json() + return invite_data.get("guild") + + id_converter = IDConverter() + if id_converter._get_id_match(server_invite): + raise BadArgument("Guild IDs are not supported, only invites.") + + raise BadArgument("This does not appear to be a valid Discord server invite.") + + +class ValidFilterListType(Converter): + """ + A converter that checks whether the given string is a valid FilterList type. + + Raises `BadArgument` if the argument is not a valid FilterList type, and simply + passes through the given argument otherwise. + """ + + @staticmethod + async def get_valid_types(bot: Bot) -> list: + """ + Try to get a list of valid filter list types. + + Raise a BadArgument if the API can't respond. + """ + try: + valid_types = await bot.api_client.get('bot/filter-lists/get-types') + except ResponseCodeError: + raise BadArgument("Cannot validate list_type: Unable to fetch valid types from API.") + + return [enum for enum, classname in valid_types] + + async def convert(self, ctx: Context, list_type: str) -> str: + """Checks whether the given string is a valid FilterList type.""" + valid_types = await self.get_valid_types(ctx.bot) + list_type = list_type.upper() + + if list_type not in valid_types: + + # Maybe the user is using the plural form of this type, + # e.g. "guild_invites" instead of "guild_invite". + # + # This code will support the simple plural form (a single 's' at the end), + # which works for all current list types, but if a list type is added in the future + # which has an irregular plural form (like 'ies'), this code will need to be + # refactored to support this. + if list_type.endswith("S") and list_type[:-1] in valid_types: + list_type = list_type[:-1] + + else: + valid_types_list = '\n'.join([f"• {type_.lower()}" for type_ in valid_types]) + raise BadArgument( + f"You have provided an invalid list type!\n\n" + f"Please provide one of the following: \n{valid_types_list}" + ) + return list_type + + class ValidPythonIdentifier(Converter): """ A converter that checks whether the given string is a valid Python identifier. @@ -237,6 +324,32 @@ class Duration(DurationDelta): raise BadArgument(f"`{duration}` results in a datetime outside the supported range.") +class OffTopicName(Converter): + """A converter that ensures an added off-topic name is valid.""" + + async def convert(self, ctx: Context, argument: str) -> str: + """Attempt to replace any invalid characters with their approximate Unicode equivalent.""" + allowed_characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ!?'`-" + + # Chain multiple words to a single one + argument = "-".join(argument.split()) + + if not (2 <= len(argument) <= 96): + raise BadArgument("Channel name must be between 2 and 96 chars long") + + elif not all(c.isalnum() or c in allowed_characters for c in argument): + raise BadArgument( + "Channel name must only consist of " + "alphanumeric characters, minus signs or apostrophes." + ) + + # Replace invalid characters with unicode alternatives. + table = str.maketrans( + allowed_characters, '𝖠𝖡𝖢𝖣𝖤𝖥𝖦𝖧𝖨𝖩𝖪𝖫𝖬𝖭𝖮𝖯𝖰𝖱𝖲𝖳𝖴𝖵𝖶𝖷𝖸𝖹ǃ？’’-' + ) + return argument.translate(table) + + class ISODateTime(Converter): """Converts an ISO-8601 datetime string into a datetime.datetime.""" diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py index 58cfe1df5..52b689b49 100644 --- a/bot/utils/redis_cache.py +++ b/bot/utils/redis_cache.py @@ -226,7 +226,6 @@ class RedisCache: for attribute in vars(instance).values(): if isinstance(attribute, Bot): self.bot = attribute - self._redis = self.bot.redis_session return self else: error_message = ( @@ -251,7 +250,7 @@ class RedisCache: value = self._value_to_typestring(value) log.trace(f"Setting {key} to {value}.") - await self._redis.hset(self._namespace, key, value) + await self.bot.redis_session.hset(self._namespace, key, value) async def get(self, key: RedisKeyType, default: Optional[RedisValueType] = None) -> Optional[RedisValueType]: """Get an item from the Redis cache.""" @@ -259,7 +258,7 @@ class RedisCache: key = self._key_to_typestring(key) log.trace(f"Attempting to retrieve {key}.") - value = await self._redis.hget(self._namespace, key) + value = await self.bot.redis_session.hget(self._namespace, key) if value is None: log.trace(f"Value not found, returning default value {default}") @@ -281,7 +280,7 @@ class RedisCache: key = self._key_to_typestring(key) log.trace(f"Attempting to delete {key}.") - return await self._redis.hdel(self._namespace, key) + return await self.bot.redis_session.hdel(self._namespace, key) async def contains(self, key: RedisKeyType) -> bool: """ @@ -291,7 +290,7 @@ class RedisCache: """ await self._validate_cache() key = self._key_to_typestring(key) - exists = await self._redis.hexists(self._namespace, key) + exists = await self.bot.redis_session.hexists(self._namespace, key) log.trace(f"Testing if {key} exists in the RedisCache - Result is {exists}") return exists @@ -314,7 +313,7 @@ class RedisCache: """ await self._validate_cache() items = self._dict_from_typestring( - await self._redis.hgetall(self._namespace) + await self.bot.redis_session.hgetall(self._namespace) ).items() log.trace(f"Retrieving all key/value pairs from cache, total of {len(items)} items.") @@ -323,7 +322,7 @@ class RedisCache: async def length(self) -> int: """Return the number of items in the Redis cache.""" await self._validate_cache() - number_of_items = await self._redis.hlen(self._namespace) + number_of_items = await self.bot.redis_session.hlen(self._namespace) log.trace(f"Returning length. Result is {number_of_items}.") return number_of_items @@ -335,7 +334,7 @@ class RedisCache: """Deletes the entire hash from the Redis cache.""" await self._validate_cache() log.trace("Clearing the cache of all key/value pairs.") - await self._redis.delete(self._namespace) + await self.bot.redis_session.delete(self._namespace) async def pop(self, key: RedisKeyType, default: Optional[RedisValueType] = None) -> RedisValueType: """Get the item, remove it from the cache, and provide a default if not found.""" @@ -364,7 +363,7 @@ class RedisCache: """ await self._validate_cache() log.trace(f"Updating the cache with the following items:\n{items}") - await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) + await self.bot.redis_session.hmset_dict(self._namespace, self._dict_to_typestring(items)) async def increment(self, key: RedisKeyType, amount: Optional[int, float] = 1) -> None: """ diff --git a/bot/utils/regex.py b/bot/utils/regex.py new file mode 100644 index 000000000..0d2068f90 --- /dev/null +++ b/bot/utils/regex.py @@ -0,0 +1,12 @@ +import re + +INVITE_RE = re.compile( + r"(?:discord(?:[\.,]|dot)gg|" # Could be discord.gg/ + r"discord(?:[\.,]|dot)com(?:\/|slash)invite|" # or discord.com/invite/ + r"discordapp(?:[\.,]|dot)com(?:\/|slash)invite|" # or discordapp.com/invite/ + r"discord(?:[\.,]|dot)me|" # or discord.me + r"discord(?:[\.,]|dot)io" # or discord.io. + r")(?:[\/]|slash)" # / or 'slash' + r"([a-zA-Z0-9\-]+)", # the invite code itself + flags=re.IGNORECASE +) diff --git a/config-default.yml b/config-default.yml index 21a3eca87..58bdbe20f 100644 --- a/config-default.yml +++ b/config-default.yml @@ -278,107 +278,6 @@ filter: ping_everyone: true offensive_msg_delete_days: 7 # How many days before deleting an offensive message? - guild_invite_whitelist: - - 280033776820813825 # Functional Programming - - 267624335836053506 # Python Discord - - 440186186024222721 # Python Discord: Emojis 1 - - 578587418123304970 # Python Discord: Emojis 2 - - 273944235143593984 # STEM - - 348658686962696195 # RLBot - - 531221516914917387 # Pallets - - 249111029668249601 # Gentoo - - 327254708534116352 # Adafruit - - 544525886180032552 # kennethreitz.org - - 590806733924859943 # Discord Hack Week - - 423249981340778496 # Kivy - - 197038439483310086 # Discord Testers - - 286633898581164032 # Ren'Py - - 349505959032389632 # PyGame - - 438622377094414346 # Pyglet - - 524691714909274162 # Panda3D - - 336642139381301249 # discord.py - - 405403391410438165 # Sentdex - - 172018499005317120 # The Coding Den - - 666560367173828639 # PyWeek - - 702724176489873509 # Microsoft Python - - 150662382874525696 # Microsoft Community - - 81384788765712384 # Discord API - - 613425648685547541 # Discord Developers - - 185590609631903755 # Blender Hub - - 420324994703163402 # /r/FlutterDev - - 488751051629920277 # Python Atlanta - - 143867839282020352 # C# - - 159039020565790721 # Django - - 238666723824238602 # Programming Discussions - - 433980600391696384 # JetBrains Community - - 204621105720328193 # Raspberry Pi - - 244230771232079873 # Programmers Hangout - - 239433591950540801 # SpeakJS - - 174075418410876928 # DevCord - - 489222168727519232 # Unity - - 494558898880118785 # Programmer Humor - - domain_blacklist: - - pornhub.com - - liveleak.com - - grabify.link - - bmwforum.co - - leancoding.co - - spottyfly.com - - stopify.co - - yoütu.be - - discörd.com - - minecräft.com - - freegiftcards.co - - disçordapp.com - - fortnight.space - - fortnitechat.site - - joinmy.site - - curiouscat.club - - catsnthings.fun - - yourtube.site - - youtubeshort.watch - - catsnthing.com - - youtubeshort.pro - - canadianlumberjacks.online - - poweredbydialup.club - - poweredbydialup.online - - poweredbysecurity.org - - poweredbysecurity.online - - ssteam.site - - steamwalletgift.com - - discord.gift - - lmgtfy.com - - word_watchlist: - - goo+ks* - - ky+s+ - - ki+ke+s* - - beaner+s? - - coo+ns* - - nig+lets* - - slant-eyes* - - towe?l-?head+s* - - chi*n+k+s* - - spick*s* - - kill* +(?:yo)?urself+ - - jew+s* - - suicide - - rape - - (re+)tar+(d+|t+)(ed)? - - ta+r+d+ - - cunts* - - trann*y - - shemale - - token_watchlist: - - fa+g+s* - - 卐 - - 卍 - - cuck(?!oo+) - - nigg+(?:e*r+|a+h*?|u+h+)s? - - fag+o+t+s* - # Censor doesn't apply to these channel_whitelist: - *ADMINS @@ -459,6 +358,10 @@ anti_spam: interval: 10 max: 7 + burst_shared: + interval: 10 + max: 20 + chars: interval: 5 max: 3_000 @@ -489,35 +392,6 @@ anti_spam: max: 3 -anti_malware: - whitelist: - - '.3gp' - - '.3g2' - - '.avi' - - '.bmp' - - '.gif' - - '.h264' - - '.jpg' - - '.jpeg' - - '.m4v' - - '.mkv' - - '.mov' - - '.mp4' - - '.mpeg' - - '.mpg' - - '.png' - - '.tiff' - - '.wmv' - - '.svg' - - '.psd' # Photoshop - - '.ai' # Illustrator - - '.aep' # After Effects - - '.xcf' # GIMP - - '.mp3' - - '.wav' - - '.ogg' - - reddit: subreddits: - 'r/Python' diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py index f219fc1ba..ecb7abf00 100644 --- a/tests/bot/cogs/test_antimalware.py +++ b/tests/bot/cogs/test_antimalware.py @@ -1,28 +1,33 @@ import unittest -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import AsyncMock, Mock from discord import NotFound from bot.cogs import antimalware -from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES +from bot.constants import Channels, STAFF_ROLES from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole -MODULE = "bot.cogs.antimalware" - -@patch(f"{MODULE}.AntiMalwareConfig.whitelist", new=[".first", ".second", ".third"]) class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): """Test the AntiMalware cog.""" def setUp(self): """Sets up fresh objects for each test.""" self.bot = MockBot() + self.bot.filter_list_cache = { + "FILE_FORMAT.True": { + ".first": {}, + ".second": {}, + ".third": {}, + } + } self.cog = antimalware.AntiMalware(self.bot) self.message = MockMessage() + self.whitelist = [".first", ".second", ".third"] async def test_message_with_allowed_attachment(self): """Messages with allowed extensions should not be deleted""" - attachment = MockAttachment(filename=f"python{AntiMalwareConfig.whitelist[0]}") + attachment = MockAttachment(filename="python.first") self.message.attachments = [attachment] await self.cog.on_message(self.message) @@ -93,7 +98,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(embed.description, antimalware.TXT_EMBED_DESCRIPTION.format.return_value) antimalware.TXT_EMBED_DESCRIPTION.format.assert_called_with(cmd_channel_mention=cmd_channel.mention) - async def test_other_disallowed_extention_embed_description(self): + async def test_other_disallowed_extension_embed_description(self): """Test the description for a non .py/.txt disallowed extension.""" attachment = MockAttachment(filename="python.disallowed") self.message.attachments = [attachment] @@ -109,6 +114,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(embed.description, antimalware.DISALLOWED_EMBED_DESCRIPTION.format.return_value) antimalware.DISALLOWED_EMBED_DESCRIPTION.format.assert_called_with( + joined_whitelist=", ".join(self.whitelist), blocked_extensions_str=".disallowed", meta_channel_mention=meta_channel.mention ) @@ -135,7 +141,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): """The return value should include all non-whitelisted extensions.""" test_values = ( ([], []), - (AntiMalwareConfig.whitelist, []), + (self.whitelist, []), ([".first"], []), ([".first", ".disallowed"], [".disallowed"]), ([".disallowed"], [".disallowed"]), @@ -145,7 +151,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): for extensions, expected_disallowed_extensions in test_values: with self.subTest(extensions=extensions, expected_disallowed_extensions=expected_disallowed_extensions): self.message.attachments = [MockAttachment(filename=f"filename{extension}") for extension in extensions] - disallowed_extensions = self.cog.get_disallowed_extensions(self.message) + disallowed_extensions = self.cog._get_disallowed_extensions(self.message) self.assertCountEqual(disallowed_extensions, expected_disallowed_extensions) |