aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorGravatar Leon Sandøy <[email protected]>2020-08-03 10:36:04 +0200
committerGravatar GitHub <[email protected]>2020-08-03 10:36:04 +0200
commit1240fa7163bbffcff367fdc48c92ab698dd791f4 (patch)
tree314b90bc6e9a36789d6b3eab01a82c57c049ddeb
parentRemove superfluous Available help channels. (diff)
parentMerge branch 'master' into whitelist_system (diff)
Merge pull request #1058 from python-discord/whitelist_system
FilterLists: Manage whitelisting and blacklisting via the bot
Diffstat
-rw-r--r--Pipfile2
-rw-r--r--Pipfile.lock362
-rw-r--r--bot/__main__.py8
-rw-r--r--bot/bot.py107
-rw-r--r--bot/cogs/antimalware.py24
-rw-r--r--bot/cogs/error_handler.py32
-rw-r--r--bot/cogs/filter_lists.py250
-rw-r--r--bot/cogs/filtering.py92
-rw-r--r--bot/cogs/off_topic_names.py31
-rw-r--r--bot/constants.py10
-rw-r--r--bot/converters.py103
-rw-r--r--bot/utils/regex.py12
-rw-r--r--config-default.yml130
-rw-r--r--tests/bot/cogs/test_antimalware.py24
14 files changed, 727 insertions, 460 deletions
diff --git a/Pipfile b/Pipfile
index 2d6b45aa9..4db8a238b 100644
--- a/Pipfile
+++ b/Pipfile
@@ -28,7 +28,7 @@ statsd = "~=3.3"
[dev-packages]
coverage = "~=5.0"
-flake8 = "~=3.7"
+flake8 = "~=3.8"
flake8-annotations = "~=2.0"
flake8-bugbear = "~=20.1"
flake8-docstrings = "~=1.4"
diff --git a/Pipfile.lock b/Pipfile.lock
index 4b9d092d4..c8cd96d3d 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "8a53baefbbd2a0f3fbaf831f028b23d257a5e28b5efa1260661d74604f4113b8"
+ "sha256": "eab4852974d26bd2c10362540c3e01d34af62446cb4e1915ec9a0bf2bddf4d94"
},
"pipfile-spec": 6,
"requires": {
@@ -115,36 +115,36 @@
},
"cffi": {
"hashes": [
- "sha256:001bf3242a1bb04d985d63e138230802c6c8d4db3668fb545fb5005ddf5bb5ff",
- "sha256:00789914be39dffba161cfc5be31b55775de5ba2235fe49aa28c148236c4e06b",
- "sha256:028a579fc9aed3af38f4892bdcc7390508adabc30c6af4a6e4f611b0c680e6ac",
- "sha256:14491a910663bf9f13ddf2bc8f60562d6bc5315c1f09c704937ef17293fb85b0",
- "sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384",
- "sha256:2089ed025da3919d2e75a4d963d008330c96751127dd6f73c8dc0c65041b4c26",
- "sha256:2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6",
- "sha256:337d448e5a725bba2d8293c48d9353fc68d0e9e4088d62a9571def317797522b",
- "sha256:399aed636c7d3749bbed55bc907c3288cb43c65c4389964ad5ff849b6370603e",
- "sha256:3b911c2dbd4f423b4c4fcca138cadde747abdb20d196c4a48708b8a2d32b16dd",
- "sha256:3d311bcc4a41408cf5854f06ef2c5cab88f9fded37a3b95936c9879c1640d4c2",
- "sha256:62ae9af2d069ea2698bf536dcfe1e4eed9090211dbaafeeedf5cb6c41b352f66",
- "sha256:66e41db66b47d0d8672d8ed2708ba91b2f2524ece3dee48b5dfb36be8c2f21dc",
- "sha256:675686925a9fb403edba0114db74e741d8181683dcf216be697d208857e04ca8",
- "sha256:7e63cbcf2429a8dbfe48dcc2322d5f2220b77b2e17b7ba023d6166d84655da55",
- "sha256:8a6c688fefb4e1cd56feb6c511984a6c4f7ec7d2a1ff31a10254f3c817054ae4",
- "sha256:8c0ffc886aea5df6a1762d0019e9cb05f825d0eec1f520c51be9d198701daee5",
- "sha256:95cd16d3dee553f882540c1ffe331d085c9e629499ceadfbda4d4fde635f4b7d",
- "sha256:99f748a7e71ff382613b4e1acc0ac83bf7ad167fb3802e35e90d9763daba4d78",
- "sha256:b8c78301cefcf5fd914aad35d3c04c2b21ce8629b5e4f4e45ae6812e461910fa",
- "sha256:c420917b188a5582a56d8b93bdd8e0f6eca08c84ff623a4c16e809152cd35793",
- "sha256:c43866529f2f06fe0edc6246eb4faa34f03fe88b64a0a9a942561c8e22f4b71f",
- "sha256:cab50b8c2250b46fe738c77dbd25ce017d5e6fb35d3407606e7a4180656a5a6a",
- "sha256:cef128cb4d5e0b3493f058f10ce32365972c554572ff821e175dbc6f8ff6924f",
- "sha256:cf16e3cf6c0a5fdd9bc10c21687e19d29ad1fe863372b5543deaec1039581a30",
- "sha256:e56c744aa6ff427a607763346e4170629caf7e48ead6921745986db3692f987f",
- "sha256:e577934fc5f8779c554639376beeaa5657d54349096ef24abe8c74c5d9c117c3",
- "sha256:f2b0fa0c01d8a0c7483afd9f31d7ecf2d71760ca24499c8697aeb5ca37dc090c"
- ],
- "version": "==1.14.0"
+ "sha256:267adcf6e68d77ba154334a3e4fc921b8e63cbb38ca00d33d40655d4228502bc",
+ "sha256:26f33e8f6a70c255767e3c3f957ccafc7f1f706b966e110b855bfe944511f1f9",
+ "sha256:3cd2c044517f38d1b577f05927fb9729d3396f1d44d0c659a445599e79519792",
+ "sha256:4a03416915b82b81af5502459a8a9dd62a3c299b295dcdf470877cb948d655f2",
+ "sha256:4ce1e995aeecf7cc32380bc11598bfdfa017d592259d5da00fc7ded11e61d022",
+ "sha256:4f53e4128c81ca3212ff4cf097c797ab44646a40b42ec02a891155cd7a2ba4d8",
+ "sha256:4fa72a52a906425416f41738728268072d5acfd48cbe7796af07a923236bcf96",
+ "sha256:66dd45eb9530e3dde8f7c009f84568bc7cac489b93d04ac86e3111fb46e470c2",
+ "sha256:6923d077d9ae9e8bacbdb1c07ae78405a9306c8fd1af13bfa06ca891095eb995",
+ "sha256:833401b15de1bb92791d7b6fb353d4af60dc688eaa521bd97203dcd2d124a7c1",
+ "sha256:8416ed88ddc057bab0526d4e4e9f3660f614ac2394b5e019a628cdfff3733849",
+ "sha256:892daa86384994fdf4856cb43c93f40cbe80f7f95bb5da94971b39c7f54b3a9c",
+ "sha256:98be759efdb5e5fa161e46d404f4e0ce388e72fbf7d9baf010aff16689e22abe",
+ "sha256:a6d28e7f14ecf3b2ad67c4f106841218c8ab12a0683b1528534a6c87d2307af3",
+ "sha256:b1d6ebc891607e71fd9da71688fcf332a6630b7f5b7f5549e6e631821c0e5d90",
+ "sha256:b2a2b0d276a136146e012154baefaea2758ef1f56ae9f4e01c612b0831e0bd2f",
+ "sha256:b87dfa9f10a470eee7f24234a37d1d5f51e5f5fa9eeffda7c282e2b8f5162eb1",
+ "sha256:bac0d6f7728a9cc3c1e06d4fcbac12aaa70e9379b3025b27ec1226f0e2d404cf",
+ "sha256:c991112622baee0ae4d55c008380c32ecfd0ad417bcd0417ba432e6ba7328caa",
+ "sha256:cda422d54ee7905bfc53ee6915ab68fe7b230cacf581110df4272ee10462aadc",
+ "sha256:d3148b6ba3923c5850ea197a91a42683f946dba7e8eb82dfa211ab7e708de939",
+ "sha256:d6033b4ffa34ef70f0b8086fd4c3df4bf801fee485a8a7d4519399818351aa8e",
+ "sha256:ddff0b2bd7edcc8c82d1adde6dbbf5e60d57ce985402541cd2985c27f7bec2a0",
+ "sha256:e23cb7f1d8e0f93addf0cae3c5b6f00324cccb4a7949ee558d7b6ca973ab8ae9",
+ "sha256:effd2ba52cee4ceff1a77f20d2a9f9bf8d50353c854a282b8760ac15b9833168",
+ "sha256:f90c2267101010de42f7273c94a1f026e56cbc043f9330acd8a80e64300aba33",
+ "sha256:f960375e9823ae6a07072ff7f8a85954e5a6434f97869f50d0e41649a1c8144f",
+ "sha256:fcf32bf76dc25e30ed793145a57426064520890d7c02866eb93d3e4abe516948"
+ ],
+ "version": "==1.14.1"
},
"chardet": {
"hashes": [
@@ -216,49 +216,55 @@
},
"hiredis": {
"hashes": [
- "sha256:01b577f84c20ecc9c07fc4c184231b08e3c3942de096fa99978e053de231c423",
- "sha256:01ff0900134166961c9e339df77c33b72f7edc5cb41739f0babcd9faa345926e",
- "sha256:03ed34a13316d0c34213c4fd46e0fa3a5299073f4d4f08e93fed8c2108b399b3",
- "sha256:040436e91df5143aff9e0debb49530d0b17a6bd52200ce568621c31ef581b10d",
- "sha256:091eb38fbf968d1c5b703e412bbbd25f43a7967d8400842cee33a5a07b33c27b",
- "sha256:102f9b9dc6ed57feb3a7c9bdf7e71cb7c278fe8df1edfcfe896bc3e0c2be9447",
- "sha256:2b4b392c7e3082860c8371fab3ae762139090f9115819e12d9f56060f9ede05d",
- "sha256:2c9cc0b986397b833073f466e6b9e9c70d1d4dc2c2c1b3e9cae3a23102ff296c",
- "sha256:2fa65a9df683bca72073cd77709ddeb289ea2b114d3775d225fbbcc5faf808c5",
- "sha256:38437a681f17c975fd22349e72c29bc643f8e7eb2d6dc5df419eac59afa4d7ce",
- "sha256:3b3428fa3cf1ee178807b52c9bee8950ab94cd4eaa9bfae8c1bbae3c49501d34",
- "sha256:3dd8c2fae7f5494978facb0e93297dd627b1a3f536f3b070cf0a7d9157a07dcb",
- "sha256:4414a96c212e732723b5c3d7c04d386ebbb2ec359e1de646322cbc3f875cbd0d",
- "sha256:48c627581ad4ef60adbac980981407939acf13a0e18f093502c7b542223c4f19",
- "sha256:4a60e71625a2d78d8ab84dfb2fa2cfd9458c964b6e6c04fea76d9ade153fb371",
- "sha256:585ace09f434e43d8a8dbeb366865b1a044d7c06319b3c7372a0a00e63b860f4",
- "sha256:74b364b3f06c9cf0a53f7df611045bc9437ed972a283fa1f0b12537236d23ddc",
- "sha256:75c65c3850e89e9daa68d1b9bedd5806f177d60aa5a7b0953b4829481cfc1f72",
- "sha256:7f052de8bf744730a9120dbdc67bfeb7605a01f69fb8e7ba5c475af33c24e145",
- "sha256:8113a7d5e87ecf57cd4ae263cc9e429adb9a3e59f5a7768da5d3312a8d0a051a",
- "sha256:84857ce239eb8ed191ac78e77ff65d52902f00f30f4ee83bf80eb71da73b70e6",
- "sha256:8644a48ddc4a40b3e3a6b9443f396c2ee353afb2d45656c4fc68d04a82e8e3f7",
- "sha256:936aa565e673536e8a211e43ec43197406f24cd1f290138bd143765079c8ba00",
- "sha256:9afeb88c67bbc663b9f27385c496da056d06ad87f55df6e393e1516cfecb0461",
- "sha256:9d62cc7880110e4f83b0a51d218f465d3095e2751fbddd34e553dbd106a929ff",
- "sha256:a1fadd062fc8d647ff39220c57ea2b48c99bb73f18223828ec97f88fc27e7898",
- "sha256:a7754a783b1e5d6f627c19d099b178059c62f782ab62b4d8ba165b9fbc2ee34c",
- "sha256:aa59dd63bb3f736de4fc2d080114429d5d369dfb3265f771778e8349d67a97a4",
- "sha256:ae2ee0992f8de249715435942137843a93db204dd7db1e7cc9bdc5a8436443e8",
- "sha256:b36842d7cf32929d568f37ec5b3173b72b2ec6572dec4d6be6ce774762215aee",
- "sha256:bcbf9379c553b5facc6c04c1e5569b44b38ff16bcbf354676287698d61ee0c92",
- "sha256:cbccbda6f1c62ab460449d9c85fdf24d0d32a6bf45176581151e53cc26a5d910",
- "sha256:d0caf98dfb8af395d6732bd16561c0a2458851bea522e39f12f04802dbf6f502",
- "sha256:d6456afeddba036def1a36d8a2758eca53202308d83db20ab5d0b66590919627",
- "sha256:dbaef9a21a4f10bc281684ee4124f169e62bb533c2a92b55f8c06f64f9af7b8f",
- "sha256:dce84916c09aaece006272b37234ae84a8ed13abb3a4d341a23933b8701abfb5",
- "sha256:eb8c9c8b9869539d58d60ff4a28373a22514d40495911451343971cb4835b7a9",
- "sha256:efc98b14ee3a8595e40b1425e8d42f5fd26f11a7b215a81ef9259068931754f4",
- "sha256:fa2dc05b87d97acc1c6ae63f3e0f39eae5246565232484b08db6bf2dc1580678",
- "sha256:fe7d6ce9f6a5fbe24f09d95ea93e9c7271abc4e1565da511e1449b107b4d7848"
+ "sha256:06a039208f83744a702279b894c8cf24c14fd63c59cd917dcde168b79eef0680",
+ "sha256:0a909bf501459062aa1552be1461456518f367379fdc9fdb1f2ca5e4a1fdd7c0",
+ "sha256:18402d9e54fb278cb9a8c638df6f1550aca36a009d47ecf5aa263a38600f35b0",
+ "sha256:1e4cbbc3858ec7e680006e5ca590d89a5e083235988f26a004acf7244389ac01",
+ "sha256:23344e3c2177baf6975fbfa361ed92eb7d36d08f454636e5054b3faa7c2aff8a",
+ "sha256:289b31885b4996ce04cadfd5fc03d034dce8e2a8234479f7c9e23b9e245db06b",
+ "sha256:2c1c570ae7bf1bab304f29427e2475fe1856814312c4a1cf1cd0ee133f07a3c6",
+ "sha256:2c227c0ed371771ffda256034427320870e8ea2e4fd0c0a618c766e7c49aad73",
+ "sha256:3bb9b63d319402cead8bbd9dd55dca3b667d2997e9a0d8a1f9b6cc274db4baee",
+ "sha256:3ef2183de67b59930d2db8b8e8d4d58e00a50fcc5e92f4f678f6eed7a1c72d55",
+ "sha256:43b8ed3dbfd9171e44c554cb4acf4ee4505caa84c5e341858b50ea27dd2b6e12",
+ "sha256:47bcf3c5e6c1e87ceb86cdda2ee983fa0fe56a999e6185099b3c93a223f2fa9b",
+ "sha256:5263db1e2e1e8ae30500cdd75a979ff99dcc184201e6b4b820d0de74834d2323",
+ "sha256:5b1451727f02e7acbdf6aae4e06d75f66ee82966ff9114550381c3271a90f56c",
+ "sha256:6996883a8a6ff9117cbb3d6f5b0dcbbae6fb9e31e1a3e4e2f95e0214d9a1c655",
+ "sha256:6c96f64a54f030366657a54bb90b3093afc9c16c8e0dfa29fc0d6dbe169103a5",
+ "sha256:7332d5c3e35154cd234fd79573736ddcf7a0ade7a986db35b6196b9171493e75",
+ "sha256:7885b6f32c4a898e825bb7f56f36a02781ac4a951c63e4169f0afcf9c8c30dfb",
+ "sha256:7b0f63f10a166583ab744a58baad04e0f52cfea1ac27bfa1b0c21a48d1003c23",
+ "sha256:819f95d4eba3f9e484dd115ab7ab72845cf766b84286a00d4ecf76d33f1edca1",
+ "sha256:8968eeaa4d37a38f8ca1f9dbe53526b69628edc9c42229a5b2f56d98bb828c1f",
+ "sha256:89ebf69cb19a33d625db72d2ac589d26e936b8f7628531269accf4a3196e7872",
+ "sha256:8daecd778c1da45b8bd54fd41ffcd471a86beed3d8e57a43acf7a8d63bba4058",
+ "sha256:955ba8ea73cf3ed8bd2f963b4cb9f8f0dcb27becd2f4b3dd536fd24c45533454",
+ "sha256:964f18a59f5a64c0170f684c417f4fe3e695a536612e13074c4dd5d1c6d7c882",
+ "sha256:969843fbdfbf56cdb71da6f0bdf50f9985b8b8aeb630102945306cf10a9c6af2",
+ "sha256:996021ef33e0f50b97ff2d6b5f422a0fe5577de21a8873b58a779a5ddd1c3132",
+ "sha256:9e9c9078a7ce07e6fce366bd818be89365a35d2e4b163268f0ca9ba7e13bb2f6",
+ "sha256:a04901757cb0fb0f5602ac11dda48f5510f94372144d06c2563ba56c480b467c",
+ "sha256:a7bf1492429f18d205f3a818da3ff1f242f60aa59006e53dee00b4ef592a3363",
+ "sha256:aa0af2deb166a5e26e0d554b824605e660039b161e37ed4f01b8d04beec184f3",
+ "sha256:abfb15a6a7822f0fae681785cb38860e7a2cb1616a708d53df557b3d76c5bfd4",
+ "sha256:b253fe4df2afea4dfa6b1fa8c5fef212aff8bcaaeb4207e81eed05cb5e4a7919",
+ "sha256:b27f082f47d23cffc4cf1388b84fdc45c4ef6015f906cd7e0d988d9e35d36349",
+ "sha256:b33aea449e7f46738811fbc6f0b3177c6777a572207412bbbf6f525ffed001ae",
+ "sha256:b44f9421c4505c548435244d74037618f452844c5d3c67719d8a55e2613549da",
+ "sha256:bcc371151d1512201d0214c36c0c150b1dc64f19c2b1a8c9cb1d7c7c15ebd93f",
+ "sha256:c2851deeabd96d3f6283e9c6b26e0bfed4de2dc6fb15edf913e78b79fc5909ed",
+ "sha256:cdfd501c7ac5b198c15df800a3a34c38345f5182e5f80770caf362bccca65628",
+ "sha256:d2c0caffa47606d6d7c8af94ba42547bd2a441f06c74fd90a1ffe328524a6c64",
+ "sha256:dcb2db95e629962db5a355047fb8aefb012df6c8ae608930d391619dbd96fd86",
+ "sha256:e0eeb9c112fec2031927a1745788a181d0eecbacbed941fc5c4f7bc3f7b273bf",
+ "sha256:e154891263306200260d7f3051982774d7b9ef35af3509d5adbbe539afd2610c",
+ "sha256:e2e023a42dcbab8ed31f97c2bcdb980b7fbe0ada34037d87ba9d799664b58ded",
+ "sha256:e64be68255234bb489a574c4f2f8df7029c98c81ec4d160d6cd836e7f0679390",
+ "sha256:e82d6b930e02e80e5109b678c663a9ed210680ded81c1abaf54635d88d1da298"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
- "version": "==1.0.1"
+ "version": "==1.1.0"
},
"humanfriendly": {
"hashes": [
@@ -294,36 +300,40 @@
},
"lxml": {
"hashes": [
- "sha256:06748c7192eab0f48e3d35a7adae609a329c6257495d5e53878003660dc0fec6",
- "sha256:0790ddca3f825dd914978c94c2545dbea5f56f008b050e835403714babe62a5f",
- "sha256:1aa7a6197c1cdd65d974f3e4953764eee3d9c7b67e3966616b41fab7f8f516b7",
- "sha256:22c6d34fdb0e65d5f782a4d1a1edb52e0a8365858dafb1c08cb1d16546cf0786",
- "sha256:2754d4406438c83144f9ffd3628bbe2dcc6d62b20dbc5c1ec4bc4385e5d44b42",
- "sha256:27ee0faf8077c7c1a589573b1450743011117f1aa1a91d5ae776bbc5ca6070f2",
- "sha256:2b02c106709466a93ed424454ce4c970791c486d5fcdf52b0d822a7e29789626",
- "sha256:2d1ddce96cf15f1254a68dba6935e6e0f1fe39247de631c115e84dd404a6f031",
- "sha256:4f282737d187ae723b2633856085c31ae5d4d432968b7f3f478a48a54835f5c4",
- "sha256:51bb4edeb36d24ec97eb3e6a6007be128b720114f9a875d6b370317d62ac80b9",
- "sha256:7eee37c1b9815e6505847aa5e68f192e8a1b730c5c7ead39ff317fde9ce29448",
- "sha256:7fd88cb91a470b383aafad554c3fe1ccf6dfb2456ff0e84b95335d582a799804",
- "sha256:9144ce36ca0824b29ebc2e02ca186e54040ebb224292072250467190fb613b96",
- "sha256:925baf6ff1ef2c45169f548cc85204433e061360bfa7d01e1be7ae38bef73194",
- "sha256:a636346c6c0e1092ffc202d97ec1843a75937d8c98aaf6771348ad6422e44bb0",
- "sha256:a87dbee7ad9dce3aaefada2081843caf08a44a8f52e03e0a4cc5819f8398f2f4",
- "sha256:a9e3b8011388e7e373565daa5e92f6c9cb844790dc18e43073212bb3e76f7007",
- "sha256:afb53edf1046599991fb4a7d03e601ab5f5422a5435c47ee6ba91ec3b61416a6",
- "sha256:b26719890c79a1dae7d53acac5f089d66fd8cc68a81f4e4bd355e45470dc25e1",
- "sha256:b7462cdab6fffcda853338e1741ce99706cdf880d921b5a769202ea7b94e8528",
- "sha256:b77975465234ff49fdad871c08aa747aae06f5e5be62866595057c43f8d2f62c",
- "sha256:c47a8a5d00060122ca5908909478abce7bbf62d812e3fc35c6c802df8fb01fe7",
- "sha256:c79e5debbe092e3c93ca4aee44c9a7631bdd407b2871cb541b979fd350bbbc29",
- "sha256:d8d40e0121ca1606aa9e78c28a3a7d88a05c06b3ca61630242cded87d8ce55fa",
- "sha256:ee2be8b8f72a2772e72ab926a3bccebf47bb727bda41ae070dc91d1fb759b726",
- "sha256:f95d28193c3863132b1f55c1056036bf580b5a488d908f7d22a04ace8935a3a9",
- "sha256:fadd2a63a2bfd7fb604508e553d1cf68eca250b2fbdbd81213b5f6f2fbf23529"
- ],
- "index": "pypi",
- "version": "==4.5.1"
+ "sha256:05a444b207901a68a6526948c7cc8f9fe6d6f24c70781488e32fd74ff5996e3f",
+ "sha256:08fc93257dcfe9542c0a6883a25ba4971d78297f63d7a5a26ffa34861ca78730",
+ "sha256:107781b213cf7201ec3806555657ccda67b1fccc4261fb889ef7fc56976db81f",
+ "sha256:121b665b04083a1e85ff1f5243d4a93aa1aaba281bc12ea334d5a187278ceaf1",
+ "sha256:1fa21263c3aba2b76fd7c45713d4428dbcc7644d73dcf0650e9d344e433741b3",
+ "sha256:2b30aa2bcff8e958cd85d907d5109820b01ac511eae5b460803430a7404e34d7",
+ "sha256:4b4a111bcf4b9c948e020fd207f915c24a6de3f1adc7682a2d92660eb4e84f1a",
+ "sha256:5591c4164755778e29e69b86e425880f852464a21c7bb53c7ea453bbe2633bbe",
+ "sha256:59daa84aef650b11bccd18f99f64bfe44b9f14a08a28259959d33676554065a1",
+ "sha256:5a9c8d11aa2c8f8b6043d845927a51eb9102eb558e3f936df494e96393f5fd3e",
+ "sha256:5dd20538a60c4cc9a077d3b715bb42307239fcd25ef1ca7286775f95e9e9a46d",
+ "sha256:74f48ec98430e06c1fa8949b49ebdd8d27ceb9df8d3d1c92e1fdc2773f003f20",
+ "sha256:786aad2aa20de3dbff21aab86b2fb6a7be68064cbbc0219bde414d3a30aa47ae",
+ "sha256:7ad7906e098ccd30d8f7068030a0b16668ab8aa5cda6fcd5146d8d20cbaa71b5",
+ "sha256:80a38b188d20c0524fe8959c8ce770a8fdf0e617c6912d23fc97c68301bb9aba",
+ "sha256:8f0ec6b9b3832e0bd1d57af41f9238ea7709bbd7271f639024f2fc9d3bb01293",
+ "sha256:92282c83547a9add85ad658143c76a64a8d339028926d7dc1998ca029c88ea6a",
+ "sha256:94150231f1e90c9595ccc80d7d2006c61f90a5995db82bccbca7944fd457f0f6",
+ "sha256:9dc9006dcc47e00a8a6a029eb035c8f696ad38e40a27d073a003d7d1443f5d88",
+ "sha256:a76979f728dd845655026ab991df25d26379a1a8fc1e9e68e25c7eda43004bed",
+ "sha256:aa8eba3db3d8761db161003e2d0586608092e217151d7458206e243be5a43843",
+ "sha256:bea760a63ce9bba566c23f726d72b3c0250e2fa2569909e2d83cda1534c79443",
+ "sha256:c3f511a3c58676147c277eff0224c061dd5a6a8e1373572ac817ac6324f1b1e0",
+ "sha256:c9d317efde4bafbc1561509bfa8a23c5cab66c44d49ab5b63ff690f5159b2304",
+ "sha256:cc411ad324a4486b142c41d9b2b6a722c534096963688d879ea6fa8a35028258",
+ "sha256:cdc13a1682b2a6241080745b1953719e7fe0850b40a5c71ca574f090a1391df6",
+ "sha256:cfd7c5dd3c35c19cec59c63df9571c67c6d6e5c92e0fe63517920e97f61106d1",
+ "sha256:e1cacf4796b20865789083252186ce9dc6cc59eca0c2e79cca332bdff24ac481",
+ "sha256:e70d4e467e243455492f5de463b72151cc400710ac03a0678206a5f27e79ddef",
+ "sha256:ecc930ae559ea8a43377e8b60ca6f8d61ac532fc57efb915d899de4a67928efd",
+ "sha256:f161af26f596131b63b236372e4ce40f3167c1b5b5d459b29d2514bd8c9dc9ee"
+ ],
+ "index": "pypi",
+ "version": "==4.5.2"
},
"markdownify": {
"hashes": [
@@ -532,11 +542,11 @@
},
"sentry-sdk": {
"hashes": [
- "sha256:da06bc3641e81ec2c942f87a0676cd9180044fa3d1697524a0005345997542e2",
- "sha256:e80d61af85d99a1222c1a3e2a24023618374cd50a99673aa7fa3cf920e7d813b"
+ "sha256:2de15b13836fa3522815a933bd9c887c77f4868071043349f94f1b896c1bcfb8",
+ "sha256:38bb09d0277117f76507c8728d9a5156f09a47ac5175bb8072513859d19a593b"
],
"index": "pypi",
- "version": "==0.16.0"
+ "version": "==0.16.2"
},
"six": {
"hashes": [
@@ -632,13 +642,21 @@
"index": "pypi",
"version": "==3.3.0"
},
+ "typing-extensions": {
+ "hashes": [
+ "sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5",
+ "sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae",
+ "sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392"
+ ],
+ "version": "==3.7.4.2"
+ },
"urllib3": {
"hashes": [
- "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527",
- "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"
+ "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a",
+ "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
- "version": "==1.25.9"
+ "version": "==1.25.10"
},
"websockets": {
"hashes": [
@@ -670,26 +688,26 @@
},
"yarl": {
"hashes": [
- "sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce",
- "sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6",
- "sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce",
- "sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae",
- "sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d",
- "sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f",
- "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b",
- "sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b",
- "sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb",
- "sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462",
- "sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea",
- "sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70",
- "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1",
- "sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a",
- "sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b",
- "sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080",
- "sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2"
+ "sha256:1707230e1ea48ea06a3e20acb4ce05a38d2465bd9566c21f48f6212a88e47536",
+ "sha256:1f269e8e6676193a94635399a77c9059e1826fb6265c9204c9e5a8ccd36006e1",
+ "sha256:2657716c1fc998f5f2675c0ee6ce91282e0da0ea9e4a94b584bb1917e11c1559",
+ "sha256:431faa6858f0ea323714d8b7b4a7da1db2eeb9403607f0eaa3800ab2c5a4b627",
+ "sha256:5bbcb195da7de57f4508b7508c33f7593e9516e27732d08b9aad8586c7b8c384",
+ "sha256:5c82f5b1499342339f22c83b97dbe2b8a09e47163fab86cd934a8dd46620e0fb",
+ "sha256:5d410f69b4f92c5e1e2a8ffb73337cd8a274388c6975091735795588a538e605",
+ "sha256:66b4f345e9573e004b1af184bc00431145cf5e089a4dcc1351505c1f5750192c",
+ "sha256:875b2a741ce0208f3b818008a859ab5d0f461e98a32bbdc6af82231a9e761c55",
+ "sha256:9a3266b047d15e78bba38c8455bf68b391c040231ca5965ef867f7cbbc60bde5",
+ "sha256:9a592c4aa642249e9bdaf76897d90feeb08118626b363a6be8788a9b300274b5",
+ "sha256:a1772068401d425e803999dada29a6babf041786e08be5e79ef63c9ecc4c9575",
+ "sha256:b065a5c3e050395ae563019253cc6c769a50fd82d7fa92d07476273521d56b7c",
+ "sha256:b325fefd574ebef50e391a1072d1712a60348ca29c183e1d546c9d87fec2cd32",
+ "sha256:cf5eb664910d759bbae0b76d060d6e21f8af5098242d66c448bbebaf2a7bfa70",
+ "sha256:f058b6541477022c7b54db37229f87dacf3b565de4f901ff5a0a78556a174fea",
+ "sha256:f5cfed0766837303f688196aa7002730d62c5cc802d98c6395ea1feb87252727"
],
"markers": "python_version >= '3.5'",
- "version": "==1.4.2"
+ "version": "==1.5.0"
}
},
"develop": {
@@ -718,43 +736,43 @@
},
"coverage": {
"hashes": [
- "sha256:0fc4e0d91350d6f43ef6a61f64a48e917637e1dcfcba4b4b7d543c628ef82c2d",
- "sha256:10f2a618a6e75adf64329f828a6a5b40244c1c50f5ef4ce4109e904e69c71bd2",
- "sha256:12eaccd86d9a373aea59869bc9cfa0ab6ba8b1477752110cb4c10d165474f703",
- "sha256:1874bdc943654ba46d28f179c1846f5710eda3aeb265ff029e0ac2b52daae404",
- "sha256:1dcebae667b73fd4aa69237e6afb39abc2f27520f2358590c1b13dd90e32abe7",
- "sha256:1e58fca3d9ec1a423f1b7f2aa34af4f733cbfa9020c8fe39ca451b6071237405",
- "sha256:214eb2110217f2636a9329bc766507ab71a3a06a8ea30cdeebb47c24dce5972d",
- "sha256:25fe74b5b2f1b4abb11e103bb7984daca8f8292683957d0738cd692f6a7cc64c",
- "sha256:32ecee61a43be509b91a526819717d5e5650e009a8d5eda8631a59c721d5f3b6",
- "sha256:3740b796015b889e46c260ff18b84683fa2e30f0f75a171fb10d2bf9fb91fc70",
- "sha256:3b2c34690f613525672697910894b60d15800ac7e779fbd0fccf532486c1ba40",
- "sha256:41d88736c42f4a22c494c32cc48a05828236e37c991bd9760f8923415e3169e4",
- "sha256:42fa45a29f1059eda4d3c7b509589cc0343cd6bbf083d6118216830cd1a51613",
- "sha256:4bb385a747e6ae8a65290b3df60d6c8a692a5599dc66c9fa3520e667886f2e10",
- "sha256:509294f3e76d3f26b35083973fbc952e01e1727656d979b11182f273f08aa80b",
- "sha256:5c74c5b6045969b07c9fb36b665c9cac84d6c174a809fc1b21bdc06c7836d9a0",
- "sha256:60a3d36297b65c7f78329b80120f72947140f45b5c7a017ea730f9112b40f2ec",
- "sha256:6f91b4492c5cde83bfe462f5b2b997cdf96a138f7c58b1140f05de5751623cf1",
- "sha256:7403675df5e27745571aba1c957c7da2dacb537c21e14007ec3a417bf31f7f3d",
- "sha256:87bdc8135b8ee739840eee19b184804e5d57f518578ffc797f5afa2c3c297913",
- "sha256:8a3decd12e7934d0254939e2bf434bf04a5890c5bf91a982685021786a08087e",
- "sha256:9702e2cb1c6dec01fb8e1a64c015817c0800a6eca287552c47a5ee0ebddccf62",
- "sha256:a4d511012beb967a39580ba7d2549edf1e6865a33e5fe51e4dce550522b3ac0e",
- "sha256:bbb387811f7a18bdc61a2ea3d102be0c7e239b0db9c83be7bfa50f095db5b92a",
- "sha256:bfcc811883699ed49afc58b1ed9f80428a18eb9166422bce3c31a53dba00fd1d",
- "sha256:c32aa13cc3fe86b0f744dfe35a7f879ee33ac0a560684fef0f3e1580352b818f",
- "sha256:ca63dae130a2e788f2b249200f01d7fa240f24da0596501d387a50e57aa7075e",
- "sha256:d54d7ea74cc00482a2410d63bf10aa34ebe1c49ac50779652106c867f9986d6b",
- "sha256:d67599521dff98ec8c34cd9652cbcfe16ed076a2209625fca9dc7419b6370e5c",
- "sha256:d82db1b9a92cb5c67661ca6616bdca6ff931deceebb98eecbd328812dab52032",
- "sha256:d9ad0a988ae20face62520785ec3595a5e64f35a21762a57d115dae0b8fb894a",
- "sha256:ebf2431b2d457ae5217f3a1179533c456f3272ded16f8ed0b32961a6d90e38ee",
- "sha256:ed9a21502e9223f563e071759f769c3d6a2e1ba5328c31e86830368e8d78bc9c",
- "sha256:f50632ef2d749f541ca8e6c07c9928a37f87505ce3a9f20c8446ad310f1aa87b"
- ],
- "index": "pypi",
- "version": "==5.2"
+ "sha256:098a703d913be6fbd146a8c50cc76513d726b022d170e5e98dc56d958fd592fb",
+ "sha256:16042dc7f8e632e0dcd5206a5095ebd18cb1d005f4c89694f7f8aafd96dd43a3",
+ "sha256:1adb6be0dcef0cf9434619d3b892772fdb48e793300f9d762e480e043bd8e716",
+ "sha256:27ca5a2bc04d68f0776f2cdcb8bbd508bbe430a7bf9c02315cd05fb1d86d0034",
+ "sha256:28f42dc5172ebdc32622a2c3f7ead1b836cdbf253569ae5673f499e35db0bac3",
+ "sha256:2fcc8b58953d74d199a1a4d633df8146f0ac36c4e720b4a1997e9b6327af43a8",
+ "sha256:304fbe451698373dc6653772c72c5d5e883a4aadaf20343592a7abb2e643dae0",
+ "sha256:30bc103587e0d3df9e52cd9da1dd915265a22fad0b72afe54daf840c984b564f",
+ "sha256:40f70f81be4d34f8d491e55936904db5c527b0711b2a46513641a5729783c2e4",
+ "sha256:4186fc95c9febeab5681bc3248553d5ec8c2999b8424d4fc3a39c9cba5796962",
+ "sha256:46794c815e56f1431c66d81943fa90721bb858375fb36e5903697d5eef88627d",
+ "sha256:4869ab1c1ed33953bb2433ce7b894a28d724b7aa76c19b11e2878034a4e4680b",
+ "sha256:4f6428b55d2916a69f8d6453e48a505c07b2245653b0aa9f0dee38785939f5e4",
+ "sha256:52f185ffd3291196dc1aae506b42e178a592b0b60a8610b108e6ad892cfc1bb3",
+ "sha256:538f2fd5eb64366f37c97fdb3077d665fa946d2b6d95447622292f38407f9258",
+ "sha256:64c4f340338c68c463f1b56e3f2f0423f7b17ba6c3febae80b81f0e093077f59",
+ "sha256:675192fca634f0df69af3493a48224f211f8db4e84452b08d5fcebb9167adb01",
+ "sha256:700997b77cfab016533b3e7dbc03b71d33ee4df1d79f2463a318ca0263fc29dd",
+ "sha256:8505e614c983834239f865da2dd336dcf9d72776b951d5dfa5ac36b987726e1b",
+ "sha256:962c44070c281d86398aeb8f64e1bf37816a4dfc6f4c0f114756b14fc575621d",
+ "sha256:9e536783a5acee79a9b308be97d3952b662748c4037b6a24cbb339dc7ed8eb89",
+ "sha256:9ea749fd447ce7fb1ac71f7616371f04054d969d412d37611716721931e36efd",
+ "sha256:a34cb28e0747ea15e82d13e14de606747e9e484fb28d63c999483f5d5188e89b",
+ "sha256:a3ee9c793ffefe2944d3a2bd928a0e436cd0ac2d9e3723152d6fd5398838ce7d",
+ "sha256:aab75d99f3f2874733946a7648ce87a50019eb90baef931698f96b76b6769a46",
+ "sha256:b1ed2bdb27b4c9fc87058a1cb751c4df8752002143ed393899edb82b131e0546",
+ "sha256:b360d8fd88d2bad01cb953d81fd2edd4be539df7bfec41e8753fe9f4456a5082",
+ "sha256:b8f58c7db64d8f27078cbf2a4391af6aa4e4767cc08b37555c4ae064b8558d9b",
+ "sha256:c1bbb628ed5192124889b51204de27c575b3ffc05a5a91307e7640eff1d48da4",
+ "sha256:c2ff24df02a125b7b346c4c9078c8936da06964cc2d276292c357d64378158f8",
+ "sha256:c890728a93fffd0407d7d37c1e6083ff3f9f211c83b4316fae3778417eab9811",
+ "sha256:c96472b8ca5dc135fb0aa62f79b033f02aa434fb03a8b190600a5ae4102df1fd",
+ "sha256:ce7866f29d3025b5b34c2e944e66ebef0d92e4a4f2463f7266daa03a1332a651",
+ "sha256:e26c993bd4b220429d4ec8c1468eca445a4064a61c74ca08da7429af9bc53bb0"
+ ],
+ "index": "pypi",
+ "version": "==5.2.1"
},
"distlib": {
"hashes": [
@@ -780,11 +798,11 @@
},
"flake8-annotations": {
"hashes": [
- "sha256:babc81a17a5f1a63464195917e20d3e8663fb712b3633d4522dbfc407cff31b3",
- "sha256:fcd833b415726a7a374922c95a5c47a7a4d8ea71cb4a586369c665e7476146e1"
+ "sha256:7816a5d8f65ffdf37b8e21e5b17e0fd1e492aa92638573276de066e889a22b26",
+ "sha256:8d18db74a750dd97f40b483cc3ef80d07d03f687525bad8fd83365dcd3bfd414"
],
"index": "pypi",
- "version": "==2.2.0"
+ "version": "==2.3.0"
},
"flake8-bugbear": {
"hashes": [
@@ -842,11 +860,11 @@
},
"identify": {
"hashes": [
- "sha256:c4d07f2b979e3931894170a9e0d4b8281e6905ea6d018c326f7ffefaf20db680",
- "sha256:dac33eff90d57164e289fb20bf4e131baef080947ee9bf45efcd0da8d19064bf"
+ "sha256:110ed090fec6bce1aabe3c72d9258a9de82207adeaa5a05cd75c635880312f9a",
+ "sha256:ccd88716b890ecbe10920659450a635d2d25de499b9a638525a48b48261d989b"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
- "version": "==1.4.21"
+ "version": "==1.4.25"
},
"mccabe": {
"hashes": [
@@ -950,11 +968,11 @@
},
"virtualenv": {
"hashes": [
- "sha256:c11a475400e98450403c0364eb3a2d25d42f71cf1493da64390487b666de4324",
- "sha256:e10cc66f40cbda459720dfe1d334c4dc15add0d80f09108224f171006a97a172"
+ "sha256:688a61d7976d82b92f7906c367e83bb4b3f0af96f8f75bfcd3da95608fe8ac6c",
+ "sha256:8f582a030156282a9ee9d319984b759a232b07f86048c1d6a9e394afa44e78c8"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
- "version": "==20.0.26"
+ "version": "==20.0.28"
}
}
}
diff --git a/bot/__main__.py b/bot/__main__.py
index 5382f5502..f698b5662 100644
--- a/bot/__main__.py
+++ b/bot/__main__.py
@@ -34,21 +34,20 @@ bot = Bot(
)
# Internal/debug
+bot.load_extension("bot.cogs.config_verifier")
bot.load_extension("bot.cogs.error_handler")
bot.load_extension("bot.cogs.filtering")
bot.load_extension("bot.cogs.logging")
bot.load_extension("bot.cogs.security")
-bot.load_extension("bot.cogs.config_verifier")
# Commands, etc
bot.load_extension("bot.cogs.antimalware")
bot.load_extension("bot.cogs.antispam")
bot.load_extension("bot.cogs.bot")
bot.load_extension("bot.cogs.clean")
+bot.load_extension("bot.cogs.doc")
bot.load_extension("bot.cogs.extensions")
bot.load_extension("bot.cogs.help")
-
-bot.load_extension("bot.cogs.doc")
bot.load_extension("bot.cogs.verification")
# Feature cogs
@@ -57,11 +56,12 @@ bot.load_extension("bot.cogs.defcon")
bot.load_extension("bot.cogs.dm_relay")
bot.load_extension("bot.cogs.duck_pond")
bot.load_extension("bot.cogs.eval")
+bot.load_extension("bot.cogs.filter_lists")
bot.load_extension("bot.cogs.information")
bot.load_extension("bot.cogs.jams")
bot.load_extension("bot.cogs.moderation")
-bot.load_extension("bot.cogs.python_news")
bot.load_extension("bot.cogs.off_topic_names")
+bot.load_extension("bot.cogs.python_news")
bot.load_extension("bot.cogs.reddit")
bot.load_extension("bot.cogs.reminders")
bot.load_extension("bot.cogs.site")
diff --git a/bot/bot.py b/bot/bot.py
index 313652d11..4492feaa9 100644
--- a/bot/bot.py
+++ b/bot/bot.py
@@ -2,7 +2,8 @@ import asyncio
import logging
import socket
import warnings
-from typing import Optional
+from collections import defaultdict
+from typing import Dict, Optional
import aiohttp
import aioredis
@@ -34,6 +35,7 @@ class Bot(commands.Bot):
self.redis_ready = asyncio.Event()
self.redis_closed = False
self.api_client = api.APIClient(loop=self.loop)
+ self.filter_list_cache = defaultdict(dict)
self._connector = None
self._resolver = None
@@ -49,6 +51,13 @@ class Bot(commands.Bot):
self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot")
+ async def _cache_filter_list_data(self) -> None:
+ """Cache all the data in the FilterList on the site."""
+ full_cache = await self.api_client.get('bot/filter-lists')
+
+ for item in full_cache:
+ self.insert_item_into_filter_list_cache(item)
+
async def _create_redis_session(self) -> None:
"""
Create the Redis connection pool, and then open the redis event gate.
@@ -73,6 +82,49 @@ class Bot(commands.Bot):
self.redis_closed = False
self.redis_ready.set()
+ def _recreate(self) -> None:
+ """Re-create the connector, aiohttp session, the APIClient and the Redis session."""
+ # Use asyncio for DNS resolution instead of threads so threads aren't spammed.
+ # Doesn't seem to have any state with regards to being closed, so no need to worry?
+ self._resolver = aiohttp.AsyncResolver()
+
+ # Its __del__ does send a warning but it doesn't always show up for some reason.
+ if self._connector and not self._connector._closed:
+ log.warning(
+ "The previous connector was not closed; it will remain open and be overwritten"
+ )
+
+ if self.redis_session and not self.redis_session.closed:
+ log.warning(
+ "The previous redis pool was not closed; it will remain open and be overwritten"
+ )
+
+ # Create the redis session
+ self.loop.create_task(self._create_redis_session())
+
+ # Use AF_INET as its socket family to prevent HTTPS related problems both locally
+ # and in production.
+ self._connector = aiohttp.TCPConnector(
+ resolver=self._resolver,
+ family=socket.AF_INET,
+ )
+
+ # Client.login() will call HTTPClient.static_login() which will create a session using
+ # this connector attribute.
+ self.http.connector = self._connector
+
+ # Its __del__ does send a warning but it doesn't always show up for some reason.
+ if self.http_session and not self.http_session.closed:
+ log.warning(
+ "The previous session was not closed; it will remain open and be overwritten"
+ )
+
+ self.http_session = aiohttp.ClientSession(connector=self._connector)
+ self.api_client.recreate(force=True, connector=self._connector)
+
+ # Build the FilterList cache
+ self.loop.create_task(self._cache_filter_list_data())
+
def add_cog(self, cog: commands.Cog) -> None:
"""Adds a "cog" to the bot and logs the operation."""
super().add_cog(cog)
@@ -113,52 +165,25 @@ class Bot(commands.Bot):
self.redis_ready.clear()
await self.redis_session.wait_closed()
+ def insert_item_into_filter_list_cache(self, item: Dict[str, str]) -> None:
+ """Add an item to the bots filter_list_cache."""
+ type_ = item["type"]
+ allowed = item["allowed"]
+ content = item["content"]
+
+ self.filter_list_cache[f"{type_}.{allowed}"][content] = {
+ "id": item["id"],
+ "comment": item["comment"],
+ "created_at": item["created_at"],
+ "updated_at": item["updated_at"],
+ }
+
async def login(self, *args, **kwargs) -> None:
"""Re-create the connector and set up sessions before logging into Discord."""
self._recreate()
await self.stats.create_socket()
await super().login(*args, **kwargs)
- def _recreate(self) -> None:
- """Re-create the connector, aiohttp session, the APIClient and the Redis session."""
- # Use asyncio for DNS resolution instead of threads so threads aren't spammed.
- # Doesn't seem to have any state with regards to being closed, so no need to worry?
- self._resolver = aiohttp.AsyncResolver()
-
- # Its __del__ does send a warning but it doesn't always show up for some reason.
- if self._connector and not self._connector._closed:
- log.warning(
- "The previous connector was not closed; it will remain open and be overwritten"
- )
-
- if self.redis_session and not self.redis_session.closed:
- log.warning(
- "The previous redis pool was not closed; it will remain open and be overwritten"
- )
-
- # Create the redis session
- self.loop.create_task(self._create_redis_session())
-
- # Use AF_INET as its socket family to prevent HTTPS related problems both locally
- # and in production.
- self._connector = aiohttp.TCPConnector(
- resolver=self._resolver,
- family=socket.AF_INET,
- )
-
- # Client.login() will call HTTPClient.static_login() which will create a session using
- # this connector attribute.
- self.http.connector = self._connector
-
- # Its __del__ does send a warning but it doesn't always show up for some reason.
- if self.http_session and not self.http_session.closed:
- log.warning(
- "The previous session was not closed; it will remain open and be overwritten"
- )
-
- self.http_session = aiohttp.ClientSession(connector=self._connector)
- self.api_client.recreate(force=True, connector=self._connector)
-
async def on_guild_available(self, guild: discord.Guild) -> None:
"""
Set the internal guild available event when constants.Guild.id becomes available.
diff --git a/bot/cogs/antimalware.py b/bot/cogs/antimalware.py
index ea257442e..c76bd2c60 100644
--- a/bot/cogs/antimalware.py
+++ b/bot/cogs/antimalware.py
@@ -6,7 +6,7 @@ from discord import Embed, Message, NotFound
from discord.ext.commands import Cog
from bot.bot import Bot
-from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES, URLs
+from bot.constants import Channels, STAFF_ROLES, URLs
log = logging.getLogger(__name__)
@@ -27,7 +27,7 @@ TXT_EMBED_DESCRIPTION = (
DISALLOWED_EMBED_DESCRIPTION = (
"It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). "
- f"We currently allow the following file types: **{', '.join(AntiMalwareConfig.whitelist)}**.\n\n"
+ "We currently allow the following file types: **{joined_whitelist}**.\n\n"
"Feel free to ask in {meta_channel_mention} if you think this is a mistake."
)
@@ -38,6 +38,16 @@ class AntiMalware(Cog):
def __init__(self, bot: Bot):
self.bot = bot
+ def _get_whitelisted_file_formats(self) -> list:
+ """Get the file formats currently on the whitelist."""
+ return self.bot.filter_list_cache['FILE_FORMAT.True'].keys()
+
+ def _get_disallowed_extensions(self, message: Message) -> t.Iterable[str]:
+ """Get an iterable containing all the disallowed extensions of attachments."""
+ file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments}
+ extensions_blocked = file_extensions - set(self._get_whitelisted_file_formats())
+ return extensions_blocked
+
@Cog.listener()
async def on_message(self, message: Message) -> None:
"""Identify messages with prohibited attachments."""
@@ -51,7 +61,7 @@ class AntiMalware(Cog):
return
embed = Embed()
- extensions_blocked = self.get_disallowed_extensions(message)
+ extensions_blocked = self._get_disallowed_extensions(message)
blocked_extensions_str = ', '.join(extensions_blocked)
if ".py" in extensions_blocked:
# Short-circuit on *.py files to provide a pastebin link
@@ -63,6 +73,7 @@ class AntiMalware(Cog):
elif extensions_blocked:
meta_channel = self.bot.get_channel(Channels.meta)
embed.description = DISALLOWED_EMBED_DESCRIPTION.format(
+ joined_whitelist=', '.join(self._get_whitelisted_file_formats()),
blocked_extensions_str=blocked_extensions_str,
meta_channel_mention=meta_channel.mention,
)
@@ -81,13 +92,6 @@ class AntiMalware(Cog):
except NotFound:
log.info(f"Tried to delete message `{message.id}`, but message could not be found.")
- @classmethod
- def get_disallowed_extensions(cls, message: Message) -> t.Iterable[str]:
- """Get an iterable containing all the disallowed extensions of attachments."""
- file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments}
- extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist)
- return extensions_blocked
-
def setup(bot: Bot) -> None:
"""Load the AntiMalware cog."""
diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py
index 233851e41..f9d4de638 100644
--- a/bot/cogs/error_handler.py
+++ b/bot/cogs/error_handler.py
@@ -2,12 +2,13 @@ import contextlib
import logging
import typing as t
+from discord import Embed
from discord.ext.commands import Cog, Context, errors
from sentry_sdk import push_scope
from bot.api import ResponseCodeError
from bot.bot import Bot
-from bot.constants import Channels
+from bot.constants import Channels, Colours
from bot.converters import TagNameConverter
from bot.utils.checks import InWhitelistCheckFailure
@@ -20,6 +21,14 @@ class ErrorHandler(Cog):
def __init__(self, bot: Bot):
self.bot = bot
+ def _get_error_embed(self, title: str, body: str) -> Embed:
+ """Return an embed that contains the exception."""
+ return Embed(
+ title=title,
+ colour=Colours.soft_red,
+ description=body
+ )
+
@Cog.listener()
async def on_command_error(self, ctx: Context, e: errors.CommandError) -> None:
"""
@@ -162,25 +171,34 @@ class ErrorHandler(Cog):
prepared_help_command = self.get_help_command(ctx)
if isinstance(e, errors.MissingRequiredArgument):
- await ctx.send(f"Missing required argument `{e.param.name}`.")
+ embed = self._get_error_embed("Missing required argument", e.param.name)
+ await ctx.send(embed=embed)
await prepared_help_command
self.bot.stats.incr("errors.missing_required_argument")
elif isinstance(e, errors.TooManyArguments):
- await ctx.send("Too many arguments provided.")
+ embed = self._get_error_embed("Too many arguments", str(e))
+ await ctx.send(embed=embed)
await prepared_help_command
self.bot.stats.incr("errors.too_many_arguments")
elif isinstance(e, errors.BadArgument):
- await ctx.send("Bad argument: Please double-check your input arguments and try again.\n")
+ embed = self._get_error_embed("Bad argument", str(e))
+ await ctx.send(embed=embed)
await prepared_help_command
self.bot.stats.incr("errors.bad_argument")
elif isinstance(e, errors.BadUnionArgument):
- await ctx.send(f"Bad argument: {e}\n```{e.errors[-1]}```")
+ embed = self._get_error_embed("Bad argument", f"{e}\n{e.errors[-1]}")
+ await ctx.send(embed=embed)
self.bot.stats.incr("errors.bad_union_argument")
elif isinstance(e, errors.ArgumentParsingError):
- await ctx.send(f"Argument parsing error: {e}")
+ embed = self._get_error_embed("Argument parsing error", str(e))
+ await ctx.send(embed=embed)
self.bot.stats.incr("errors.argument_parsing_error")
else:
- await ctx.send("Something about your input seems off. Check the arguments:")
+ embed = self._get_error_embed(
+ "Input error",
+ "Something about your input seems off. Check the arguments and try again."
+ )
+ await ctx.send(embed=embed)
await prepared_help_command
self.bot.stats.incr("errors.other_user_input_error")
diff --git a/bot/cogs/filter_lists.py b/bot/cogs/filter_lists.py
new file mode 100644
index 000000000..8aa5a0a08
--- /dev/null
+++ b/bot/cogs/filter_lists.py
@@ -0,0 +1,250 @@
+import logging
+from typing import Optional
+
+from discord import Colour, Embed
+from discord.ext.commands import BadArgument, Cog, Context, IDConverter, group
+
+from bot import constants
+from bot.api import ResponseCodeError
+from bot.bot import Bot
+from bot.converters import ValidDiscordServerInvite, ValidFilterListType
+from bot.pagination import LinePaginator
+from bot.utils.checks import with_role_check
+
+log = logging.getLogger(__name__)
+
+
+class FilterLists(Cog):
+ """Commands for blacklisting and whitelisting things."""
+
+ methods_with_filterlist_types = [
+ "allow_add",
+ "allow_delete",
+ "allow_get",
+ "deny_add",
+ "deny_delete",
+ "deny_get",
+ ]
+
+ def __init__(self, bot: Bot) -> None:
+ self.bot = bot
+ self.bot.loop.create_task(self._amend_docstrings())
+
+ async def _amend_docstrings(self) -> None:
+ """Add the valid FilterList types to the docstrings, so they'll appear in !help invocations."""
+ await self.bot.wait_until_guild_available()
+
+ # Add valid filterlist types to the docstrings
+ valid_types = await ValidFilterListType.get_valid_types(self.bot)
+ valid_types = [f"`{type_.lower()}`" for type_ in valid_types]
+
+ for method_name in self.methods_with_filterlist_types:
+ command = getattr(self, method_name)
+ command.help = (
+ f"{command.help}\n\nValid **list_type** values are {', '.join(valid_types)}."
+ )
+
+ async def _add_data(
+ self,
+ ctx: Context,
+ allowed: bool,
+ list_type: ValidFilterListType,
+ content: str,
+ comment: Optional[str] = None,
+ ) -> None:
+ """Add an item to a filterlist."""
+ allow_type = "whitelist" if allowed else "blacklist"
+
+ # If this is a server invite, we gotta validate it.
+ if list_type == "GUILD_INVITE":
+ guild_data = await self._validate_guild_invite(ctx, content)
+ content = guild_data.get("id")
+
+ # Unless the user has specified another comment, let's
+ # use the server name as the comment so that the list
+ # of guild IDs will be more easily readable when we
+ # display it.
+ if not comment:
+ comment = guild_data.get("name")
+
+ # If it's a file format, let's make sure it has a leading dot.
+ elif list_type == "FILE_FORMAT" and not content.startswith("."):
+ content = f".{content}"
+
+ # Try to add the item to the database
+ log.trace(f"Trying to add the {content} item to the {list_type} {allow_type}")
+ payload = {
+ "allowed": allowed,
+ "type": list_type,
+ "content": content,
+ "comment": comment,
+ }
+
+ try:
+ item = await self.bot.api_client.post(
+ "bot/filter-lists",
+ json=payload
+ )
+ except ResponseCodeError as e:
+ if e.status == 400:
+ await ctx.message.add_reaction("❌")
+ log.debug(
+ f"{ctx.author} tried to add data to a {allow_type}, but the API returned 400, "
+ "probably because the request violated the UniqueConstraint."
+ )
+ raise BadArgument(
+ f"Unable to add the item to the {allow_type}. "
+ "The item probably already exists. Keep in mind that a "
+ "blacklist and a whitelist for the same item cannot co-exist, "
+ "and we do not permit any duplicates."
+ )
+ raise
+
+ # Insert the item into the cache
+ self.bot.insert_item_into_filter_list_cache(item)
+ await ctx.message.add_reaction("✅")
+
+ async def _delete_data(self, ctx: Context, allowed: bool, list_type: ValidFilterListType, content: str) -> None:
+ """Remove an item from a filterlist."""
+ allow_type = "whitelist" if allowed else "blacklist"
+
+ # If this is a server invite, we need to convert it.
+ if list_type == "GUILD_INVITE" and not IDConverter()._get_id_match(content):
+ guild_data = await self._validate_guild_invite(ctx, content)
+ content = guild_data.get("id")
+
+ # If it's a file format, let's make sure it has a leading dot.
+ elif list_type == "FILE_FORMAT" and not content.startswith("."):
+ content = f".{content}"
+
+ # Find the content and delete it.
+ log.trace(f"Trying to delete the {content} item from the {list_type} {allow_type}")
+ item = self.bot.filter_list_cache[f"{list_type}.{allowed}"].get(content)
+
+ if item is not None:
+ try:
+ await self.bot.api_client.delete(
+ f"bot/filter-lists/{item['id']}"
+ )
+ del self.bot.filter_list_cache[f"{list_type}.{allowed}"][content]
+ await ctx.message.add_reaction("✅")
+ except ResponseCodeError as e:
+ log.debug(
+ f"{ctx.author} tried to delete an item with the id {item['id']}, but "
+ f"the API raised an unexpected error: {e}"
+ )
+ await ctx.message.add_reaction("❌")
+ else:
+ await ctx.message.add_reaction("❌")
+
+ async def _list_all_data(self, ctx: Context, allowed: bool, list_type: ValidFilterListType) -> None:
+ """Paginate and display all items in a filterlist."""
+ allow_type = "whitelist" if allowed else "blacklist"
+ result = self.bot.filter_list_cache[f"{list_type}.{allowed}"]
+
+ # Build a list of lines we want to show in the paginator
+ lines = []
+ for content, metadata in result.items():
+ line = f"• `{content}`"
+
+ if comment := metadata.get("comment"):
+ line += f" - {comment}"
+
+ lines.append(line)
+ lines = sorted(lines)
+
+ # Build the embed
+ list_type_plural = list_type.lower().replace("_", " ").title() + "s"
+ embed = Embed(
+ title=f"{allow_type.title()}ed {list_type_plural} ({len(result)} total)",
+ colour=Colour.blue()
+ )
+ log.trace(f"Trying to list {len(result)} items from the {list_type.lower()} {allow_type}")
+
+ if result:
+ await LinePaginator.paginate(lines, ctx, embed, max_lines=15, empty=False)
+ else:
+ embed.description = "Hmmm, seems like there's nothing here yet."
+ await ctx.send(embed=embed)
+ await ctx.message.add_reaction("❌")
+
+ @staticmethod
+ async def _validate_guild_invite(ctx: Context, invite: str) -> dict:
+ """
+ Validates a guild invite, and returns the guild info as a dict.
+
+ Will raise a BadArgument if the guild invite is invalid.
+ """
+ log.trace(f"Attempting to validate whether or not {invite} is a guild invite.")
+ validator = ValidDiscordServerInvite()
+ guild_data = await validator.convert(ctx, invite)
+
+ # If we make it this far without raising a BadArgument, the invite is
+ # valid. Let's return a dict of guild information.
+ log.trace(f"{invite} validated as server invite. Converting to ID.")
+ return guild_data
+
+ @group(aliases=("allowlist", "allow", "al", "wl"))
+ async def whitelist(self, ctx: Context) -> None:
+ """Group for whitelisting commands."""
+ if not ctx.invoked_subcommand:
+ await ctx.send_help(ctx.command)
+
+ @group(aliases=("denylist", "deny", "bl", "dl"))
+ async def blacklist(self, ctx: Context) -> None:
+ """Group for blacklisting commands."""
+ if not ctx.invoked_subcommand:
+ await ctx.send_help(ctx.command)
+
+ @whitelist.command(name="add", aliases=("a", "set"))
+ async def allow_add(
+ self,
+ ctx: Context,
+ list_type: ValidFilterListType,
+ content: str,
+ *,
+ comment: Optional[str] = None,
+ ) -> None:
+ """Add an item to the specified allowlist."""
+ await self._add_data(ctx, True, list_type, content, comment)
+
+ @blacklist.command(name="add", aliases=("a", "set"))
+ async def deny_add(
+ self,
+ ctx: Context,
+ list_type: ValidFilterListType,
+ content: str,
+ *,
+ comment: Optional[str] = None,
+ ) -> None:
+ """Add an item to the specified denylist."""
+ await self._add_data(ctx, False, list_type, content, comment)
+
+ @whitelist.command(name="remove", aliases=("delete", "rm",))
+ async def allow_delete(self, ctx: Context, list_type: ValidFilterListType, content: str) -> None:
+ """Remove an item from the specified allowlist."""
+ await self._delete_data(ctx, True, list_type, content)
+
+ @blacklist.command(name="remove", aliases=("delete", "rm",))
+ async def deny_delete(self, ctx: Context, list_type: ValidFilterListType, content: str) -> None:
+ """Remove an item from the specified denylist."""
+ await self._delete_data(ctx, False, list_type, content)
+
+ @whitelist.command(name="get", aliases=("list", "ls", "fetch", "show"))
+ async def allow_get(self, ctx: Context, list_type: ValidFilterListType) -> None:
+ """Get the contents of a specified allowlist."""
+ await self._list_all_data(ctx, True, list_type)
+
+ @blacklist.command(name="get", aliases=("list", "ls", "fetch", "show"))
+ async def deny_get(self, ctx: Context, list_type: ValidFilterListType) -> None:
+ """Get the contents of a specified denylist."""
+ await self._list_all_data(ctx, False, list_type)
+
+ def cog_check(self, ctx: Context) -> bool:
+ """Only allow moderators to invoke the commands in this cog."""
+ return with_role_check(ctx, *constants.MODERATION_ROLES)
+
+
+def setup(bot: Bot) -> None:
+ """Load the FilterLists cog."""
+ bot.add_cog(FilterLists(bot))
diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py
index 29aac812f..64afd184d 100644
--- a/bot/cogs/filtering.py
+++ b/bot/cogs/filtering.py
@@ -18,44 +18,18 @@ from bot.constants import (
Filter, Icons, URLs
)
from bot.utils.redis_cache import RedisCache
+from bot.utils.regex import INVITE_RE
from bot.utils.scheduling import Scheduler
log = logging.getLogger(__name__)
-INVITE_RE = re.compile(
- r"(?:discord(?:[\.,]|dot)gg|" # Could be discord.gg/
- r"discord(?:[\.,]|dot)com(?:\/|slash)invite|" # or discord.com/invite/
- r"discordapp(?:[\.,]|dot)com(?:\/|slash)invite|" # or discordapp.com/invite/
- r"discord(?:[\.,]|dot)me|" # or discord.me
- r"discord(?:[\.,]|dot)io" # or discord.io.
- r")(?:[\/]|slash)" # / or 'slash'
- r"([a-zA-Z0-9]+)", # the invite code itself
- flags=re.IGNORECASE
-)
-
+# Regular expressions
SPOILER_RE = re.compile(r"(\|\|.+?\|\|)", re.DOTALL)
URL_RE = re.compile(r"(https?://[^\s]+)", flags=re.IGNORECASE)
ZALGO_RE = re.compile(r"[\u0300-\u036F\u0489]")
-WORD_WATCHLIST_PATTERNS = [
- re.compile(fr'\b{expression}\b', flags=re.IGNORECASE) for expression in Filter.word_watchlist
-]
-TOKEN_WATCHLIST_PATTERNS = [
- re.compile(fr'{expression}', flags=re.IGNORECASE) for expression in Filter.token_watchlist
-]
-WATCHLIST_PATTERNS = WORD_WATCHLIST_PATTERNS + TOKEN_WATCHLIST_PATTERNS
-
+# Other constants.
DAYS_BETWEEN_ALERTS = 3
-
-
-def expand_spoilers(text: str) -> str:
- """Return a string containing all interpretations of a spoilered message."""
- split_text = SPOILER_RE.split(text)
- return ''.join(
- split_text[0::2] + split_text[1::2] + split_text
- )
-
-
OFFENSIVE_MSG_DELETE_TIME = timedelta(days=Filter.offensive_msg_delete_days)
@@ -125,6 +99,18 @@ class Filtering(Cog):
self.bot.loop.create_task(self.reschedule_offensive_msg_deletion())
+ def _get_filterlist_items(self, list_type: str, *, allowed: bool) -> list:
+ """Fetch items from the filter_list_cache."""
+ return self.bot.filter_list_cache[f"{list_type.upper()}.{allowed}"].keys()
+
+ @staticmethod
+ def _expand_spoilers(text: str) -> str:
+ """Return a string containing all interpretations of a spoilered message."""
+ split_text = SPOILER_RE.split(text)
+ return ''.join(
+ split_text[0::2] + split_text[1::2] + split_text
+ )
+
@property
def mod_log(self) -> ModLog:
"""Get currently loaded ModLog cog instance."""
@@ -149,12 +135,12 @@ class Filtering(Cog):
delta = relativedelta(after.edited_at, before.edited_at).microseconds
await self._filter_message(after, delta)
- @staticmethod
- def get_name_matches(name: str) -> List[re.Match]:
+ def get_name_matches(self, name: str) -> List[re.Match]:
"""Check bad words from passed string (name). Return list of matches."""
matches = []
- for pattern in WATCHLIST_PATTERNS:
- if match := pattern.search(name):
+ watchlist_patterns = self._get_filterlist_items('filter_token', allowed=False)
+ for pattern in watchlist_patterns:
+ if match := re.search(pattern, name, flags=re.IGNORECASE):
matches.append(match)
return matches
@@ -370,14 +356,14 @@ class Filtering(Cog):
# They have no data so additional embeds can't be created for them.
if name == "filter_invites" and match is not True:
additional_embeds = []
- for invite, data in match.items():
+ for _, data in match.items():
embed = discord.Embed(description=(
f"**Members:**\n{data['members']}\n"
f"**Active:**\n{data['active']}"
))
embed.set_author(name=data["name"])
embed.set_thumbnail(url=data["icon"])
- embed.set_footer(text=f"Guild Invite Code: {invite}")
+ embed.set_footer(text=f"Guild ID: {data['id']}")
additional_embeds.append(embed)
additional_embeds_msg = "For the following guild(s):"
@@ -403,8 +389,7 @@ class Filtering(Cog):
and not msg.author.bot # Author not a bot
)
- @staticmethod
- async def _has_watch_regex_match(text: str) -> Union[bool, re.Match]:
+ async def _has_watch_regex_match(self, text: str) -> Union[bool, re.Match]:
"""
Return True if `text` matches any regex from `word_watchlist` or `token_watchlist` configs.
@@ -412,26 +397,27 @@ class Filtering(Cog):
matched as-is. Spoilers are expanded, if any, and URLs are ignored.
"""
if SPOILER_RE.search(text):
- text = expand_spoilers(text)
+ text = self._expand_spoilers(text)
# Make sure it's not a URL
if URL_RE.search(text):
return False
- for pattern in WATCHLIST_PATTERNS:
- match = pattern.search(text)
+ watchlist_patterns = self._get_filterlist_items('filter_token', allowed=False)
+ for pattern in watchlist_patterns:
+ match = re.search(pattern, text, flags=re.IGNORECASE)
if match:
return match
- @staticmethod
- async def _has_urls(text: str) -> bool:
+ async def _has_urls(self, text: str) -> bool:
"""Returns True if the text contains one of the blacklisted URLs from the config file."""
if not URL_RE.search(text):
return False
text = text.lower()
+ domain_blacklist = self._get_filterlist_items("domain_name", allowed=False)
- for url in Filter.domain_blacklist:
+ for url in domain_blacklist:
if url.lower() in text:
return True
@@ -455,7 +441,7 @@ class Filtering(Cog):
Attempts to catch some of common ways to try to cheat the system.
"""
- # Remove backslashes to prevent escape character around fuckery like
+ # Remove backslashes to prevent escape character aroundfuckery like
# discord\.gg/gdudes-pony-farm
text = text.replace("\\", "")
@@ -476,9 +462,22 @@ class Filtering(Cog):
# between invalid and expired invites
return True
- guild_id = int(guild.get("id"))
+ guild_id = guild.get("id")
+ guild_invite_whitelist = self._get_filterlist_items("guild_invite", allowed=True)
+ guild_invite_blacklist = self._get_filterlist_items("guild_invite", allowed=False)
+
+ # Is this invite allowed?
+ guild_partnered_or_verified = (
+ 'PARTNERED' in guild.get("features", [])
+ or 'VERIFIED' in guild.get("features", [])
+ )
+ invite_not_allowed = (
+ guild_id in guild_invite_blacklist # Blacklisted guilds are never permitted.
+ or guild_id not in guild_invite_whitelist # Whitelisted guilds are always permitted.
+ and not guild_partnered_or_verified # Otherwise guilds have to be Verified or Partnered.
+ )
- if guild_id not in Filter.guild_invite_whitelist:
+ if invite_not_allowed:
guild_icon_hash = guild["icon"]
guild_icon = (
"https://cdn.discordapp.com/icons/"
@@ -487,6 +486,7 @@ class Filtering(Cog):
invite_data[invite] = {
"name": guild["name"],
+ "id": guild['id'],
"icon": guild_icon,
"members": response["approximate_member_count"],
"active": response["approximate_presence_count"]
diff --git a/bot/cogs/off_topic_names.py b/bot/cogs/off_topic_names.py
index 201579a0b..ce95450e0 100644
--- a/bot/cogs/off_topic_names.py
+++ b/bot/cogs/off_topic_names.py
@@ -4,46 +4,19 @@ import logging
from datetime import datetime, timedelta
from discord import Colour, Embed
-from discord.ext.commands import BadArgument, Cog, Context, Converter, group
+from discord.ext.commands import Cog, Context, group
from bot.api import ResponseCodeError
from bot.bot import Bot
from bot.constants import Channels, MODERATION_ROLES
+from bot.converters import OffTopicName
from bot.decorators import with_role
from bot.pagination import LinePaginator
-
CHANNELS = (Channels.off_topic_0, Channels.off_topic_1, Channels.off_topic_2)
log = logging.getLogger(__name__)
-class OffTopicName(Converter):
- """A converter that ensures an added off-topic name is valid."""
-
- @staticmethod
- async def convert(ctx: Context, argument: str) -> str:
- """Attempt to replace any invalid characters with their approximate Unicode equivalent."""
- allowed_characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ!?'`-"
-
- # Chain multiple words to a single one
- argument = "-".join(argument.split())
-
- if not (2 <= len(argument) <= 96):
- raise BadArgument("Channel name must be between 2 and 96 chars long")
-
- elif not all(c.isalnum() or c in allowed_characters for c in argument):
- raise BadArgument(
- "Channel name must only consist of "
- "alphanumeric characters, minus signs or apostrophes."
- )
-
- # Replace invalid characters with unicode alternatives.
- table = str.maketrans(
- allowed_characters, '𝖠𝖡𝖢𝖣𝖤𝖥𝖦𝖧𝖨𝖩𝖪𝖫𝖬𝖭𝖮𝖯𝖰𝖱𝖲𝖳𝖴𝖵𝖶𝖷𝖸𝖹ǃ？’’-'
- )
- return argument.translate(table)
-
-
async def update_names(bot: Bot) -> None:
"""Background updater task that performs the daily channel name update."""
while True:
diff --git a/bot/constants.py b/bot/constants.py
index cf4f3f666..9d00eac36 100644
--- a/bot/constants.py
+++ b/bot/constants.py
@@ -227,10 +227,6 @@ class Filter(metaclass=YAMLGetter):
ping_everyone: bool
offensive_msg_delete_days: int
- guild_invite_whitelist: List[int]
- domain_blacklist: List[str]
- word_watchlist: List[str]
- token_watchlist: List[str]
channel_whitelist: List[int]
role_whitelist: List[int]
@@ -537,12 +533,6 @@ class AntiSpam(metaclass=YAMLGetter):
rules: Dict[str, Dict[str, int]]
-class AntiMalware(metaclass=YAMLGetter):
- section = "anti_malware"
-
- whitelist: list
-
-
class BigBrother(metaclass=YAMLGetter):
section = 'big_brother'
diff --git a/bot/converters.py b/bot/converters.py
index 4a0633951..c9f525dd1 100644
--- a/bot/converters.py
+++ b/bot/converters.py
@@ -9,8 +9,11 @@ import dateutil.tz
import discord
from aiohttp import ClientConnectorError
from dateutil.relativedelta import relativedelta
-from discord.ext.commands import BadArgument, Context, Converter, UserConverter
+from discord.ext.commands import BadArgument, Bot, Context, Converter, IDConverter, UserConverter
+from bot.api import ResponseCodeError
+from bot.constants import URLs
+from bot.utils.regex import INVITE_RE
log = logging.getLogger(__name__)
@@ -34,6 +37,78 @@ def allowed_strings(*values, preserve_case: bool = False) -> t.Callable[[str], s
return converter
+class ValidDiscordServerInvite(Converter):
+ """
+ A converter that validates whether a given string is a valid Discord server invite.
+
+ Raises 'BadArgument' if:
+ - The string is not a valid Discord server invite.
+ - The string is valid, but is an invite for a group DM.
+ - The string is valid, but is expired.
+
+ Returns a (partial) guild object if:
+ - The string is a valid vanity
+ - The string is a full invite URI
+ - The string contains the invite code (the stuff after discord.gg/)
+
+ See the Discord API docs for documentation on the guild object:
+ https://discord.com/developers/docs/resources/guild#guild-object
+ """
+
+ async def convert(self, ctx: Context, server_invite: str) -> dict:
+ """Check whether the string is a valid Discord server invite."""
+ invite_code = INVITE_RE.search(server_invite)
+ if invite_code:
+ response = await ctx.bot.http_session.get(
+ f"{URLs.discord_invite_api}/{invite_code[1]}"
+ )
+ if response.status != 404:
+ invite_data = await response.json()
+ return invite_data.get("guild")
+
+ id_converter = IDConverter()
+ if id_converter._get_id_match(server_invite):
+ raise BadArgument("Guild IDs are not supported, only invites.")
+
+ raise BadArgument("This does not appear to be a valid Discord server invite.")
+
+
+class ValidFilterListType(Converter):
+ """
+ A converter that checks whether the given string is a valid FilterList type.
+
+ Raises `BadArgument` if the argument is not a valid FilterList type, and simply
+ passes through the given argument otherwise.
+ """
+
+ @staticmethod
+ async def get_valid_types(bot: Bot) -> list:
+ """
+ Try to get a list of valid filter list types.
+
+ Raise a BadArgument if the API can't respond.
+ """
+ try:
+ valid_types = await bot.api_client.get('bot/filter-lists/get-types')
+ except ResponseCodeError:
+ raise BadArgument("Cannot validate list_type: Unable to fetch valid types from API.")
+
+ return [enum for enum, classname in valid_types]
+
+ async def convert(self, ctx: Context, list_type: str) -> str:
+ """Checks whether the given string is a valid FilterList type."""
+ valid_types = await self.get_valid_types(ctx.bot)
+ list_type = list_type.upper()
+
+ if list_type not in valid_types:
+ valid_types_list = '\n'.join([f"• {type_.lower()}" for type_ in valid_types])
+ raise BadArgument(
+ f"You have provided an invalid list type!\n\n"
+ f"Please provide one of the following: \n{valid_types_list}"
+ )
+ return list_type
+
+
class ValidPythonIdentifier(Converter):
"""
A converter that checks whether the given string is a valid Python identifier.
@@ -237,6 +312,32 @@ class Duration(DurationDelta):
raise BadArgument(f"`{duration}` results in a datetime outside the supported range.")
+class OffTopicName(Converter):
+ """A converter that ensures an added off-topic name is valid."""
+
+ async def convert(self, ctx: Context, argument: str) -> str:
+ """Attempt to replace any invalid characters with their approximate Unicode equivalent."""
+ allowed_characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ!?'`-"
+
+ # Chain multiple words to a single one
+ argument = "-".join(argument.split())
+
+ if not (2 <= len(argument) <= 96):
+ raise BadArgument("Channel name must be between 2 and 96 chars long")
+
+ elif not all(c.isalnum() or c in allowed_characters for c in argument):
+ raise BadArgument(
+ "Channel name must only consist of "
+ "alphanumeric characters, minus signs or apostrophes."
+ )
+
+ # Replace invalid characters with unicode alternatives.
+ table = str.maketrans(
+ allowed_characters, '𝖠𝖡𝖢𝖣𝖤𝖥𝖦𝖧𝖨𝖩𝖪𝖫𝖬𝖭𝖮𝖯𝖰𝖱𝖲𝖳𝖴𝖵𝖶𝖷𝖸𝖹ǃ？’’-'
+ )
+ return argument.translate(table)
+
+
class ISODateTime(Converter):
"""Converts an ISO-8601 datetime string into a datetime.datetime."""
diff --git a/bot/utils/regex.py b/bot/utils/regex.py
new file mode 100644
index 000000000..d194f93cb
--- /dev/null
+++ b/bot/utils/regex.py
@@ -0,0 +1,12 @@
+import re
+
+INVITE_RE = re.compile(
+ r"(?:discord(?:[\.,]|dot)gg|" # Could be discord.gg/
+ r"discord(?:[\.,]|dot)com(?:\/|slash)invite|" # or discord.com/invite/
+ r"discordapp(?:[\.,]|dot)com(?:\/|slash)invite|" # or discordapp.com/invite/
+ r"discord(?:[\.,]|dot)me|" # or discord.me
+ r"discord(?:[\.,]|dot)io" # or discord.io.
+ r")(?:[\/]|slash)" # / or 'slash'
+ r"([a-zA-Z0-9]+)", # the invite code itself
+ flags=re.IGNORECASE
+)
diff --git a/config-default.yml b/config-default.yml
index fc093cc32..14a073611 100644
--- a/config-default.yml
+++ b/config-default.yml
@@ -278,107 +278,6 @@ filter:
ping_everyone: true
offensive_msg_delete_days: 7 # How many days before deleting an offensive message?
- guild_invite_whitelist:
- - 280033776820813825 # Functional Programming
- - 267624335836053506 # Python Discord
- - 440186186024222721 # Python Discord: Emojis 1
- - 578587418123304970 # Python Discord: Emojis 2
- - 273944235143593984 # STEM
- - 348658686962696195 # RLBot
- - 531221516914917387 # Pallets
- - 249111029668249601 # Gentoo
- - 327254708534116352 # Adafruit
- - 544525886180032552 # kennethreitz.org
- - 590806733924859943 # Discord Hack Week
- - 423249981340778496 # Kivy
- - 197038439483310086 # Discord Testers
- - 286633898581164032 # Ren'Py
- - 349505959032389632 # PyGame
- - 438622377094414346 # Pyglet
- - 524691714909274162 # Panda3D
- - 336642139381301249 # discord.py
- - 405403391410438165 # Sentdex
- - 172018499005317120 # The Coding Den
- - 666560367173828639 # PyWeek
- - 702724176489873509 # Microsoft Python
- - 150662382874525696 # Microsoft Community
- - 81384788765712384 # Discord API
- - 613425648685547541 # Discord Developers
- - 185590609631903755 # Blender Hub
- - 420324994703163402 # /r/FlutterDev
- - 488751051629920277 # Python Atlanta
- - 143867839282020352 # C#
- - 159039020565790721 # Django
- - 238666723824238602 # Programming Discussions
- - 433980600391696384 # JetBrains Community
- - 204621105720328193 # Raspberry Pi
- - 244230771232079873 # Programmers Hangout
- - 239433591950540801 # SpeakJS
- - 174075418410876928 # DevCord
- - 489222168727519232 # Unity
- - 494558898880118785 # Programmer Humor
-
- domain_blacklist:
- - pornhub.com
- - liveleak.com
- - grabify.link
- - bmwforum.co
- - leancoding.co
- - spottyfly.com
- - stopify.co
- - yoütu.be
- - discörd.com
- - minecräft.com
- - freegiftcards.co
- - disçordapp.com
- - fortnight.space
- - fortnitechat.site
- - joinmy.site
- - curiouscat.club
- - catsnthings.fun
- - yourtube.site
- - youtubeshort.watch
- - catsnthing.com
- - youtubeshort.pro
- - canadianlumberjacks.online
- - poweredbydialup.club
- - poweredbydialup.online
- - poweredbysecurity.org
- - poweredbysecurity.online
- - ssteam.site
- - steamwalletgift.com
- - discord.gift
- - lmgtfy.com
-
- word_watchlist:
- - goo+ks*
- - ky+s+
- - ki+ke+s*
- - beaner+s?
- - coo+ns*
- - nig+lets*
- - slant-eyes*
- - towe?l-?head+s*
- - chi*n+k+s*
- - spick*s*
- - kill* +(?:yo)?urself+
- - jew+s*
- - suicide
- - rape
- - (re+)tar+(d+|t+)(ed)?
- - ta+r+d+
- - cunts*
- - trann*y
- - shemale
-
- token_watchlist:
- - fa+g+s*
- - 卐
- - 卍
- - cuck(?!oo+)
- - nigg+(?:e*r+|a+h*?|u+h+)s?
- - fag+o+t+s*
-
# Censor doesn't apply to these
channel_whitelist:
- *ADMINS
@@ -489,35 +388,6 @@ anti_spam:
max: 3
-anti_malware:
- whitelist:
- - '.3gp'
- - '.3g2'
- - '.avi'
- - '.bmp'
- - '.gif'
- - '.h264'
- - '.jpg'
- - '.jpeg'
- - '.m4v'
- - '.mkv'
- - '.mov'
- - '.mp4'
- - '.mpeg'
- - '.mpg'
- - '.png'
- - '.tiff'
- - '.wmv'
- - '.svg'
- - '.psd' # Photoshop
- - '.ai' # Illustrator
- - '.aep' # After Effects
- - '.xcf' # GIMP
- - '.mp3'
- - '.wav'
- - '.ogg'
-
-
reddit:
subreddits:
- 'r/Python'
diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py
index f219fc1ba..ecb7abf00 100644
--- a/tests/bot/cogs/test_antimalware.py
+++ b/tests/bot/cogs/test_antimalware.py
@@ -1,28 +1,33 @@
import unittest
-from unittest.mock import AsyncMock, Mock, patch
+from unittest.mock import AsyncMock, Mock
from discord import NotFound
from bot.cogs import antimalware
-from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES
+from bot.constants import Channels, STAFF_ROLES
from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole
-MODULE = "bot.cogs.antimalware"
-
-@patch(f"{MODULE}.AntiMalwareConfig.whitelist", new=[".first", ".second", ".third"])
class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase):
"""Test the AntiMalware cog."""
def setUp(self):
"""Sets up fresh objects for each test."""
self.bot = MockBot()
+ self.bot.filter_list_cache = {
+ "FILE_FORMAT.True": {
+ ".first": {},
+ ".second": {},
+ ".third": {},
+ }
+ }
self.cog = antimalware.AntiMalware(self.bot)
self.message = MockMessage()
+ self.whitelist = [".first", ".second", ".third"]
async def test_message_with_allowed_attachment(self):
"""Messages with allowed extensions should not be deleted"""
- attachment = MockAttachment(filename=f"python{AntiMalwareConfig.whitelist[0]}")
+ attachment = MockAttachment(filename="python.first")
self.message.attachments = [attachment]
await self.cog.on_message(self.message)
@@ -93,7 +98,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(embed.description, antimalware.TXT_EMBED_DESCRIPTION.format.return_value)
antimalware.TXT_EMBED_DESCRIPTION.format.assert_called_with(cmd_channel_mention=cmd_channel.mention)
- async def test_other_disallowed_extention_embed_description(self):
+ async def test_other_disallowed_extension_embed_description(self):
"""Test the description for a non .py/.txt disallowed extension."""
attachment = MockAttachment(filename="python.disallowed")
self.message.attachments = [attachment]
@@ -109,6 +114,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(embed.description, antimalware.DISALLOWED_EMBED_DESCRIPTION.format.return_value)
antimalware.DISALLOWED_EMBED_DESCRIPTION.format.assert_called_with(
+ joined_whitelist=", ".join(self.whitelist),
blocked_extensions_str=".disallowed",
meta_channel_mention=meta_channel.mention
)
@@ -135,7 +141,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase):
"""The return value should include all non-whitelisted extensions."""
test_values = (
([], []),
- (AntiMalwareConfig.whitelist, []),
+ (self.whitelist, []),
([".first"], []),
([".first", ".disallowed"], [".disallowed"]),
([".disallowed"], [".disallowed"]),
@@ -145,7 +151,7 @@ class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase):
for extensions, expected_disallowed_extensions in test_values:
with self.subTest(extensions=extensions, expected_disallowed_extensions=expected_disallowed_extensions):
self.message.attachments = [MockAttachment(filename=f"filename{extension}") for extension in extensions]
- disallowed_extensions = self.cog.get_disallowed_extensions(self.message)
+ disallowed_extensions = self.cog._get_disallowed_extensions(self.message)
self.assertCountEqual(disallowed_extensions, expected_disallowed_extensions)
generated by cgit v1.2.3 (git 2.39.1) at 2025-10-24 13:24:15 +0000