diff options
65 files changed, 1298 insertions, 577 deletions
@@ -19,6 +19,7 @@ deepdiff = "~=4.0" requests = "~=2.22" more_itertools = "~=7.2" urllib3 = ">=1.24.2,<1.25" +prometheus-async = {extras = ["aiohttp"],version = "~=19.2"} [dev-packages] coverage = "~=4.5" diff --git a/Pipfile.lock b/Pipfile.lock index 69caf4646..ab5dfb538 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c27d699b4aeeed204dee41f924f682ae2a670add8549a8826e58776594370582" + "sha256": "d9349e8c704b2b2403004039856d8d75aaebc76e4aa93390c4d177f583e73b71" }, "pipfile-spec": 6, "requires": { @@ -18,11 +18,11 @@ "default": { "aio-pika": { "hashes": [ - "sha256:1da038b3d2c1b49e0e816d87424e702912bb77f9b5197f2bf279217915b4f7ed", - "sha256:29fe851374b86c997a22174c04352b5941bc1c2e36bbf542918ac18a76cfc9d3" + "sha256:a5837277e53755078db3a9e8c45bbca605c8ba9ecba7a02d74a7a1779f444723", + "sha256:fa32e33b4b7d0804dcf439ae6ff24d2f0a83d1ba280ee9f555e647d71d394ff5" ], "index": "pypi", - "version": "==6.3.0" + "version": "==6.4.1" }, "aiodns": { "hashes": [ @@ -62,10 +62,10 @@ }, "aiormq": { "hashes": [ - "sha256:afc0d46837b121585e4faec0a7646706429b4e2f5110ae8d0b5cdc3708b4b0e5", - "sha256:dc0fbbc7f8ad5af6a2cc18e00ccc5f925984cde3db6e8fe952c07b7ef157b5f2" + "sha256:8c215a970133ab5ee7c478decac55b209af7731050f52d11439fe910fa0f9e9d", + "sha256:9210f3389200aee7d8067f6435f4a9eff2d3a30b88beb5eaae406ccc11c0fc01" ], - "version": "==2.9.1" + "version": "==3.2.0" }, "alabaster": { "hashes": [ @@ -90,25 +90,25 @@ }, "babel": { "hashes": [ - "sha256:af92e6106cb7c55286b25b38ad7695f8b4efb36a90ba483d7f7a6628c46158ab", - "sha256:e86135ae101e31e2c8ec20a4e0c5220f4eed12487d5cf3f78be7e98d3a57fc28" + "sha256:1aac2ae2d0d8ea368fa90906567f5c08463d98ade155c0c4bfedd6a0f7160e38", + "sha256:d670ea0b10f8b723672d3a6abeb87b565b244da220d76b4dba1b66269ec152d4" ], - "version": "==2.7.0" + "version": "==2.8.0" }, "beautifulsoup4": { "hashes": [ - "sha256:5279c36b4b2ec2cb4298d723791467e3000e5384a43ea0cdf5d45207c7e97169", - "sha256:6135db2ba678168c07950f9a16c4031822c6f4aec75a65e0a97bc5ca09789931", - "sha256:dcdef580e18a76d54002088602eba453eec38ebbcafafeaabd8cab12b6155d57" + "sha256:05fd825eb01c290877657a56df4c6e4c311b3965bda790c613a3d6fb01a5462a", + "sha256:9fbb4d6e48ecd30bcacc5b63b94088192dcda178513b2ae3c394229f8911b887", + "sha256:e1505eeed31b0f4ce2dbb3bc8eb256c04cc2b3b72af7d551a4ab6efd5cbe5dae" ], - "version": "==4.8.1" + "version": "==4.8.2" }, "certifi": { "hashes": [ - "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", - "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" ], - "version": "==2019.9.11" + "version": "==2019.11.28" }, "cffi": { "hashes": [ @@ -195,10 +195,10 @@ }, "imagesize": { "hashes": [ - "sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8", - "sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5" + "sha256:6965f19a6a2039c7d48bca7dba2473069ff854c36ae6f19d2cde309d998228a1", + "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1" ], - "version": "==1.1.0" + "version": "==1.2.0" }, "jinja2": { "hashes": [ @@ -223,35 +223,35 @@ }, "lxml": { "hashes": [ - "sha256:02ca7bf899da57084041bb0f6095333e4d239948ad3169443f454add9f4e9cb4", - "sha256:096b82c5e0ea27ce9138bcbb205313343ee66a6e132f25c5ed67e2c8d960a1bc", - "sha256:0a920ff98cf1aac310470c644bc23b326402d3ef667ddafecb024e1713d485f1", - "sha256:1409b14bf83a7d729f92e2a7fbfe7ec929d4883ca071b06e95c539ceedb6497c", - "sha256:17cae1730a782858a6e2758fd20dd0ef7567916c47757b694a06ffafdec20046", - "sha256:17e3950add54c882e032527795c625929613adbd2ce5162b94667334458b5a36", - "sha256:1f4f214337f6ee5825bf90a65d04d70aab05526c08191ab888cb5149501923c5", - "sha256:2e8f77db25b0a96af679e64ff9bf9dddb27d379c9900c3272f3041c4d1327c9d", - "sha256:4dffd405390a45ecb95ab5ab1c1b847553c18b0ef8ed01e10c1c8b1a76452916", - "sha256:6b899931a5648862c7b88c795eddff7588fb585e81cecce20f8d9da16eff96e0", - "sha256:726c17f3e0d7a7200718c9a890ccfeab391c9133e363a577a44717c85c71db27", - "sha256:760c12276fee05c36f95f8040180abc7fbebb9e5011447a97cdc289b5d6ab6fc", - "sha256:796685d3969815a633827c818863ee199440696b0961e200b011d79b9394bbe7", - "sha256:891fe897b49abb7db470c55664b198b1095e4943b9f82b7dcab317a19116cd38", - "sha256:9277562f175d2334744ad297568677056861070399cec56ff06abbe2564d1232", - "sha256:a471628e20f03dcdfde00770eeaf9c77811f0c331c8805219ca7b87ac17576c5", - "sha256:a63b4fd3e2cabdcc9d918ed280bdde3e8e9641e04f3c59a2a3109644a07b9832", - "sha256:ae88588d687bd476be588010cbbe551e9c2872b816f2da8f01f6f1fda74e1ef0", - "sha256:b0b84408d4eabc6de9dd1e1e0bc63e7731e890c0b378a62443e5741cfd0ae90a", - "sha256:be78485e5d5f3684e875dab60f40cddace2f5b2a8f7fede412358ab3214c3a6f", - "sha256:c27eaed872185f047bb7f7da2d21a7d8913457678c9a100a50db6da890bc28b9", - "sha256:c7fccd08b14aa437fe096c71c645c0f9be0655a9b1a4b7cffc77bcb23b3d61d2", - "sha256:c81cb40bff373ab7a7446d6bbca0190bccc5be3448b47b51d729e37799bb5692", - "sha256:d11874b3c33ee441059464711cd365b89fa1a9cf19ae75b0c189b01fbf735b84", - "sha256:e9c028b5897901361d81a4718d1db217b716424a0283afe9d6735fe0caf70f79", - "sha256:fe489d486cd00b739be826e8c1be188ddb74c7a1ca784d93d06fda882a6a1681" - ], - "index": "pypi", - "version": "==4.4.1" + "sha256:00ac0d64949fef6b3693813fe636a2d56d97a5a49b5bbb86e4cc4cc50ebc9ea2", + "sha256:0571e607558665ed42e450d7bf0e2941d542c18e117b1ebbf0ba72f287ad841c", + "sha256:0e3f04a7615fdac0be5e18b2406529521d6dbdb0167d2a690ee328bef7807487", + "sha256:13cf89be53348d1c17b453867da68704802966c433b2bb4fa1f970daadd2ef70", + "sha256:217262fcf6a4c2e1c7cb1efa08bd9ebc432502abc6c255c4abab611e8be0d14d", + "sha256:223e544828f1955daaf4cefbb4853bc416b2ec3fd56d4f4204a8b17007c21250", + "sha256:277cb61fede2f95b9c61912fefb3d43fbd5f18bf18a14fae4911b67984486f5d", + "sha256:3213f753e8ae86c396e0e066866e64c6b04618e85c723b32ecb0909885211f74", + "sha256:4690984a4dee1033da0af6df0b7a6bde83f74e1c0c870623797cec77964de34d", + "sha256:4fcc472ef87f45c429d3b923b925704aa581f875d65bac80f8ab0c3296a63f78", + "sha256:61409bd745a265a742f2693e4600e4dbd45cc1daebe1d5fad6fcb22912d44145", + "sha256:678f1963f755c5d9f5f6968dded7b245dd1ece8cf53c1aa9d80e6734a8c7f41d", + "sha256:6c6d03549d4e2734133badb9ab1c05d9f0ef4bcd31d83e5d2b4747c85cfa21da", + "sha256:6e74d5f4d6ecd6942375c52ffcd35f4318a61a02328f6f1bd79fcb4ffedf969e", + "sha256:7b4fc7b1ecc987ca7aaf3f4f0e71bbfbd81aaabf87002558f5bc95da3a865bcd", + "sha256:7ed386a40e172ddf44c061ad74881d8622f791d9af0b6f5be20023029129bc85", + "sha256:8f54f0924d12c47a382c600c880770b5ebfc96c9fd94cf6f6bdc21caf6163ea7", + "sha256:ad9b81351fdc236bda538efa6879315448411a81186c836d4b80d6ca8217cdb9", + "sha256:bbd00e21ea17f7bcc58dccd13869d68441b32899e89cf6cfa90d624a9198ce85", + "sha256:c3c289762cc09735e2a8f8a49571d0e8b4f57ea831ea11558247b5bdea0ac4db", + "sha256:cf4650942de5e5685ad308e22bcafbccfe37c54aa7c0e30cd620c2ee5c93d336", + "sha256:cfcbc33c9c59c93776aa41ab02e55c288a042211708b72fdb518221cc803abc8", + "sha256:e301055deadfedbd80cf94f2f65ff23126b232b0d1fea28f332ce58137bcdb18", + "sha256:ebbfe24df7f7b5c6c7620702496b6419f6a9aa2fd7f005eb731cc80d7b4692b9", + "sha256:eff69ddbf3ad86375c344339371168640951c302450c5d3e9936e98d6459db06", + "sha256:f6ed60a62c5f1c44e789d2cf14009423cb1646b44a43e40a9cf6a21f077678a1" + ], + "index": "pypi", + "version": "==4.4.2" }, "markdownify": { "hashes": [ @@ -303,37 +303,25 @@ }, "multidict": { "hashes": [ - "sha256:024b8129695a952ebd93373e45b5d341dbb87c17ce49637b34000093f243dd4f", - "sha256:041e9442b11409be5e4fc8b6a97e4bcead758ab1e11768d1e69160bdde18acc3", - "sha256:045b4dd0e5f6121e6f314d81759abd2c257db4634260abcfe0d3f7083c4908ef", - "sha256:047c0a04e382ef8bd74b0de01407e8d8632d7d1b4db6f2561106af812a68741b", - "sha256:068167c2d7bbeebd359665ac4fff756be5ffac9cda02375b5c5a7c4777038e73", - "sha256:148ff60e0fffa2f5fad2eb25aae7bef23d8f3b8bdaf947a65cdbe84a978092bc", - "sha256:1d1c77013a259971a72ddaa83b9f42c80a93ff12df6a4723be99d858fa30bee3", - "sha256:1d48bc124a6b7a55006d97917f695effa9725d05abe8ee78fd60d6588b8344cd", - "sha256:31dfa2fc323097f8ad7acd41aa38d7c614dd1960ac6681745b6da124093dc351", - "sha256:34f82db7f80c49f38b032c5abb605c458bac997a6c3142e0d6c130be6fb2b941", - "sha256:3d5dd8e5998fb4ace04789d1d008e2bb532de501218519d70bb672c4c5a2fc5d", - "sha256:4a6ae52bd3ee41ee0f3acf4c60ceb3f44e0e3bc52ab7da1c2b2aa6703363a3d1", - "sha256:4b02a3b2a2f01d0490dd39321c74273fed0568568ea0e7ea23e02bd1fb10a10b", - "sha256:4b843f8e1dd6a3195679d9838eb4670222e8b8d01bc36c9894d6c3538316fa0a", - "sha256:5de53a28f40ef3c4fd57aeab6b590c2c663de87a5af76136ced519923d3efbb3", - "sha256:61b2b33ede821b94fa99ce0b09c9ece049c7067a33b279f343adfe35108a4ea7", - "sha256:6a3a9b0f45fd75dc05d8e93dc21b18fc1670135ec9544d1ad4acbcf6b86781d0", - "sha256:76ad8e4c69dadbb31bad17c16baee61c0d1a4a73bed2590b741b2e1a46d3edd0", - "sha256:7ba19b777dc00194d1b473180d4ca89a054dd18de27d0ee2e42a103ec9b7d014", - "sha256:7c1b7eab7a49aa96f3db1f716f0113a8a2e93c7375dd3d5d21c4941f1405c9c5", - "sha256:7fc0eee3046041387cbace9314926aa48b681202f8897f8bff3809967a049036", - "sha256:8ccd1c5fff1aa1427100ce188557fc31f1e0a383ad8ec42c559aabd4ff08802d", - "sha256:8e08dd76de80539d613654915a2f5196dbccc67448df291e69a88712ea21e24a", - "sha256:c18498c50c59263841862ea0501da9f2b3659c00db54abfbf823a80787fde8ce", - "sha256:c49db89d602c24928e68c0d510f4fcf8989d77defd01c973d6cbe27e684833b1", - "sha256:ce20044d0317649ddbb4e54dab3c1bcc7483c78c27d3f58ab3d0c7e6bc60d26a", - "sha256:d1071414dd06ca2eafa90c85a079169bfeb0e5f57fd0b45d44c092546fcd6fd9", - "sha256:d3be11ac43ab1a3e979dac80843b42226d5d3cccd3986f2e03152720a4297cd7", - "sha256:db603a1c235d110c860d5f39988ebc8218ee028f07a7cbc056ba6424372ca31b" - ], - "version": "==4.5.2" + "sha256:13f3ebdb5693944f52faa7b2065b751cb7e578b8dd0a5bb8e4ab05ad0188b85e", + "sha256:26502cefa86d79b86752e96639352c7247846515c864d7c2eb85d036752b643c", + "sha256:4fba5204d32d5c52439f88437d33ad14b5f228e25072a192453f658bddfe45a7", + "sha256:527124ef435f39a37b279653ad0238ff606b58328ca7989a6df372fd75d7fe26", + "sha256:5414f388ffd78c57e77bd253cf829373721f450613de53dc85a08e34d806e8eb", + "sha256:5eee66f882ab35674944dfa0d28b57fa51e160b4dce0ce19e47f495fdae70703", + "sha256:63810343ea07f5cd86ba66ab66706243a6f5af075eea50c01e39b4ad6bc3c57a", + "sha256:6bd10adf9f0d6a98ccc792ab6f83d18674775986ba9bacd376b643fe35633357", + "sha256:83c6ddf0add57c6b8a7de0bc7e2d656be3eefeff7c922af9a9aae7e49f225625", + "sha256:93166e0f5379cf6cd29746989f8a594fa7204dcae2e9335ddba39c870a287e1c", + "sha256:9a7b115ee0b9b92d10ebc246811d8f55d0c57e82dbb6a26b23c9a9a6ad40ce0c", + "sha256:a38baa3046cce174a07a59952c9f876ae8875ef3559709639c17fdf21f7b30dd", + "sha256:a6d219f49821f4b2c85c6d426346a5d84dab6daa6f85ca3da6c00ed05b54022d", + "sha256:a8ed33e8f9b67e3b592c56567135bb42e7e0e97417a4b6a771e60898dfd5182b", + "sha256:d7d428488c67b09b26928950a395e41cc72bb9c3d5abfe9f0521940ee4f796d4", + "sha256:dcfed56aa085b89d644af17442cdc2debaa73388feba4b8026446d168ca8dad7", + "sha256:f29b885e4903bd57a7789f09fe9d60b6475a6c1a4c0eca874d8558f00f9d4b51" + ], + "version": "==4.7.4" }, "ordered-set": { "hashes": [ @@ -343,10 +331,10 @@ }, "packaging": { "hashes": [ - "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", - "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" + "sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb", + "sha256:fe1d8331dfa7cc0a883b49d75fc76380b2ab2734b220fbb87d774e4fd4b851f8" ], - "version": "==19.2" + "version": "==20.0" }, "pamqp": { "hashes": [ @@ -355,23 +343,56 @@ ], "version": "==2.3.0" }, + "prometheus-async": { + "extras": [ + "aiohttp" + ], + "hashes": [ + "sha256:227f516e5bf98a0dc602348381e182358f8b2ed24a8db05e8e34d9cf027bab83", + "sha256:3cc68d1f39e9bbf16dbd0b51103d87671b3cbd1d75a72cda472cd9a35cc9d0d2" + ], + "index": "pypi", + "version": "==19.2.0" + }, + "prometheus-client": { + "hashes": [ + "sha256:71cd24a2b3eb335cb800c7159f423df1bd4dcd5171b234be15e3f31ec9f622da" + ], + "version": "==0.7.1" + }, "pycares": { "hashes": [ - "sha256:2ca080db265ea238dc45f997f94effb62b979a617569889e265c26a839ed6305", - "sha256:6f79c6afb6ce603009db2042fddc2e348ad093ece9784cbe2daa809499871a23", - "sha256:70918d06eb0603016d37092a5f2c0228509eb4e6c5a3faacb4184f6ab7be7650", - "sha256:755187d28d24a9ea63aa2b4c0638be31d65fbf7f0ce16d41261b9f8cb55a1b99", - "sha256:7baa4b1f2146eb8423ff8303ebde3a20fb444a60db761fba0430d104fe35ddbf", - "sha256:90b27d4df86395f465a171386bc341098d6d47b65944df46518814ae298f6cc6", - "sha256:9e090dd6b2afa65cb51c133883b2bf2240fd0f717b130b0048714b33fb0f47ce", - "sha256:a11b7d63c3718775f6e805d6464cb10943780395ab042c7e5a0a7a9f612735dd", - "sha256:b253f5dcaa0ac7076b79388a3ac80dd8f3bd979108f813baade40d3a9b8bf0bd", - "sha256:c7f4f65e44ba35e35ad3febc844270665bba21cfb0fb7d749434e705b556e087", - "sha256:cdb342e6a254f035bd976d95807a2184038fc088d957a5104dcaab8be602c093", - "sha256:cf08e164f8bfb83b9fe633feb56f2754fae6baefcea663593794fa0518f8f98c", - "sha256:df9bc694cf03673878ea8ce674082c5acd134991d64d6c306d4bd61c0c1df98f" - ], - "version": "==3.0.0" + "sha256:050f00b39ed77ea8a4e555f09417d4b1a6b5baa24bb9531a3e15d003d2319b3f", + "sha256:0a24d2e580a8eb567140d7b69f12cb7de90c836bd7b6488ec69394d308605ac3", + "sha256:0c5bd1f6f885a219d5e972788d6eef7b8043b55c3375a845e5399638436e0bba", + "sha256:11c628402cc8fc8ef461076d4e47f88afc1f8609989ebbff0dbffcd54c97239f", + "sha256:18dfd4fd300f570d6c4536c1d987b7b7673b2a9d14346592c5d6ed716df0d104", + "sha256:1917b82494907a4a342db420bc4dd5bac355a5fa3984c35ba9bf51422b020b48", + "sha256:1b90fa00a89564df059fb18e796458864cc4e00cb55e364dbf921997266b7c55", + "sha256:1d8d177c40567de78108a7835170f570ab04f09084bfd32df9919c0eaec47aa1", + "sha256:236286f81664658b32c141c8e79d20afc3d54f6e2e49dfc8b702026be7265855", + "sha256:2e4f74677542737fb5af4ea9a2e415ec5ab31aa67e7b8c3c969fdb15c069f679", + "sha256:48a7750f04e69e1f304f4332b755728067e7c4b1abe2760bba1cacd9ff7a847a", + "sha256:7d86e62b700b21401ffe7fd1bbfe91e08489416fecae99c6570ab023c6896022", + "sha256:7e2d7effd08d2e5a3cb95d98a7286ebab71ab2fbce84fa93cc2dd56caf7240dd", + "sha256:81edb016d9e43dde7473bc3999c29cdfee3a6b67308fed1ea21049f458e83ae0", + "sha256:96c90e11b4a4c7c0b8ff5aaaae969c5035493136586043ff301979aae0623941", + "sha256:9a0a1845f8cb2e62332bca0aaa9ad5494603ac43fb60d510a61d5b5b170d7216", + "sha256:a05bbfdfd41f8410a905a818f329afe7510cbd9ee65c60f8860a72b6c64ce5dc", + "sha256:a5089fd660f0b0d228b14cdaa110d0d311edfa5a63f800618dbf1321dcaef66b", + "sha256:c457a709e6f2befea7e2996c991eda6d79705dd075f6521593ba6ebc1485b811", + "sha256:c5cb72644b04e5e5abfb1e10a0e7eb75da6684ea0e60871652f348e412cf3b11", + "sha256:cce46dd4717debfd2aab79d6d7f0cbdf6b1e982dc4d9bebad81658d59ede07c2", + "sha256:cfdd1f90bcf373b00f4b2c55ea47868616fe2f779f792fc913fa82a3d64ffe43", + "sha256:d88a279cbc5af613f73e86e19b3f63850f7a2e2736e249c51995dedcc830b1bb", + "sha256:eba9a9227438da5e78fc8eee32f32eb35d9a50cf0a0bd937eb6275c7cc3015fe", + "sha256:eee7b6a5f5b5af050cb7d66ab28179287b416f06d15a8974ac831437fec51336", + "sha256:f41ac1c858687e53242828c9f59c2e7b0b95dbcd5bdd09c7e5d3c48b0f89a25a", + "sha256:f8deaefefc3a589058df1b177275f79233e8b0eeee6734cf4336d80164ecd022", + "sha256:fa78e919f3bd7d6d075db262aa41079b4c02da315c6043c6f43881e2ebcdd623", + "sha256:fadb97d2e02dabdc15a0091591a972a938850d79ddde23d385d813c1731983f0" + ], + "version": "==3.1.1" }, "pycparser": { "hashes": [ @@ -381,17 +402,17 @@ }, "pygments": { "hashes": [ - "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", - "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" + "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b", + "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe" ], - "version": "==2.4.2" + "version": "==2.5.2" }, "pyparsing": { "hashes": [ - "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", - "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" + "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", + "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" ], - "version": "==2.4.5" + "version": "==2.4.6" }, "python-dateutil": { "hashes": [ @@ -416,22 +437,20 @@ }, "pyyaml": { "hashes": [ - "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", - "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", - "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", - "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", - "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", - "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", - "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", - "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", - "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", - "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", - "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", - "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", - "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" + "sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6", + "sha256:1cf708e2ac57f3aabc87405f04b86354f66799c8e62c28c5fc5f88b5521b2dbf", + "sha256:24521fa2890642614558b492b473bee0ac1f8057a7263156b02e8b14c88ce6f5", + "sha256:4fee71aa5bc6ed9d5f116327c04273e25ae31a3020386916905767ec4fc5317e", + "sha256:70024e02197337533eef7b85b068212420f950319cc8c580261963aefc75f811", + "sha256:74782fbd4d4f87ff04159e986886931456a1894c61229be9eaf4de6f6e44b99e", + "sha256:940532b111b1952befd7db542c370887a8611660d2b9becff75d39355303d82d", + "sha256:cb1f2f5e426dc9f07a7681419fe39cee823bb74f723f36f70399123f439e9b20", + "sha256:dbbb2379c19ed6042e8f11f2a2c66d39cceb8aeace421bfc29d085d93eda3689", + "sha256:e3a057b7a64f1222b56e47bcff5e4b94c4f61faac04c7c4ecb1985e18caa3994", + "sha256:e9f45bd5b92c7974e59bcd2dcc8631a6b6cc380a904725fce7bc08872e691615" ], "index": "pypi", - "version": "==5.1.2" + "version": "==5.3" }, "requests": { "hashes": [ @@ -464,11 +483,11 @@ }, "sphinx": { "hashes": [ - "sha256:31088dfb95359384b1005619827eaee3056243798c62724fd3fa4b84ee4d71bd", - "sha256:52286a0b9d7caa31efee301ec4300dbdab23c3b05da1c9024b4e84896fb73d79" + "sha256:298537cb3234578b2d954ff18c5608468229e116a9757af3b831c2b2b4819159", + "sha256:e6e766b74f85f37a5f3e0773a1e1be8db3fcb799deb58ca6d18b70b0b44542a5" ], "index": "pypi", - "version": "==2.2.1" + "version": "==2.3.1" }, "sphinxcontrib-applehelp": { "hashes": [ @@ -546,21 +565,33 @@ ], "version": "==6.0" }, - "yarl": { + "wrapt": { "hashes": [ - "sha256:024ecdc12bc02b321bc66b41327f930d1c2c543fa9a561b39861da9388ba7aa9", - "sha256:2f3010703295fbe1aec51023740871e64bb9664c789cba5a6bdf404e93f7568f", - "sha256:3890ab952d508523ef4881457c4099056546593fa05e93da84c7250516e632eb", - "sha256:3e2724eb9af5dc41648e5bb304fcf4891adc33258c6e14e2a7414ea32541e320", - "sha256:5badb97dd0abf26623a9982cd448ff12cb39b8e4c94032ccdedf22ce01a64842", - "sha256:73f447d11b530d860ca1e6b582f947688286ad16ca42256413083d13f260b7a0", - "sha256:7ab825726f2940c16d92aaec7d204cfc34ac26c0040da727cf8ba87255a33829", - "sha256:b25de84a8c20540531526dfbb0e2d2b648c13fd5dd126728c496d7c3fea33310", - "sha256:c6e341f5a6562af74ba55205dbd56d248daf1b5748ec48a0200ba227bb9e33f4", - "sha256:c9bb7c249c4432cd47e75af3864bc02d26c9594f49c82e2a28624417f0ae63b8", - "sha256:e060906c0c585565c718d1c3841747b61c5439af2211e185f6739a9412dfbde1" + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" ], - "version": "==1.3.0" + "version": "==1.11.2" + }, + "yarl": { + "hashes": [ + "sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce", + "sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6", + "sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce", + "sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae", + "sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d", + "sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f", + "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b", + "sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b", + "sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb", + "sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462", + "sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea", + "sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70", + "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1", + "sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a", + "sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b", + "sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080", + "sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2" + ], + "version": "==1.4.2" } }, "develop": { @@ -580,10 +611,10 @@ }, "certifi": { "hashes": [ - "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", - "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" ], - "version": "==2019.9.11" + "version": "==2019.11.28" }, "cfgv": { "hashes": [ @@ -646,10 +677,11 @@ }, "dodgy": { "hashes": [ - "sha256:65e13cf878d7aff129f1461c13cb5fd1bb6dfe66bb5327e09379c3877763280c" + "sha256:28323cbfc9352139fdd3d316fa17f325cc0e9ac74438cbba51d70f9b48f86c3a", + "sha256:51f54c0fd886fa3854387f354b19f429d38c04f984f38bc572558b703c0542a6" ], "index": "pypi", - "version": "==0.1.9" + "version": "==0.2.1" }, "dparse": { "hashes": [ @@ -675,11 +707,11 @@ }, "flake8-annotations": { "hashes": [ - "sha256:6ac7ca1e706307686b60af8043ff1db31dc2cfc1233c8210d67a3d9b8f364736", - "sha256:b51131007000d67217608fa028a35ff80aa400b474e5972f1f99c2cf9d26bd2e" + "sha256:05b85538014c850a86dce7374bb6621c64481c24e35e8e90af1315f4d7a3dbaa", + "sha256:43e5233a76fda002b91a54a7cc4510f099c4bfd6279502ec70164016250eebd1" ], "index": "pypi", - "version": "==1.1.0" + "version": "==1.1.3" }, "flake8-bugbear": { "hashes": [ @@ -730,10 +762,10 @@ }, "identify": { "hashes": [ - "sha256:4f1fe9a59df4e80fcb0213086fcf502bc1765a01ea4fe8be48da3b65afd2a017", - "sha256:d8919589bd2a5f99c66302fec0ef9027b12ae150b0b0213999ad3f695fc7296e" + "sha256:6f44e637caa40d1b4cb37f6ed3b262ede74901d28b1cc5b1fc07360871edd65d", + "sha256:72e9c4ed3bc713c7045b762b0d2e2115c572b85abfc1f4604f5a4fd4c6642b71" ], - "version": "==1.4.7" + "version": "==1.4.9" }, "idna": { "hashes": [ @@ -744,11 +776,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", - "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af" + "sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359", + "sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8" ], "markers": "python_version < '3.8'", - "version": "==0.23" + "version": "==1.4.0" }, "mccabe": { "hashes": [ @@ -767,24 +799,24 @@ }, "nodeenv": { "hashes": [ - "sha256:ad8259494cf1c9034539f6cced78a1da4840a4b157e23640bc4a0c0546b0cb7a" + "sha256:561057acd4ae3809e665a9aaaf214afff110bbb6a6d5c8a96121aea6878408b3" ], - "version": "==1.3.3" + "version": "==1.3.4" }, "packaging": { "hashes": [ - "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", - "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" + "sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb", + "sha256:fe1d8331dfa7cc0a883b49d75fc76380b2ab2734b220fbb87d774e4fd4b851f8" ], - "version": "==19.2" + "version": "==20.0" }, "pre-commit": { "hashes": [ - "sha256:9f152687127ec90642a2cc3e4d9e1e6240c4eb153615cb02aa1ad41d331cbb6e", - "sha256:c2e4810d2d3102d354947907514a78c5d30424d299dc0fe48f5aa049826e9b50" + "sha256:8f48d8637bdae6fa70cc97db9c1dd5aa7c5c8bf71968932a380628c25978b850", + "sha256:f92a359477f3252452ae2e8d3029de77aec59415c16ae4189bcfba40b757e029" ], "index": "pypi", - "version": "==1.20.0" + "version": "==1.21.0" }, "pycodestyle": { "hashes": [ @@ -795,10 +827,10 @@ }, "pydocstyle": { "hashes": [ - "sha256:04c84e034ebb56eb6396c820442b8c4499ac5eb94a3bda88951ac3dc519b6058", - "sha256:66aff87ffe34b1e49bff2dd03a88ce6843be2f3346b0c9814410d34987fbab59" + "sha256:da7831660b7355307b32778c4a0dbfb137d89254ef31a2b2978f50fc0b4d7586", + "sha256:f4f5d210610c2d153fae39093d44224c17429e2ad7da12a8b419aba5c2f614b5" ], - "version": "==4.0.1" + "version": "==5.0.2" }, "pyflakes": { "hashes": [ @@ -809,29 +841,27 @@ }, "pyparsing": { "hashes": [ - "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", - "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" + "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", + "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" ], - "version": "==2.4.5" + "version": "==2.4.6" }, "pyyaml": { "hashes": [ - "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", - "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", - "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", - "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", - "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", - "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", - "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", - "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", - "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", - "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", - "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", - "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", - "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" + "sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6", + "sha256:1cf708e2ac57f3aabc87405f04b86354f66799c8e62c28c5fc5f88b5521b2dbf", + "sha256:24521fa2890642614558b492b473bee0ac1f8057a7263156b02e8b14c88ce6f5", + "sha256:4fee71aa5bc6ed9d5f116327c04273e25ae31a3020386916905767ec4fc5317e", + "sha256:70024e02197337533eef7b85b068212420f950319cc8c580261963aefc75f811", + "sha256:74782fbd4d4f87ff04159e986886931456a1894c61229be9eaf4de6f6e44b99e", + "sha256:940532b111b1952befd7db542c370887a8611660d2b9becff75d39355303d82d", + "sha256:cb1f2f5e426dc9f07a7681419fe39cee823bb74f723f36f70399123f439e9b20", + "sha256:dbbb2379c19ed6042e8f11f2a2c66d39cceb8aeace421bfc29d085d93eda3689", + "sha256:e3a057b7a64f1222b56e47bcff5e4b94c4f61faac04c7c4ecb1985e18caa3994", + "sha256:e9f45bd5b92c7974e59bcd2dcc8631a6b6cc380a904725fce7bc08872e691615" ], "index": "pypi", - "version": "==5.1.2" + "version": "==5.3" }, "requests": { "hashes": [ @@ -893,6 +923,7 @@ "sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66", "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" ], + "markers": "python_version < '3.8'", "version": "==1.4.0" }, "unittest-xml-reporting": { @@ -913,10 +944,10 @@ }, "virtualenv": { "hashes": [ - "sha256:11cb4608930d5fd3afb545ecf8db83fa50e1f96fc4fca80c94b07d2c83146589", - "sha256:d257bb3773e48cac60e475a19b608996c73f4d333b3ba2e4e57d5ac6134e0136" + "sha256:0d62c70883c0342d59c11d0ddac0d954d0431321a41ab20851facf2b222598f3", + "sha256:55059a7a676e4e19498f1aad09b8313a38fcc0cdbe4fdddc0e9b06946d21b4bb" ], - "version": "==16.7.7" + "version": "==16.7.9" }, "zipp": { "hashes": [ @@ -1,6 +1,6 @@ # Python Utility Bot -[](https://discord.gg/2B963hn) +[](https://discord.gg/2B963hn) [](https://dev.azure.com/python-discord/Python%20Discord/_build/latest?definitionId=1&branchName=master) [](https://dev.azure.com/python-discord/Python%20Discord/_apis/build/status/Bot?branchName=master) [](https://dev.azure.com/python-discord/Python%20Discord/_apis/build/status/Bot?branchName=master) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index da3b06201..0400ac4d2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -30,7 +30,7 @@ jobs: - script: python -m flake8 displayName: 'Run linter' - - script: BOT_API_KEY=foo BOT_TOKEN=bar WOLFRAM_API_KEY=baz coverage run -m xmlrunner + - script: BOT_API_KEY=foo BOT_TOKEN=bar WOLFRAM_API_KEY=baz REDDIT_CLIENT_ID=spam REDDIT_SECRET=ham coverage run -m xmlrunner displayName: Run tests - script: coverage report -m && coverage xml -o coverage.xml diff --git a/bot/__init__.py b/bot/__init__.py index 4a2df730d..789ace5c0 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -6,6 +6,7 @@ from pathlib import Path from logmatic import JsonFormatter + logging.TRACE = 5 logging.addLevelName(logging.TRACE, "TRACE") diff --git a/bot/__main__.py b/bot/__main__.py index ea7c43a12..61271a692 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -1,18 +1,11 @@ -import asyncio -import logging -import socket - import discord -from aiohttp import AsyncResolver, ClientSession, TCPConnector -from discord.ext.commands import Bot, when_mentioned_or +from discord.ext.commands import when_mentioned_or from bot import patches -from bot.api import APIClient, APILoggingHandler +from bot.bot import Bot from bot.constants import Bot as BotConfig, DEBUG_MODE -log = logging.getLogger('bot') - bot = Bot( command_prefix=when_mentioned_or(BotConfig.prefix), activity=discord.Game(name="Commands: !help"), @@ -20,18 +13,6 @@ bot = Bot( max_messages=10_000, ) -# Global aiohttp session for all cogs -# - Uses asyncio for DNS resolution instead of threads, so we don't spam threads -# - Uses AF_INET as its socket family to prevent https related problems both locally and in prod. -bot.http_session = ClientSession( - connector=TCPConnector( - resolver=AsyncResolver(), - family=socket.AF_INET, - ) -) -bot.api_client = APIClient(loop=asyncio.get_event_loop()) -log.addHandler(APILoggingHandler(bot.api_client)) - # Internal/debug bot.load_extension("bot.cogs.error_handler") bot.load_extension("bot.cogs.filtering") @@ -59,6 +40,7 @@ bot.load_extension("bot.cogs.duck_pond") bot.load_extension("bot.cogs.free") bot.load_extension("bot.cogs.information") bot.load_extension("bot.cogs.jams") +bot.load_extension("bot.cogs.metrics") bot.load_extension("bot.cogs.moderation") bot.load_extension("bot.cogs.off_topic_names") bot.load_extension("bot.cogs.reddit") @@ -77,6 +59,3 @@ if not hasattr(discord.message.Message, '_handle_edited_timestamp'): patches.message_edited_at.apply_patch() bot.run(BotConfig.token) - -# This calls a coroutine, so it doesn't do anything at the moment. -# bot.http_session.close() # Close the aiohttp session when the bot finishes running diff --git a/bot/api.py b/bot/api.py index 7f26e5305..56db99828 100644 --- a/bot/api.py +++ b/bot/api.py @@ -32,7 +32,7 @@ class ResponseCodeError(ValueError): class APIClient: """Django Site API wrapper.""" - def __init__(self, **kwargs): + def __init__(self, loop: asyncio.AbstractEventLoop, **kwargs): auth_headers = { 'Authorization': f"Token {Keys.site_api}" } @@ -42,12 +42,39 @@ class APIClient: else: kwargs['headers'] = auth_headers - self.session = aiohttp.ClientSession(**kwargs) + self.session: Optional[aiohttp.ClientSession] = None + self.loop = loop + + self._ready = asyncio.Event(loop=loop) + self._creation_task = None + self._session_args = kwargs + + self.recreate() @staticmethod def _url_for(endpoint: str) -> str: return f"{URLs.site_schema}{URLs.site_api}/{quote_url(endpoint)}" + async def _create_session(self) -> None: + """Create the aiohttp session and set the ready event.""" + self.session = aiohttp.ClientSession(**self._session_args) + self._ready.set() + + async def close(self) -> None: + """Close the aiohttp session and unset the ready event.""" + if not self._ready.is_set(): + return + + await self.session.close() + self._ready.clear() + + def recreate(self) -> None: + """Schedule the aiohttp session to be created if it's been closed.""" + if self.session is None or self.session.closed: + # Don't schedule a task if one is already in progress. + if self._creation_task is None or self._creation_task.done(): + self._creation_task = self.loop.create_task(self._create_session()) + async def maybe_raise_for_status(self, response: aiohttp.ClientResponse, should_raise: bool) -> None: """Raise ResponseCodeError for non-OK response if an exception should be raised.""" if should_raise and response.status >= 400: @@ -60,30 +87,40 @@ class APIClient: async def get(self, endpoint: str, *args, raise_for_status: bool = True, **kwargs) -> dict: """Site API GET.""" + await self._ready.wait() + async with self.session.get(self._url_for(endpoint), *args, **kwargs) as resp: await self.maybe_raise_for_status(resp, raise_for_status) return await resp.json() async def patch(self, endpoint: str, *args, raise_for_status: bool = True, **kwargs) -> dict: """Site API PATCH.""" + await self._ready.wait() + async with self.session.patch(self._url_for(endpoint), *args, **kwargs) as resp: await self.maybe_raise_for_status(resp, raise_for_status) return await resp.json() async def post(self, endpoint: str, *args, raise_for_status: bool = True, **kwargs) -> dict: """Site API POST.""" + await self._ready.wait() + async with self.session.post(self._url_for(endpoint), *args, **kwargs) as resp: await self.maybe_raise_for_status(resp, raise_for_status) return await resp.json() async def put(self, endpoint: str, *args, raise_for_status: bool = True, **kwargs) -> dict: """Site API PUT.""" + await self._ready.wait() + async with self.session.put(self._url_for(endpoint), *args, **kwargs) as resp: await self.maybe_raise_for_status(resp, raise_for_status) return await resp.json() async def delete(self, endpoint: str, *args, raise_for_status: bool = True, **kwargs) -> Optional[dict]: """Site API DELETE.""" + await self._ready.wait() + async with self.session.delete(self._url_for(endpoint), *args, **kwargs) as resp: if resp.status == 204: return None diff --git a/bot/bot.py b/bot/bot.py new file mode 100644 index 000000000..930aaf70e --- /dev/null +++ b/bot/bot.py @@ -0,0 +1,56 @@ +import logging +import socket +from typing import Optional + +import aiohttp +from discord.ext import commands +from prometheus_async.aio.web import start_http_server as start_prometheus_http_server + +from bot import api + +log = logging.getLogger('bot') + + +class Bot(commands.Bot): + """A subclass of `discord.ext.commands.Bot` with an aiohttp session and an API client.""" + + def __init__(self, *args, **kwargs): + # Use asyncio for DNS resolution instead of threads so threads aren't spammed. + # Use AF_INET as its socket family to prevent HTTPS related problems both locally + # and in production. + self.connector = aiohttp.TCPConnector( + resolver=aiohttp.AsyncResolver(), + family=socket.AF_INET, + ) + + super().__init__(*args, connector=self.connector, **kwargs) + + self.http_session: Optional[aiohttp.ClientSession] = None + self.api_client = api.APIClient(loop=self.loop, connector=self.connector) + + log.addHandler(api.APILoggingHandler(self.api_client)) + + def add_cog(self, cog: commands.Cog) -> None: + """Adds a "cog" to the bot and logs the operation.""" + super().add_cog(cog) + log.info(f"Cog loaded: {cog.qualified_name}") + + def clear(self) -> None: + """Clears the internal state of the bot and resets the API client.""" + super().clear() + self.api_client.recreate() + + async def close(self) -> None: + """Close the aiohttp session after closing the Discord connection.""" + await super().close() + + await self.http_session.close() + await self.api_client.close() + + async def start(self, *args, **kwargs) -> None: + """Open an aiohttp session before logging in and connecting to Discord.""" + self.http_session = aiohttp.ClientSession(connector=self.connector) + + await start_prometheus_http_server(addr="0.0.0.0", port=9330) + log.debug("Started Prometheus server on port 9330.") + await super().start(*args, **kwargs) diff --git a/bot/cogs/alias.py b/bot/cogs/alias.py index 5190c559b..d05a6a715 100644 --- a/bot/cogs/alias.py +++ b/bot/cogs/alias.py @@ -3,8 +3,12 @@ import logging from typing import Union from discord import Colour, Embed, Member, User -from discord.ext.commands import Bot, Cog, Command, Context, clean_content, command, group +from discord.ext.commands import ( + Cog, Command, Context, Greedy, + clean_content, command, group, +) +from bot.bot import Bot from bot.cogs.extensions import Extension from bot.cogs.watchchannels.watchchannel import proxy_user from bot.converters import TagNameConverter @@ -80,7 +84,7 @@ class Alias (Cog): await self.invoke(ctx, "site faq") @command(name="rules", aliases=("rule",), hidden=True) - async def site_rules_alias(self, ctx: Context, *rules: int) -> None: + async def site_rules_alias(self, ctx: Context, rules: Greedy[int], *_: str) -> None: """Alias for invoking <prefix>site rules.""" await self.invoke(ctx, "site rules", *rules) @@ -147,6 +151,5 @@ class Alias (Cog): def setup(bot: Bot) -> None: - """Alias cog load.""" + """Load the Alias cog.""" bot.add_cog(Alias(bot)) - log.info("Cog loaded: Alias") diff --git a/bot/cogs/antimalware.py b/bot/cogs/antimalware.py index 602819191..28e3e5d96 100644 --- a/bot/cogs/antimalware.py +++ b/bot/cogs/antimalware.py @@ -1,8 +1,9 @@ import logging from discord import Embed, Message, NotFound -from discord.ext.commands import Bot, Cog +from discord.ext.commands import Cog +from bot.bot import Bot from bot.constants import AntiMalware as AntiMalwareConfig, Channels, URLs log = logging.getLogger(__name__) @@ -49,6 +50,5 @@ class AntiMalware(Cog): def setup(bot: Bot) -> None: - """Antimalware cog load.""" + """Load the AntiMalware cog.""" bot.add_cog(AntiMalware(bot)) - log.info("Cog loaded: AntiMalware") diff --git a/bot/cogs/antispam.py b/bot/cogs/antispam.py index 1340eb608..f454061a6 100644 --- a/bot/cogs/antispam.py +++ b/bot/cogs/antispam.py @@ -7,9 +7,10 @@ from operator import itemgetter from typing import Dict, Iterable, List, Set from discord import Colour, Member, Message, NotFound, Object, TextChannel -from discord.ext.commands import Bot, Cog +from discord.ext.commands import Cog from bot import rules +from bot.bot import Bot from bot.cogs.moderation import ModLog from bot.constants import ( AntiSpam as AntiSpamConfig, Channels, @@ -276,7 +277,6 @@ def validate_config(rules: Mapping = AntiSpamConfig.rules) -> Dict[str, str]: def setup(bot: Bot) -> None: - """Antispam cog load.""" + """Validate the AntiSpam configs and load the AntiSpam cog.""" validation_errors = validate_config() bot.add_cog(AntiSpam(bot, validation_errors)) - log.info("Cog loaded: AntiSpam") diff --git a/bot/cogs/bot.py b/bot/cogs/bot.py index ee0a463de..73b1e8f41 100644 --- a/bot/cogs/bot.py +++ b/bot/cogs/bot.py @@ -5,8 +5,10 @@ import time from typing import Optional, Tuple from discord import Embed, Message, RawMessageUpdateEvent, TextChannel -from discord.ext.commands import Bot, Cog, Context, command, group +from discord.ext.commands import Cog, Context, command, group +from bot.bot import Bot +from bot.cogs.token_remover import TokenRemover from bot.constants import Channels, DEBUG_MODE, Guild, MODERATION_ROLES, Roles, URLs from bot.decorators import with_role from bot.utils.messages import wait_for_deletion @@ -16,7 +18,7 @@ log = logging.getLogger(__name__) RE_MARKDOWN = re.compile(r'([*_~`|>])') -class Bot(Cog): +class BotCog(Cog, name="Bot"): """Bot information commands.""" def __init__(self, bot: Bot): @@ -238,9 +240,10 @@ class Bot(Cog): ) and not msg.author.bot and len(msg.content.splitlines()) > 3 + and not TokenRemover.is_token_in_message(msg) ) - if parse_codeblock: + if parse_codeblock: # no token in the msg on_cooldown = (time.time() - self.channel_cooldowns.get(msg.channel.id, 0)) < 300 if not on_cooldown or DEBUG_MODE: try: @@ -373,10 +376,9 @@ class Bot(Cog): bot_message = await channel.fetch_message(self.codeblock_message_ids[payload.message_id]) await bot_message.delete() del self.codeblock_message_ids[payload.message_id] - log.trace("User's incorrect code block has been fixed. Removing bot formatting message.") + log.trace("User's incorrect code block has been fixed. Removing bot formatting message.") def setup(bot: Bot) -> None: - """Bot cog load.""" - bot.add_cog(Bot(bot)) - log.info("Cog loaded: Bot") + """Load the Bot cog.""" + bot.add_cog(BotCog(bot)) diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index dca411d01..2104efe57 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -3,9 +3,10 @@ import random import re from typing import Optional -from discord import Colour, Embed, Message, User -from discord.ext.commands import Bot, Cog, Context, group +from discord import Colour, Embed, Message, TextChannel, User +from discord.ext.commands import Cog, Context, group +from bot.bot import Bot from bot.cogs.moderation import ModLog from bot.constants import ( Channels, CleanMessages, Colours, Event, @@ -37,9 +38,13 @@ class Clean(Cog): return self.bot.get_cog("ModLog") async def _clean_messages( - self, amount: int, ctx: Context, - bots_only: bool = False, user: User = None, - regex: Optional[str] = None + self, + amount: int, + ctx: Context, + bots_only: bool = False, + user: User = None, + regex: Optional[str] = None, + channel: Optional[TextChannel] = None ) -> None: """A helper function that does the actual message cleaning.""" def predicate_bots_only(message: Message) -> bool: @@ -104,6 +109,10 @@ class Clean(Cog): else: predicate = None # Delete all messages + # Default to using the invoking context's channel + if not channel: + channel = ctx.channel + # Look through the history and retrieve message data messages = [] message_ids = [] @@ -111,7 +120,7 @@ class Clean(Cog): invocation_deleted = False # To account for the invocation message, we index `amount + 1` messages. - async for message in ctx.channel.history(limit=amount + 1): + async for message in channel.history(limit=amount + 1): # If at any point the cancel command is invoked, we should stop. if not self.cleaning: @@ -135,7 +144,7 @@ class Clean(Cog): self.mod_log.ignore(Event.message_delete, *message_ids) # Use bulk delete to actually do the cleaning. It's far faster. - await ctx.channel.purge( + await channel.purge( limit=amount, check=predicate ) @@ -155,7 +164,7 @@ class Clean(Cog): # Build the embed and send it message = ( - f"**{len(message_ids)}** messages deleted in <#{ctx.channel.id}> by **{ctx.author.name}**\n\n" + f"**{len(message_ids)}** messages deleted in <#{channel.id}> by **{ctx.author.name}**\n\n" f"A log of the deleted messages can be found [here]({log_url})." ) @@ -167,7 +176,7 @@ class Clean(Cog): channel_id=Channels.modlog, ) - @group(invoke_without_command=True, name="clean", hidden=True) + @group(invoke_without_command=True, name="clean", aliases=["purge"]) @with_role(*MODERATION_ROLES) async def clean_group(self, ctx: Context) -> None: """Commands for cleaning messages in channels.""" @@ -175,27 +184,49 @@ class Clean(Cog): @clean_group.command(name="user", aliases=["users"]) @with_role(*MODERATION_ROLES) - async def clean_user(self, ctx: Context, user: User, amount: int = 10) -> None: + async def clean_user( + self, + ctx: Context, + user: User, + amount: Optional[int] = 10, + channel: TextChannel = None + ) -> None: """Delete messages posted by the provided user, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, user=user) + await self._clean_messages(amount, ctx, user=user, channel=channel) @clean_group.command(name="all", aliases=["everything"]) @with_role(*MODERATION_ROLES) - async def clean_all(self, ctx: Context, amount: int = 10) -> None: + async def clean_all( + self, + ctx: Context, + amount: Optional[int] = 10, + channel: TextChannel = None + ) -> None: """Delete all messages, regardless of poster, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx) + await self._clean_messages(amount, ctx, channel=channel) @clean_group.command(name="bots", aliases=["bot"]) @with_role(*MODERATION_ROLES) - async def clean_bots(self, ctx: Context, amount: int = 10) -> None: + async def clean_bots( + self, + ctx: Context, + amount: Optional[int] = 10, + channel: TextChannel = None + ) -> None: """Delete all messages posted by a bot, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, bots_only=True) + await self._clean_messages(amount, ctx, bots_only=True, channel=channel) @clean_group.command(name="regex", aliases=["word", "expression"]) @with_role(*MODERATION_ROLES) - async def clean_regex(self, ctx: Context, regex: str, amount: int = 10) -> None: + async def clean_regex( + self, + ctx: Context, + regex: str, + amount: Optional[int] = 10, + channel: TextChannel = None + ) -> None: """Delete all messages that match a certain regex, stop cleaning after traversing `amount` messages.""" - await self._clean_messages(amount, ctx, regex=regex) + await self._clean_messages(amount, ctx, regex=regex, channel=channel) @clean_group.command(name="stop", aliases=["cancel", "abort"]) @with_role(*MODERATION_ROLES) @@ -211,6 +242,5 @@ class Clean(Cog): def setup(bot: Bot) -> None: - """Clean cog load.""" + """Load the Clean cog.""" bot.add_cog(Clean(bot)) - log.info("Cog loaded: Clean") diff --git a/bot/cogs/defcon.py b/bot/cogs/defcon.py index bedd70c86..3e7350fcc 100644 --- a/bot/cogs/defcon.py +++ b/bot/cogs/defcon.py @@ -6,8 +6,9 @@ from datetime import datetime, timedelta from enum import Enum from discord import Colour, Embed, Member -from discord.ext.commands import Bot, Cog, Context, group +from discord.ext.commands import Cog, Context, group +from bot.bot import Bot from bot.cogs.moderation import ModLog from bot.constants import Channels, Colours, Emojis, Event, Icons, Roles from bot.decorators import with_role @@ -236,6 +237,5 @@ class Defcon(Cog): def setup(bot: Bot) -> None: - """DEFCON cog load.""" + """Load the Defcon cog.""" bot.add_cog(Defcon(bot)) - log.info("Cog loaded: Defcon") diff --git a/bot/cogs/doc.py b/bot/cogs/doc.py index e5b3a4062..6e7c00b6a 100644 --- a/bot/cogs/doc.py +++ b/bot/cogs/doc.py @@ -5,6 +5,7 @@ import re import textwrap from collections import OrderedDict from contextlib import suppress +from types import SimpleNamespace from typing import Any, Callable, Optional, Tuple import discord @@ -17,6 +18,7 @@ from requests import ConnectTimeout, ConnectionError, HTTPError from sphinx.ext import intersphinx from urllib3.exceptions import ProtocolError +from bot.bot import Bot from bot.constants import MODERATION_ROLES, RedirectOutput from bot.converters import ValidPythonIdentifier, ValidURL from bot.decorators import with_role @@ -26,6 +28,16 @@ from bot.pagination import LinePaginator log = logging.getLogger(__name__) logging.getLogger('urllib3').setLevel(logging.WARNING) +# Since Intersphinx is intended to be used with Sphinx, +# we need to mock its configuration. +SPHINX_MOCK_APP = SimpleNamespace( + config=SimpleNamespace( + intersphinx_timeout=3, + tls_verify=True, + user_agent="python3:python-discord/bot:1.0.0" + ) +) + NO_OVERRIDE_GROUPS = ( "2to3fixer", "token", @@ -101,18 +113,6 @@ def markdownify(html: str) -> DocMarkdownConverter: return DocMarkdownConverter(bullets='•').convert(html) -class DummyObject(object): - """A dummy object which supports assigning anything, which the builtin `object()` does not support normally.""" - - -class SphinxConfiguration: - """Dummy configuration for use with intersphinx.""" - - config = DummyObject() - config.intersphinx_timeout = 3 - config.tls_verify = True - - class InventoryURL(commands.Converter): """ Represents an Intersphinx inventory URL. @@ -127,7 +127,7 @@ class InventoryURL(commands.Converter): async def convert(ctx: commands.Context, url: str) -> str: """Convert url to Intersphinx inventory URL.""" try: - intersphinx.fetch_inventory(SphinxConfiguration(), '', url) + intersphinx.fetch_inventory(SPHINX_MOCK_APP, '', url) except AttributeError: raise commands.BadArgument(f"Failed to fetch Intersphinx inventory from URL `{url}`.") except ConnectionError: @@ -147,7 +147,7 @@ class InventoryURL(commands.Converter): class Doc(commands.Cog): """A set of commands for querying & displaying documentation.""" - def __init__(self, bot: commands.Bot): + def __init__(self, bot: Bot): self.base_urls = {} self.bot = bot self.inventories = {} @@ -161,7 +161,7 @@ class Doc(commands.Cog): await self.refresh_inventory() async def update_single( - self, package_name: str, base_url: str, inventory_url: str, config: SphinxConfiguration + self, package_name: str, base_url: str, inventory_url: str ) -> None: """ Rebuild the inventory for a single package. @@ -172,12 +172,10 @@ class Doc(commands.Cog): absolute paths that link to specific symbols * `inventory_url` is the absolute URL to the intersphinx inventory, fetched by running `intersphinx.fetch_inventory` in an executor on the bot's event loop - * `config` is a `SphinxConfiguration` instance to mock the regular sphinx - project layout, required for use with intersphinx """ self.base_urls[package_name] = base_url - package = await self._fetch_inventory(inventory_url, config) + package = await self._fetch_inventory(inventory_url) if not package: return None @@ -219,15 +217,11 @@ class Doc(commands.Cog): self.renamed_symbols.clear() async_cache.cache = OrderedDict() - # Since Intersphinx is intended to be used with Sphinx, - # we need to mock its configuration. - config = SphinxConfiguration() - # Run all coroutines concurrently - since each of them performs a HTTP # request, this speeds up fetching the inventory data heavily. coros = [ self.update_single( - package["package"], package["base_url"], package["inventory_url"], config + package["package"], package["base_url"], package["inventory_url"] ) for package in await self.bot.api_client.get('bot/documentation-links') ] await asyncio.gather(*coros) @@ -305,10 +299,17 @@ class Doc(commands.Cog): # of a double newline (interpreted as a paragraph) before index 1000. if len(description) > 1000: shortened = description[:1000] - last_paragraph_end = shortened.rfind('\n\n', 100) - if last_paragraph_end == -1: - last_paragraph_end = shortened.rfind('. ') - description = description[:last_paragraph_end] + description_cutoff = shortened.rfind('\n\n', 100) + if description_cutoff == -1: + # Search the shortened version for cutoff points in decreasing desirability, + # cutoff at 1000 if none are found. + for string in (". ", ", ", ",", " "): + description_cutoff = shortened.rfind(string) + if description_cutoff != -1: + break + else: + description_cutoff = 1000 + description = description[:description_cutoff] # If there is an incomplete code block, cut it out if description.count("```") % 2: @@ -317,7 +318,6 @@ class Doc(commands.Cog): description += f"... [read more]({permalink})" description = WHITESPACE_AFTER_NEWLINES_RE.sub('', description) - if signatures is None: # If symbol is a module, don't show signature. embed_description = description @@ -469,9 +469,9 @@ class Doc(commands.Cog): ) await ctx.send(embed=embed) - async def _fetch_inventory(self, inventory_url: str, config: SphinxConfiguration) -> Optional[dict]: + async def _fetch_inventory(self, inventory_url: str) -> Optional[dict]: """Get and return inventory from `inventory_url`. If fetching fails, return None.""" - fetch_func = functools.partial(intersphinx.fetch_inventory, config, '', inventory_url) + fetch_func = functools.partial(intersphinx.fetch_inventory, SPHINX_MOCK_APP, '', inventory_url) for retry in range(1, FAILED_REQUEST_RETRY_AMOUNT+1): try: package = await self.bot.loop.run_in_executor(None, fetch_func) @@ -506,7 +506,6 @@ class Doc(commands.Cog): return tag.name == "table" -def setup(bot: commands.Bot) -> None: - """Doc cog load.""" +def setup(bot: Bot) -> None: + """Load the Doc cog.""" bot.add_cog(Doc(bot)) - log.info("Cog loaded: Doc") diff --git a/bot/cogs/duck_pond.py b/bot/cogs/duck_pond.py index 2d25cd17e..345d2856c 100644 --- a/bot/cogs/duck_pond.py +++ b/bot/cogs/duck_pond.py @@ -3,9 +3,10 @@ from typing import Optional, Union import discord from discord import Color, Embed, Member, Message, RawReactionActionEvent, User, errors -from discord.ext.commands import Bot, Cog +from discord.ext.commands import Cog from bot import constants +from bot.bot import Bot from bot.utils.messages import send_attachments log = logging.getLogger(__name__) @@ -177,6 +178,5 @@ class DuckPond(Cog): def setup(bot: Bot) -> None: - """Load the duck pond cog.""" + """Load the DuckPond cog.""" bot.add_cog(DuckPond(bot)) - log.info("Cog loaded: DuckPond") diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index 49411814c..52893b2ee 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -14,9 +14,10 @@ from discord.ext.commands import ( NoPrivateMessage, UserInputError, ) -from discord.ext.commands import Bot, Cog, Context +from discord.ext.commands import Cog, Context from bot.api import ResponseCodeError +from bot.bot import Bot from bot.constants import Channels from bot.decorators import InChannelCheckFailure @@ -75,6 +76,16 @@ class ErrorHandler(Cog): tags_get_command = self.bot.get_command("tags get") ctx.invoked_from_error_handler = True + log_msg = "Cancelling attempt to fall back to a tag due to failed checks." + try: + if not await tags_get_command.can_run(ctx): + log.debug(log_msg) + return + except CommandError as tag_error: + log.debug(log_msg) + await self.on_command_error(ctx, tag_error) + return + # Return to not raise the exception with contextlib.suppress(ResponseCodeError): await ctx.invoke(tags_get_command, tag_name=ctx.invoked_with) @@ -143,6 +154,5 @@ class ErrorHandler(Cog): def setup(bot: Bot) -> None: - """Error handler cog load.""" + """Load the ErrorHandler cog.""" bot.add_cog(ErrorHandler(bot)) - log.info("Cog loaded: Events") diff --git a/bot/cogs/eval.py b/bot/cogs/eval.py index 00b988dde..9c729f28a 100644 --- a/bot/cogs/eval.py +++ b/bot/cogs/eval.py @@ -9,8 +9,9 @@ from io import StringIO from typing import Any, Optional, Tuple import discord -from discord.ext.commands import Bot, Cog, Context, group +from discord.ext.commands import Cog, Context, group +from bot.bot import Bot from bot.constants import Roles from bot.decorators import with_role from bot.interpreter import Interpreter @@ -197,6 +198,5 @@ async def func(): # (None,) -> Any def setup(bot: Bot) -> None: - """Code eval cog load.""" + """Load the CodeEval cog.""" bot.add_cog(CodeEval(bot)) - log.info("Cog loaded: Eval") diff --git a/bot/cogs/extensions.py b/bot/cogs/extensions.py index bb66e0b8e..f16e79fb7 100644 --- a/bot/cogs/extensions.py +++ b/bot/cogs/extensions.py @@ -6,8 +6,9 @@ from pkgutil import iter_modules from discord import Colour, Embed from discord.ext import commands -from discord.ext.commands import Bot, Context, group +from discord.ext.commands import Context, group +from bot.bot import Bot from bot.constants import Emojis, MODERATION_ROLES, Roles, URLs from bot.pagination import LinePaginator from bot.utils.checks import with_role_check @@ -233,4 +234,3 @@ class Extensions(commands.Cog): def setup(bot: Bot) -> None: """Load the Extensions cog.""" bot.add_cog(Extensions(bot)) - log.info("Cog loaded: Extensions") diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 1e7521054..74538542a 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -5,8 +5,9 @@ from typing import Optional, Union import discord.errors from dateutil.relativedelta import relativedelta from discord import Colour, DMChannel, Member, Message, TextChannel -from discord.ext.commands import Bot, Cog +from discord.ext.commands import Cog +from bot.bot import Bot from bot.cogs.moderation import ModLog from bot.constants import ( Channels, Colours, @@ -370,6 +371,5 @@ class Filtering(Cog): def setup(bot: Bot) -> None: - """Filtering cog load.""" + """Load the Filtering cog.""" bot.add_cog(Filtering(bot)) - log.info("Cog loaded: Filtering") diff --git a/bot/cogs/free.py b/bot/cogs/free.py index 82285656b..49cab6172 100644 --- a/bot/cogs/free.py +++ b/bot/cogs/free.py @@ -3,8 +3,9 @@ from datetime import datetime from operator import itemgetter from discord import Colour, Embed, Member, utils -from discord.ext.commands import Bot, Cog, Context, command +from discord.ext.commands import Cog, Context, command +from bot.bot import Bot from bot.constants import Categories, Channels, Free, STAFF_ROLES from bot.decorators import redirect_output @@ -98,6 +99,5 @@ class Free(Cog): def setup(bot: Bot) -> None: - """Free cog load.""" + """Load the Free cog.""" bot.add_cog(Free()) - log.info("Cog loaded: Free") diff --git a/bot/cogs/help.py b/bot/cogs/help.py index 9607dbd8d..6385fa467 100644 --- a/bot/cogs/help.py +++ b/bot/cogs/help.py @@ -6,10 +6,11 @@ from typing import Union from discord import Colour, Embed, HTTPException, Message, Reaction, User from discord.ext import commands -from discord.ext.commands import Bot, CheckFailure, Cog as DiscordCog, Command, Context +from discord.ext.commands import CheckFailure, Cog as DiscordCog, Command, Context from fuzzywuzzy import fuzz, process from bot import constants +from bot.bot import Bot from bot.constants import Channels, STAFF_ROLES from bot.decorators import redirect_output from bot.pagination import ( diff --git a/bot/cogs/information.py b/bot/cogs/information.py index 530453600..1ede95ff4 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -9,10 +9,11 @@ from typing import Any, Mapping, Optional import discord from discord import CategoryChannel, Colour, Embed, Member, Role, TextChannel, VoiceChannel, utils from discord.ext import commands -from discord.ext.commands import Bot, BucketType, Cog, Context, command, group +from discord.ext.commands import BucketType, Cog, Context, command, group from discord.utils import escape_markdown from bot import constants +from bot.bot import Bot from bot.decorators import InChannelCheckFailure, in_channel, with_role from bot.utils.checks import cooldown_with_role_bypass, with_role_check from bot.utils.time import time_since @@ -391,6 +392,5 @@ class Information(Cog): def setup(bot: Bot) -> None: - """Information cog load.""" + """Load the Information cog.""" bot.add_cog(Information(bot)) - log.info("Cog loaded: Information") diff --git a/bot/cogs/jams.py b/bot/cogs/jams.py index be9d33e3e..985f28ce5 100644 --- a/bot/cogs/jams.py +++ b/bot/cogs/jams.py @@ -4,6 +4,7 @@ from discord import Member, PermissionOverwrite, utils from discord.ext import commands from more_itertools import unique_everseen +from bot.bot import Bot from bot.constants import Roles from bot.decorators import with_role @@ -13,7 +14,7 @@ log = logging.getLogger(__name__) class CodeJams(commands.Cog): """Manages the code-jam related parts of our server.""" - def __init__(self, bot: commands.Bot): + def __init__(self, bot: Bot): self.bot = bot @commands.command() @@ -108,7 +109,6 @@ class CodeJams(commands.Cog): ) -def setup(bot: commands.Bot) -> None: - """Code Jams cog load.""" +def setup(bot: Bot) -> None: + """Load the CodeJams cog.""" bot.add_cog(CodeJams(bot)) - log.info("Cog loaded: CodeJams") diff --git a/bot/cogs/logging.py b/bot/cogs/logging.py index c92b619ff..d1b7dcab3 100644 --- a/bot/cogs/logging.py +++ b/bot/cogs/logging.py @@ -1,8 +1,9 @@ import logging from discord import Embed -from discord.ext.commands import Bot, Cog +from discord.ext.commands import Cog +from bot.bot import Bot from bot.constants import Channels, DEBUG_MODE @@ -37,6 +38,5 @@ class Logging(Cog): def setup(bot: Bot) -> None: - """Logging cog load.""" + """Load the Logging cog.""" bot.add_cog(Logging(bot)) - log.info("Cog loaded: Logging") diff --git a/bot/cogs/metrics.py b/bot/cogs/metrics.py new file mode 100644 index 000000000..47c3cc55e --- /dev/null +++ b/bot/cogs/metrics.py @@ -0,0 +1,98 @@ +from collections import defaultdict + +from discord import Member, Message +from discord.ext.commands import Cog, Context +from prometheus_client import Counter, Gauge + +from bot.bot import Bot + + +class Metrics(Cog): + """ + Exports metrics for Prometheus. + + See https://github.com/prometheus/client_python for metric documentation. + """ + + PREFIX = 'pydis_bot' + + def __init__(self, bot: Bot) -> None: + self.bot = bot + + self.guild_members = Gauge( + name=f'{self.PREFIX}_guild_members', + documentation="Total members by guild by status.", + labelnames=('guild_id', 'status') + ) + self.guild_messages = Counter( + name=f'{self.PREFIX}_guild_messages', + documentation="Guild messages by guild by channel.", + labelnames=('channel_id', 'guild_id', 'channel_name') + ) + self.command_completions = Counter( + name=f'{self.PREFIX}_command_completions', + documentation="Completed commands by command, user, and guild.", + labelnames=('guild_id', 'user_id', 'user_name', 'command') + ) + + @Cog.listener() + async def on_ready(self) -> None: + """Initialize the guild member counter.""" + members_by_status = defaultdict(lambda: defaultdict(int)) + + for guild in self.bot.guilds: + if guild.large: + await self.bot.request_offline_members(guild) + for member in guild.members: + members_by_status[guild.id][member.status] += 1 + + for guild_id, members in members_by_status.items(): + for status, count in members.items(): + self.guild_members.labels(guild_id=guild_id, status=str(status)).set(count) + + @Cog.listener() + async def on_member_join(self, member: Member) -> None: + """Increment the member gauge.""" + self.guild_members.labels(guild_id=member.guild.id, status=str(member.status)).inc() + + @Cog.listener() + async def on_member_leave(self, member: Member) -> None: + """Decrement the member gauge.""" + self.guild_members.labels(guild_id=member.guild.id, status=str(member.status)).dec() + + @Cog.listener() + async def on_member_update(self, before: Member, after: Member) -> None: + """Update member gauges for the new and old status if applicable.""" + if before.status is not after.status: + self.guild_members.labels(guild_id=after.guild.id, status=str(before.status)).dec() + self.guild_members.labels(guild_id=after.guild.id, status=str(after.status)).inc() + + @Cog.listener() + async def on_message(self, message: Message) -> None: + """Increment the guild message counter.""" + self.guild_messages.labels( + channel_id=message.channel.id, + channel_name=message.channel.name, + guild_id=message.guild.id, + ).inc() + + @Cog.listener() + async def on_command_completion(self, ctx: Context) -> None: + """Increment the command completion counter.""" + if ctx.message.guild is not None: + if ctx.command.full_parent_name: + command = f'{ctx.command.full_parent_name} {ctx.command.name}' + else: + command = ctx.command.name + + self.command_completions.labels( + guild_id=ctx.message.guild.id, + user_id=ctx.author.id, + user_name=str(ctx.author), + command=command, + ).inc() + + +def setup(bot: Bot) -> None: + """Load the Metrics cog.""" + bot.add_cog(Metrics(bot)) diff --git a/bot/cogs/moderation/__init__.py b/bot/cogs/moderation/__init__.py index 7383ed44e..5243cb92d 100644 --- a/bot/cogs/moderation/__init__.py +++ b/bot/cogs/moderation/__init__.py @@ -1,25 +1,13 @@ -import logging - -from discord.ext.commands import Bot - +from bot.bot import Bot from .infractions import Infractions from .management import ModManagement from .modlog import ModLog from .superstarify import Superstarify -log = logging.getLogger(__name__) - def setup(bot: Bot) -> None: - """Load the moderation extension (Infractions, ModManagement, ModLog, & Superstarify cogs).""" + """Load the Infractions, ModManagement, ModLog, and Superstarify cogs.""" bot.add_cog(Infractions(bot)) - log.info("Cog loaded: Infractions") - bot.add_cog(ModLog(bot)) - log.info("Cog loaded: ModLog") - bot.add_cog(ModManagement(bot)) - log.info("Cog loaded: ModManagement") - bot.add_cog(Superstarify(bot)) - log.info("Cog loaded: Superstarify") diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 2713a1b68..fcfde1e68 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -7,6 +7,7 @@ from discord.ext import commands from discord.ext.commands import Context, command from bot import constants +from bot.bot import Bot from bot.constants import Event from bot.decorators import respect_role_hierarchy from bot.utils.checks import with_role_check @@ -25,7 +26,7 @@ class Infractions(InfractionScheduler, commands.Cog): category = "Moderation" category_description = "Server moderation tools." - def __init__(self, bot: commands.Bot): + def __init__(self, bot: Bot): super().__init__(bot, supported_infractions={"ban", "kick", "mute", "note", "warning"}) self.category = "Moderation" @@ -202,19 +203,24 @@ class Infractions(InfractionScheduler, commands.Cog): if await utils.has_active_infraction(ctx, user, "mute"): return - infraction = await utils.post_infraction(ctx, user, "mute", reason, **kwargs) + infraction = await utils.post_infraction(ctx, user, "mute", reason, active=True, **kwargs) if infraction is None: return self.mod_log.ignore(Event.member_update, user.id) - action = user.add_roles(self._muted_role, reason=reason) - await self.apply_infraction(ctx, infraction, user, action) + async def action() -> None: + await user.add_roles(self._muted_role, reason=reason) + + log.trace(f"Attempting to kick {user} from voice because they've been muted.") + await user.move_to(None, reason=reason) + + await self.apply_infraction(ctx, infraction, user, action()) @respect_role_hierarchy() async def apply_kick(self, ctx: Context, user: Member, reason: str, **kwargs) -> None: """Apply a kick infraction with kwargs passed to `post_infraction`.""" - infraction = await utils.post_infraction(ctx, user, "kick", reason, **kwargs) + infraction = await utils.post_infraction(ctx, user, "kick", reason, active=False, **kwargs) if infraction is None: return @@ -229,7 +235,7 @@ class Infractions(InfractionScheduler, commands.Cog): if await utils.has_active_infraction(ctx, user, "ban"): return - infraction = await utils.post_infraction(ctx, user, "ban", reason, **kwargs) + infraction = await utils.post_infraction(ctx, user, "ban", reason, active=True, **kwargs) if infraction is None: return diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index abfe5c2b3..9605d47b2 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -9,7 +9,8 @@ from discord.ext import commands from discord.ext.commands import Context from bot import constants -from bot.converters import InfractionSearchQuery +from bot.bot import Bot +from bot.converters import InfractionSearchQuery, allowed_strings from bot.pagination import LinePaginator from bot.utils import time from bot.utils.checks import in_channel_check, with_role_check @@ -22,21 +23,12 @@ log = logging.getLogger(__name__) UserConverter = t.Union[discord.User, utils.proxy_user] -def permanent_duration(expires_at: str) -> str: - """Only allow an expiration to be 'permanent' if it is a string.""" - expires_at = expires_at.lower() - if expires_at != "permanent": - raise commands.BadArgument - else: - return expires_at - - class ModManagement(commands.Cog): """Management of infractions.""" category = "Moderation" - def __init__(self, bot: commands.Bot): + def __init__(self, bot: Bot): self.bot = bot @property @@ -60,8 +52,8 @@ class ModManagement(commands.Cog): async def infraction_edit( self, ctx: Context, - infraction_id: int, - duration: t.Union[utils.Expiry, permanent_duration, None], + infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], + duration: t.Union[utils.Expiry, allowed_strings("p", "permanent"), None], *, reason: str = None ) -> None: @@ -78,21 +70,40 @@ class ModManagement(commands.Cog): \u2003`M` - minutes∗ \u2003`s` - seconds - Use "permanent" to mark the infraction as permanent. Alternatively, an ISO 8601 timestamp - can be provided for the duration. + Use "l", "last", or "recent" as the infraction ID to specify that the most recent infraction + authored by the command invoker should be edited. + + Use "p" or "permanent" to mark the infraction as permanent. Alternatively, an ISO 8601 + timestamp can be provided for the duration. """ if duration is None and reason is None: # Unlike UserInputError, the error handler will show a specified message for BadArgument raise commands.BadArgument("Neither a new expiry nor a new reason was specified.") # Retrieve the previous infraction for its information. - old_infraction = await self.bot.api_client.get(f'bot/infractions/{infraction_id}') + if isinstance(infraction_id, str): + params = { + "actor__id": ctx.author.id, + "ordering": "-inserted_at" + } + infractions = await self.bot.api_client.get(f"bot/infractions", params=params) + + if infractions: + old_infraction = infractions[0] + infraction_id = old_infraction["id"] + else: + await ctx.send( + f":x: Couldn't find most recent infraction; you have never given an infraction." + ) + return + else: + old_infraction = await self.bot.api_client.get(f"bot/infractions/{infraction_id}") request_data = {} confirm_messages = [] log_text = "" - if duration == "permanent": + if isinstance(duration, str): request_data['expires_at'] = None confirm_messages.append("marked as permanent") elif duration is not None: @@ -129,7 +140,8 @@ class ModManagement(commands.Cog): New expiry: {new_infraction['expires_at'] or "Permanent"} """.rstrip() - await ctx.send(f":ok_hand: Updated infraction: {' & '.join(confirm_messages)}") + changes = ' & '.join(confirm_messages) + await ctx.send(f":ok_hand: Updated infraction #{infraction_id}: {changes}") # Get information about the infraction's user user_id = new_infraction['user'] @@ -232,6 +244,12 @@ class ModManagement(commands.Cog): user_id = infraction["user"] hidden = infraction["hidden"] created = time.format_infraction(infraction["inserted_at"]) + + if active: + remaining = time.until_expiration(infraction["expires_at"]) or "Expired" + else: + remaining = "Inactive" + if infraction["expires_at"] is None: expires = "*Permanent*" else: @@ -247,6 +265,7 @@ class ModManagement(commands.Cog): Reason: {infraction["reason"] or "*None*"} Created: {created} Expires: {expires} + Remaining: {remaining} Actor: {actor.mention if actor else actor_id} ID: `{infraction["id"]}` {"**===============**" if active else "==============="} diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index 8509e9c07..7f24654d8 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -10,8 +10,9 @@ from dateutil.relativedelta import relativedelta from deepdiff import DeepDiff from discord import Colour from discord.abc import GuildChannel -from discord.ext.commands import Bot, Cog, Context +from discord.ext.commands import Cog, Context +from bot.bot import Bot from bot.constants import Channels, Colours, Emojis, Event, Guild as GuildConstant, Icons, URLs from bot.utils.time import humanize_delta from .utils import UserTypes diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index 3e0968121..01e4b1fe7 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -7,10 +7,11 @@ from gettext import ngettext import dateutil.parser import discord -from discord.ext.commands import Bot, Context +from discord.ext.commands import Context from bot import constants from bot.api import ResponseCodeError +from bot.bot import Bot from bot.constants import Colours, STAFF_CHANNELS from bot.utils import time from bot.utils.scheduling import Scheduler @@ -113,8 +114,8 @@ class InfractionScheduler(Scheduler): dm_result = ":incoming_envelope: " dm_log_text = "\nDM: Sent" else: + dm_result = f"{constants.Emojis.failmail} " dm_log_text = "\nDM: **Failed**" - log_content = ctx.author.mention if infraction["actor"] == self.bot.user.id: log.trace( @@ -146,14 +147,18 @@ class InfractionScheduler(Scheduler): if expiry: # Schedule the expiration of the infraction. self.schedule_task(ctx.bot.loop, infraction["id"], infraction) - except discord.Forbidden: + except discord.HTTPException as e: # Accordingly display that applying the infraction failed. confirm_msg = f":x: failed to apply" expiry_msg = "" log_content = ctx.author.mention log_title = "failed to apply" - log.warning(f"Failed to apply {infr_type} infraction #{id_} to {user}.") + log_msg = f"Failed to apply {infr_type} infraction #{id_} to {user}" + if isinstance(e, discord.Forbidden): + log.warning(f"{log_msg}: bot lacks permissions.") + else: + log.exception(log_msg) # Send a confirmation message to the invoking context. log.trace(f"Sending infraction #{id_} confirmation message.") @@ -250,8 +255,7 @@ class InfractionScheduler(Scheduler): if log_text.get("DM") == "Sent": dm_emoji = ":incoming_envelope: " elif "DM" in log_text: - # Mention the actor because the DM failed to send. - log_content = ctx.author.mention + dm_emoji = f"{constants.Emojis.failmail} " # Accordingly display whether the pardon failed. if "Failure" in log_text: @@ -324,12 +328,12 @@ class InfractionScheduler(Scheduler): f"Attempted to deactivate an unsupported infraction #{id_} ({type_})!" ) except discord.Forbidden: - log.warning(f"Failed to deactivate infraction #{id_} ({type_}): bot lacks permissions") + log.warning(f"Failed to deactivate infraction #{id_} ({type_}): bot lacks permissions.") log_text["Failure"] = f"The bot lacks permissions to do this (role hierarchy?)" log_content = mod_role.mention except discord.HTTPException as e: log.exception(f"Failed to deactivate infraction #{id_} ({type_})") - log_text["Failure"] = f"HTTPException with code {e.code}." + log_text["Failure"] = f"HTTPException with status {e.status} and code {e.code}." log_content = mod_role.mention # Check if the user is currently being watched by Big Brother. diff --git a/bot/cogs/moderation/superstarify.py b/bot/cogs/moderation/superstarify.py index 9b3c62403..1e19e943e 100644 --- a/bot/cogs/moderation/superstarify.py +++ b/bot/cogs/moderation/superstarify.py @@ -6,9 +6,10 @@ import typing as t from pathlib import Path from discord import Colour, Embed, Member -from discord.ext.commands import Bot, Cog, Context, command +from discord.ext.commands import Cog, Context, command from bot import constants +from bot.bot import Bot from bot.utils.checks import with_role_check from bot.utils.time import format_infraction from . import utils @@ -132,7 +133,7 @@ class Superstarify(InfractionScheduler, Cog): # Post the infraction to the API reason = reason or f"old nick: {member.display_name}" - infraction = await utils.post_infraction(ctx, member, "superstar", reason, duration) + infraction = await utils.post_infraction(ctx, member, "superstar", reason, duration, active=True) id_ = infraction["id"] old_nick = member.display_name diff --git a/bot/cogs/off_topic_names.py b/bot/cogs/off_topic_names.py index 78792240f..bf777ea5a 100644 --- a/bot/cogs/off_topic_names.py +++ b/bot/cogs/off_topic_names.py @@ -4,9 +4,10 @@ import logging from datetime import datetime, timedelta from discord import Colour, Embed -from discord.ext.commands import BadArgument, Bot, Cog, Context, Converter, group +from discord.ext.commands import BadArgument, Cog, Context, Converter, group from bot.api import ResponseCodeError +from bot.bot import Bot from bot.constants import Channels, MODERATION_ROLES from bot.decorators import with_role from bot.pagination import LinePaginator @@ -184,6 +185,5 @@ class OffTopicNames(Cog): def setup(bot: Bot) -> None: - """Off topic names cog load.""" + """Load the OffTopicNames cog.""" bot.add_cog(OffTopicNames(bot)) - log.info("Cog loaded: OffTopicNames") diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index 0d06e9c26..aa487f18e 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -2,13 +2,16 @@ import asyncio import logging import random import textwrap +from collections import namedtuple from datetime import datetime, timedelta from typing import List +from aiohttp import BasicAuth, ClientError from discord import Colour, Embed, TextChannel -from discord.ext.commands import Bot, Cog, Context, group +from discord.ext.commands import Cog, Context, group from discord.ext.tasks import loop +from bot.bot import Bot from bot.constants import Channels, ERROR_REPLIES, Emojis, Reddit as RedditConfig, STAFF_ROLES, Webhooks from bot.converters import Subreddit from bot.decorators import with_role @@ -16,25 +19,32 @@ from bot.pagination import LinePaginator log = logging.getLogger(__name__) +AccessToken = namedtuple("AccessToken", ["token", "expires_at"]) + class Reddit(Cog): """Track subreddit posts and show detailed statistics about them.""" - HEADERS = {"User-Agent": "Discord Bot: PythonDiscord (https://pythondiscord.com/)"} + HEADERS = {"User-Agent": "python3:python-discord/bot:1.0.0 (by /u/PythonDiscord)"} URL = "https://www.reddit.com" - MAX_FETCH_RETRIES = 3 + OAUTH_URL = "https://oauth.reddit.com" + MAX_RETRIES = 3 def __init__(self, bot: Bot): self.bot = bot - self.webhook = None # set in on_ready - bot.loop.create_task(self.init_reddit_ready()) + self.webhook = None + self.access_token = None + self.client_auth = BasicAuth(RedditConfig.client_id, RedditConfig.secret) + bot.loop.create_task(self.init_reddit_ready()) self.auto_poster_loop.start() def cog_unload(self) -> None: - """Stops the loops when the cog is unloaded.""" + """Stop the loop task and revoke the access token when the cog is unloaded.""" self.auto_poster_loop.cancel() + if self.access_token.expires_at < datetime.utcnow(): + self.revoke_access_token() async def init_reddit_ready(self) -> None: """Sets the reddit webhook when the cog is loaded.""" @@ -47,20 +57,82 @@ class Reddit(Cog): """Get the #reddit channel object from the bot's cache.""" return self.bot.get_channel(Channels.reddit) + async def get_access_token(self) -> None: + """ + Get a Reddit API OAuth2 access token and assign it to self.access_token. + + A token is valid for 1 hour. There will be MAX_RETRIES to get a token, after which the cog + will be unloaded and a ClientError raised if retrieval was still unsuccessful. + """ + for i in range(1, self.MAX_RETRIES + 1): + response = await self.bot.http_session.post( + url=f"{self.URL}/api/v1/access_token", + headers=self.HEADERS, + auth=self.client_auth, + data={ + "grant_type": "client_credentials", + "duration": "temporary" + } + ) + + if response.status == 200 and response.content_type == "application/json": + content = await response.json() + expiration = int(content["expires_in"]) - 60 # Subtract 1 minute for leeway. + self.access_token = AccessToken( + token=content["access_token"], + expires_at=datetime.utcnow() + timedelta(seconds=expiration) + ) + + log.debug(f"New token acquired; expires on {self.access_token.expires_at}") + return + else: + log.debug( + f"Failed to get an access token: " + f"status {response.status} & content type {response.content_type}; " + f"retrying ({i}/{self.MAX_RETRIES})" + ) + + await asyncio.sleep(3) + + self.bot.remove_cog(self.qualified_name) + raise ClientError("Authentication with the Reddit API failed. Unloading the cog.") + + async def revoke_access_token(self) -> None: + """ + Revoke the OAuth2 access token for the Reddit API. + + For security reasons, it's good practice to revoke the token when it's no longer being used. + """ + response = await self.bot.http_session.post( + url=f"{self.URL}/api/v1/revoke_token", + headers=self.HEADERS, + auth=self.client_auth, + data={ + "token": self.access_token.token, + "token_type_hint": "access_token" + } + ) + + if response.status == 204 and response.content_type == "application/json": + self.access_token = None + else: + log.warning(f"Unable to revoke access token: status {response.status}.") + async def fetch_posts(self, route: str, *, amount: int = 25, params: dict = None) -> List[dict]: """A helper method to fetch a certain amount of Reddit posts at a given route.""" # Reddit's JSON responses only provide 25 posts at most. if not 25 >= amount > 0: raise ValueError("Invalid amount of subreddit posts requested.") - if params is None: - params = {} + # Renew the token if necessary. + if not self.access_token or self.access_token.expires_at < datetime.utcnow(): + await self.get_access_token() - url = f"{self.URL}/{route}.json" - for _ in range(self.MAX_FETCH_RETRIES): + url = f"{self.OAUTH_URL}/{route}" + for _ in range(self.MAX_RETRIES): response = await self.bot.http_session.get( url=url, - headers=self.HEADERS, + headers={**self.HEADERS, "Authorization": f"bearer {self.access_token.token}"}, params=params ) if response.status == 200 and response.content_type == 'application/json': @@ -217,6 +289,5 @@ class Reddit(Cog): def setup(bot: Bot) -> None: - """Reddit cog load.""" + """Load the Reddit cog.""" bot.add_cog(Reddit(bot)) - log.info("Cog loaded: Reddit") diff --git a/bot/cogs/reminders.py b/bot/cogs/reminders.py index 81990704b..45bf9a8f4 100644 --- a/bot/cogs/reminders.py +++ b/bot/cogs/reminders.py @@ -8,8 +8,9 @@ from typing import Optional from dateutil.relativedelta import relativedelta from discord import Colour, Embed, Message -from discord.ext.commands import Bot, Cog, Context, group +from discord.ext.commands import Cog, Context, group +from bot.bot import Bot from bot.constants import Channels, Icons, NEGATIVE_REPLIES, POSITIVE_REPLIES, STAFF_ROLES from bot.converters import Duration from bot.pagination import LinePaginator @@ -290,6 +291,5 @@ class Reminders(Scheduler, Cog): def setup(bot: Bot) -> None: - """Reminders cog load.""" + """Load the Reminders cog.""" bot.add_cog(Reminders(bot)) - log.info("Cog loaded: Reminders") diff --git a/bot/cogs/security.py b/bot/cogs/security.py index 316b33d6b..c680c5e27 100644 --- a/bot/cogs/security.py +++ b/bot/cogs/security.py @@ -1,6 +1,8 @@ import logging -from discord.ext.commands import Bot, Cog, Context, NoPrivateMessage +from discord.ext.commands import Cog, Context, NoPrivateMessage + +from bot.bot import Bot log = logging.getLogger(__name__) @@ -25,6 +27,5 @@ class Security(Cog): def setup(bot: Bot) -> None: - """Security cog load.""" + """Load the Security cog.""" bot.add_cog(Security(bot)) - log.info("Cog loaded: Security") diff --git a/bot/cogs/site.py b/bot/cogs/site.py index 683613788..2ea8c7a2e 100644 --- a/bot/cogs/site.py +++ b/bot/cogs/site.py @@ -1,8 +1,9 @@ import logging from discord import Colour, Embed -from discord.ext.commands import Bot, Cog, Context, group +from discord.ext.commands import Cog, Context, group +from bot.bot import Bot from bot.constants import URLs from bot.pagination import LinePaginator @@ -138,6 +139,5 @@ class Site(Cog): def setup(bot: Bot) -> None: - """Site cog load.""" + """Load the Site cog.""" bot.add_cog(Site(bot)) - log.info("Cog loaded: Site") diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index 55a187ac1..da33e27b2 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -5,8 +5,9 @@ import textwrap from signal import Signals from typing import Optional, Tuple -from discord.ext.commands import Bot, Cog, Context, command, guild_only +from discord.ext.commands import Cog, Context, command, guild_only +from bot.bot import Bot from bot.constants import Channels, Roles, URLs from bot.decorators import in_channel from bot.utils.messages import wait_for_deletion @@ -227,6 +228,5 @@ class Snekbox(Cog): def setup(bot: Bot) -> None: - """Snekbox cog load.""" + """Load the Snekbox cog.""" bot.add_cog(Snekbox(bot)) - log.info("Cog loaded: Snekbox") diff --git a/bot/cogs/sync/__init__.py b/bot/cogs/sync/__init__.py index d4565f848..fe7df4e9b 100644 --- a/bot/cogs/sync/__init__.py +++ b/bot/cogs/sync/__init__.py @@ -1,13 +1,7 @@ -import logging - -from discord.ext.commands import Bot - +from bot.bot import Bot from .cog import Sync -log = logging.getLogger(__name__) - def setup(bot: Bot) -> None: - """Sync cog load.""" + """Load the Sync cog.""" bot.add_cog(Sync(bot)) - log.info("Cog loaded: Sync") diff --git a/bot/cogs/sync/cog.py b/bot/cogs/sync/cog.py index aaa581f96..4e6ed156b 100644 --- a/bot/cogs/sync/cog.py +++ b/bot/cogs/sync/cog.py @@ -1,12 +1,13 @@ import logging -from typing import Callable, Iterable +from typing import Callable, Dict, Iterable, Union -from discord import Guild, Member, Role +from discord import Guild, Member, Role, User from discord.ext import commands -from discord.ext.commands import Bot, Cog, Context +from discord.ext.commands import Cog, Context from bot import constants from bot.api import ResponseCodeError +from bot.bot import Bot from bot.cogs.sync import syncers log = logging.getLogger(__name__) @@ -50,6 +51,15 @@ class Sync(Cog): f"deleted `{total_deleted}`." ) + async def patch_user(self, user_id: int, updated_information: Dict[str, Union[str, int]]) -> None: + """Send a PATCH request to partially update a user in the database.""" + try: + await self.bot.api_client.patch("bot/users/" + str(user_id), json=updated_information) + except ResponseCodeError as e: + if e.response.status != 404: + raise + log.warning("Unable to update user, got 404. Assuming race condition from join event.") + @Cog.listener() async def on_guild_role_create(self, role: Role) -> None: """Adds newly create role to the database table over the API.""" @@ -142,33 +152,21 @@ class Sync(Cog): @Cog.listener() async def on_member_update(self, before: Member, after: Member) -> None: - """Updates the user information if any of relevant attributes have changed.""" - if ( - before.name != after.name - or before.avatar != after.avatar - or before.discriminator != after.discriminator - or before.roles != after.roles - ): - try: - await self.bot.api_client.put( - 'bot/users/' + str(after.id), - json={ - 'avatar_hash': after.avatar, - 'discriminator': int(after.discriminator), - 'id': after.id, - 'in_guild': True, - 'name': after.name, - 'roles': sorted(role.id for role in after.roles) - } - ) - except ResponseCodeError as e: - if e.response.status != 404: - raise - - log.warning( - "Unable to update user, got 404. " - "Assuming race condition from join event." - ) + """Update the roles of the member in the database if a change is detected.""" + if before.roles != after.roles: + updated_information = {"roles": sorted(role.id for role in after.roles)} + await self.patch_user(after.id, updated_information=updated_information) + + @Cog.listener() + async def on_user_update(self, before: User, after: User) -> None: + """Update the user information in the database if a relevant change is detected.""" + if any(getattr(before, attr) != getattr(after, attr) for attr in ("name", "discriminator", "avatar")): + updated_information = { + "name": after.name, + "discriminator": int(after.discriminator), + "avatar_hash": after.avatar, + } + await self.patch_user(after.id, updated_information=updated_information) @commands.group(name='sync') @commands.has_permissions(administrator=True) diff --git a/bot/cogs/sync/syncers.py b/bot/cogs/sync/syncers.py index 2cc5a66e1..14cf51383 100644 --- a/bot/cogs/sync/syncers.py +++ b/bot/cogs/sync/syncers.py @@ -2,7 +2,8 @@ from collections import namedtuple from typing import Dict, Set, Tuple from discord import Guild -from discord.ext.commands import Bot + +from bot.bot import Bot # These objects are declared as namedtuples because tuples are hashable, # something that we make use of when diffing site roles against guild roles. @@ -52,7 +53,7 @@ async def sync_roles(bot: Bot, guild: Guild) -> Tuple[int, int, int]: Synchronize roles found on the given `guild` with the ones on the API. Arguments: - bot (discord.ext.commands.Bot): + bot (bot.bot.Bot): The bot instance that we're running with. guild (discord.Guild): @@ -169,7 +170,7 @@ async def sync_users(bot: Bot, guild: Guild) -> Tuple[int, int, None]: Synchronize users found in the given `guild` with the ones in the API. Arguments: - bot (discord.ext.commands.Bot): + bot (bot.bot.Bot): The bot instance that we're running with. guild (discord.Guild): diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index cd70e783a..970301013 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -2,8 +2,9 @@ import logging import time from discord import Colour, Embed -from discord.ext.commands import Bot, Cog, Context, group +from discord.ext.commands import Cog, Context, group +from bot.bot import Bot from bot.constants import Channels, Cooldowns, MODERATION_ROLES, Roles from bot.converters import TagContentConverter, TagNameConverter from bot.decorators import with_role @@ -160,6 +161,5 @@ class Tags(Cog): def setup(bot: Bot) -> None: - """Tags cog load.""" + """Load the Tags cog.""" bot.add_cog(Tags(bot)) - log.info("Cog loaded: Tags") diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index 5a0d20e57..82c01ae96 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -6,9 +6,10 @@ import struct from datetime import datetime from discord import Colour, Message -from discord.ext.commands import Bot, Cog +from discord.ext.commands import Cog from discord.utils import snowflake_time +from bot.bot import Bot from bot.cogs.moderation import ModLog from bot.constants import Channels, Colours, Event, Icons @@ -52,39 +53,60 @@ class TokenRemover(Cog): See: https://discordapp.com/developers/docs/reference#snowflakes """ + if self.is_token_in_message(msg): + await self.take_action(msg) + + @Cog.listener() + async def on_message_edit(self, before: Message, after: Message) -> None: + """ + Check each edit for a string that matches Discord's token pattern. + + See: https://discordapp.com/developers/docs/reference#snowflakes + """ + if self.is_token_in_message(after): + await self.take_action(after) + + async def take_action(self, msg: Message) -> None: + """Remove the `msg` containing a token an send a mod_log message.""" + user_id, creation_timestamp, hmac = TOKEN_RE.search(msg.content).group(0).split('.') + self.mod_log.ignore(Event.message_delete, msg.id) + await msg.delete() + await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention)) + + message = ( + "Censored a seemingly valid token sent by " + f"{msg.author} (`{msg.author.id}`) in {msg.channel.mention}, token was " + f"`{user_id}.{creation_timestamp}.{'x' * len(hmac)}`" + ) + log.debug(message) + + # Send pretty mod log embed to mod-alerts + await self.mod_log.send_log_message( + icon_url=Icons.token_removed, + colour=Colour(Colours.soft_red), + title="Token removed!", + text=message, + thumbnail=msg.author.avatar_url_as(static_format="png"), + channel_id=Channels.mod_alerts, + ) + + @classmethod + def is_token_in_message(cls, msg: Message) -> bool: + """Check if `msg` contains a seemly valid token.""" if msg.author.bot: - return + return False maybe_match = TOKEN_RE.search(msg.content) if maybe_match is None: - return + return False try: user_id, creation_timestamp, hmac = maybe_match.group(0).split('.') except ValueError: - return - - if self.is_valid_user_id(user_id) and self.is_valid_timestamp(creation_timestamp): - self.mod_log.ignore(Event.message_delete, msg.id) - await msg.delete() - await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention)) - - message = ( - "Censored a seemingly valid token sent by " - f"{msg.author} (`{msg.author.id}`) in {msg.channel.mention}, token was " - f"`{user_id}.{creation_timestamp}.{'x' * len(hmac)}`" - ) - log.debug(message) - - # Send pretty mod log embed to mod-alerts - await self.mod_log.send_log_message( - icon_url=Icons.token_removed, - colour=Colour(Colours.soft_red), - title="Token removed!", - text=message, - thumbnail=msg.author.avatar_url_as(static_format="png"), - channel_id=Channels.mod_alerts, - ) + return False + + if cls.is_valid_user_id(user_id) and cls.is_valid_timestamp(creation_timestamp): + return True @staticmethod def is_valid_user_id(b64_content: str) -> bool: @@ -119,6 +141,5 @@ class TokenRemover(Cog): def setup(bot: Bot) -> None: - """Token Remover cog load.""" + """Load the TokenRemover cog.""" bot.add_cog(TokenRemover(bot)) - log.info("Cog loaded: TokenRemover") diff --git a/bot/cogs/utils.py b/bot/cogs/utils.py index 793fe4c1a..da278011a 100644 --- a/bot/cogs/utils.py +++ b/bot/cogs/utils.py @@ -8,8 +8,9 @@ from typing import Tuple from dateutil import relativedelta from discord import Colour, Embed, Message, Role -from discord.ext.commands import Bot, Cog, Context, command +from discord.ext.commands import Cog, Context, command +from bot.bot import Bot from bot.constants import Channels, MODERATION_ROLES, Mention, STAFF_ROLES from bot.decorators import in_channel, with_role from bot.utils.time import humanize_delta @@ -61,14 +62,12 @@ class Utils(Cog): pep_embed.set_thumbnail(url="https://www.python.org/static/opengraph-icon-200x200.png") # Add the interesting information - if "Status" in pep_header: - pep_embed.add_field(name="Status", value=pep_header["Status"]) - if "Python-Version" in pep_header: - pep_embed.add_field(name="Python-Version", value=pep_header["Python-Version"]) - if "Created" in pep_header: - pep_embed.add_field(name="Created", value=pep_header["Created"]) - if "Type" in pep_header: - pep_embed.add_field(name="Type", value=pep_header["Type"]) + fields_to_check = ("Status", "Python-Version", "Created", "Type") + for field in fields_to_check: + # Check for a PEP metadata field that is present but has an empty value + # embed field values can't contain an empty string + if pep_header.get(field, ""): + pep_embed.add_field(name=field, value=pep_header[field]) elif response.status != 404: # any response except 200 and 404 is expected @@ -176,6 +175,5 @@ class Utils(Cog): def setup(bot: Bot) -> None: - """Utils cog load.""" + """Load the Utils cog.""" bot.add_cog(Utils(bot)) - log.info("Cog loaded: Utils") diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index b5e8d4357..988e0d49a 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -3,15 +3,17 @@ from datetime import datetime from discord import Colour, Message, NotFound, Object from discord.ext import tasks -from discord.ext.commands import Bot, Cog, Context, command +from discord.ext.commands import Cog, Context, command +from bot.bot import Bot from bot.cogs.moderation import ModLog from bot.constants import ( Bot as BotConfig, Channels, Colours, Event, - Filter, Icons, Roles + Filter, Icons, MODERATION_ROLES, Roles ) from bot.decorators import InChannelCheckFailure, in_channel, without_role +from bot.utils.checks import without_role_check log = logging.getLogger(__name__) @@ -37,6 +39,7 @@ PERIODIC_PING = ( f"@everyone To verify that you have read our rules, please type `{BotConfig.prefix}accept`." f" If you encounter any problems during the verification process, ping the <@&{Roles.admin}> role in this channel." ) +BOT_MESSAGE_DELETE_DELAY = 10 class Verification(Cog): @@ -54,12 +57,16 @@ class Verification(Cog): @Cog.listener() async def on_message(self, message: Message) -> None: """Check new message event for messages to the checkpoint channel & process.""" - if message.author.bot: - return # They're a bot, ignore - if message.channel.id != Channels.verification: return # Only listen for #checkpoint messages + if message.author.bot: + # They're a bot, delete their message after the delay. + # But not the periodic ping; we like that one. + if message.content != PERIODIC_PING: + await message.delete(delay=BOT_MESSAGE_DELETE_DELAY) + return + # if a user mentions a role or guild member # alert the mods in mod-alerts channel if message.mentions or message.role_mentions: @@ -189,7 +196,7 @@ class Verification(Cog): @staticmethod def bot_check(ctx: Context) -> bool: """Block any command within the verification channel that is not !accept.""" - if ctx.channel.id == Channels.verification: + if ctx.channel.id == Channels.verification and without_role_check(ctx, *MODERATION_ROLES): return ctx.command.name == "accept" else: return True @@ -224,6 +231,5 @@ class Verification(Cog): def setup(bot: Bot) -> None: - """Verification cog load.""" + """Load the Verification cog.""" bot.add_cog(Verification(bot)) - log.info("Cog loaded: Verification") diff --git a/bot/cogs/watchchannels/__init__.py b/bot/cogs/watchchannels/__init__.py index 86e1050fa..69d118df6 100644 --- a/bot/cogs/watchchannels/__init__.py +++ b/bot/cogs/watchchannels/__init__.py @@ -1,18 +1,9 @@ -import logging - -from discord.ext.commands import Bot - +from bot.bot import Bot from .bigbrother import BigBrother from .talentpool import TalentPool -log = logging.getLogger(__name__) - - def setup(bot: Bot) -> None: - """Monitoring cogs load.""" + """Load the BigBrother and TalentPool cogs.""" bot.add_cog(BigBrother(bot)) - log.info("Cog loaded: BigBrother") - bot.add_cog(TalentPool(bot)) - log.info("Cog loaded: TalentPool") diff --git a/bot/cogs/watchchannels/bigbrother.py b/bot/cogs/watchchannels/bigbrother.py index 49783bb09..28c1681cf 100644 --- a/bot/cogs/watchchannels/bigbrother.py +++ b/bot/cogs/watchchannels/bigbrother.py @@ -3,8 +3,9 @@ from collections import ChainMap from typing import Union from discord import User -from discord.ext.commands import Bot, Cog, Context, group +from discord.ext.commands import Cog, Context, group +from bot.bot import Bot from bot.cogs.moderation.utils import post_infraction from bot.constants import Channels, MODERATION_ROLES, Webhooks from bot.decorators import with_role @@ -61,10 +62,10 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): return if user.id in self.watched_users: - await ctx.send(":x: The specified user is already being watched.") + await ctx.send(f":x: {user} is already being watched.") return - response = await post_infraction(ctx, user, 'watch', reason, hidden=True) + response = await post_infraction(ctx, user, 'watch', reason, hidden=True, active=True) if response is not None: self.watched_users[user.id] = response diff --git a/bot/cogs/watchchannels/talentpool.py b/bot/cogs/watchchannels/talentpool.py index 4ec42dcc1..f990ccff8 100644 --- a/bot/cogs/watchchannels/talentpool.py +++ b/bot/cogs/watchchannels/talentpool.py @@ -4,9 +4,10 @@ from collections import ChainMap from typing import Union from discord import Color, Embed, Member, User -from discord.ext.commands import Bot, Cog, Context, group +from discord.ext.commands import Cog, Context, group from bot.api import ResponseCodeError +from bot.bot import Bot from bot.constants import Channels, Guild, MODERATION_ROLES, STAFF_ROLES, Webhooks from bot.decorators import with_role from bot.pagination import LinePaginator @@ -68,7 +69,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"): return if user.id in self.watched_users: - await ctx.send(":x: The specified user is already being watched in the talent pool") + await ctx.send(f":x: {user} is already being watched in the talent pool") return # Manual request with `raise_for_status` as False because we want the actual response diff --git a/bot/cogs/watchchannels/watchchannel.py b/bot/cogs/watchchannels/watchchannel.py index 0bf75a924..bd0622554 100644 --- a/bot/cogs/watchchannels/watchchannel.py +++ b/bot/cogs/watchchannels/watchchannel.py @@ -10,9 +10,10 @@ from typing import Optional import dateutil.parser import discord from discord import Color, Embed, HTTPException, Message, Object, errors -from discord.ext.commands import BadArgument, Bot, Cog, Context +from discord.ext.commands import BadArgument, Cog, Context from bot.api import ResponseCodeError +from bot.bot import Bot from bot.cogs.moderation import ModLog from bot.constants import BigBrother as BigBrotherConfig, Guild as GuildConfig, Icons from bot.pagination import LinePaginator diff --git a/bot/cogs/wolfram.py b/bot/cogs/wolfram.py index ab0ed2472..5d6b4630b 100644 --- a/bot/cogs/wolfram.py +++ b/bot/cogs/wolfram.py @@ -7,8 +7,9 @@ import discord from dateutil.relativedelta import relativedelta from discord import Embed from discord.ext import commands -from discord.ext.commands import Bot, BucketType, Cog, Context, check, group +from discord.ext.commands import BucketType, Cog, Context, check, group +from bot.bot import Bot from bot.constants import Colours, STAFF_ROLES, Wolfram from bot.pagination import ImagePaginator from bot.utils.time import humanize_delta @@ -151,7 +152,7 @@ async def get_pod_pages(ctx: Context, bot: Bot, query: str) -> Optional[List[Tup class Wolfram(Cog): """Commands for interacting with the Wolfram|Alpha API.""" - def __init__(self, bot: commands.Bot): + def __init__(self, bot: Bot): self.bot = bot @group(name="wolfram", aliases=("wolf", "wa"), invoke_without_command=True) @@ -266,7 +267,6 @@ class Wolfram(Cog): await send_embed(ctx, message, color) -def setup(bot: commands.Bot) -> None: - """Wolfram cog load.""" +def setup(bot: Bot) -> None: + """Load the Wolfram cog.""" bot.add_cog(Wolfram(bot)) - log.info("Cog loaded: Wolfram") diff --git a/bot/constants.py b/bot/constants.py index 725792516..25c7856ba 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -256,6 +256,8 @@ class Emojis(metaclass=YAMLGetter): status_idle: str status_dnd: str + failmail: str + bullet: str new: str pencil: str @@ -268,6 +270,12 @@ class Emojis(metaclass=YAMLGetter): ducky_ninja: int ducky_devil: int ducky_tube: int + ducky_hunt: int + ducky_wizard: int + ducky_party: int + ducky_angel: int + ducky_maul: int + ducky_santa: int upvotes: str comments: str @@ -470,6 +478,8 @@ class Reddit(metaclass=YAMLGetter): section = "reddit" subreddits: list + client_id: str + secret: str class Wolfram(metaclass=YAMLGetter): diff --git a/bot/converters.py b/bot/converters.py index cf0496541..8d2ab7eb8 100644 --- a/bot/converters.py +++ b/bot/converters.py @@ -1,8 +1,8 @@ import logging import re +import typing as t from datetime import datetime from ssl import CertificateError -from typing import Union import dateutil.parser import dateutil.tz @@ -15,6 +15,25 @@ from discord.ext.commands import BadArgument, Context, Converter log = logging.getLogger(__name__) +def allowed_strings(*values, preserve_case: bool = False) -> t.Callable[[str], str]: + """ + Return a converter which only allows arguments equal to one of the given values. + + Unless preserve_case is True, the argument is converted to lowercase. All values are then + expected to have already been given in lowercase too. + """ + def converter(arg: str) -> str: + if not preserve_case: + arg = arg.lower() + + if arg not in values: + raise BadArgument(f"Only the following values are allowed:\n```{', '.join(values)}```") + else: + return arg + + return converter + + class ValidPythonIdentifier(Converter): """ A converter that checks whether the given string is a valid Python identifier. @@ -70,7 +89,7 @@ class InfractionSearchQuery(Converter): """A converter that checks if the argument is a Discord user, and if not, falls back to a string.""" @staticmethod - async def convert(ctx: Context, arg: str) -> Union[discord.Member, str]: + async def convert(ctx: Context, arg: str) -> t.Union[discord.Member, str]: """Check if the argument is a Discord user, and if not, falls back to a string.""" try: maybe_snowflake = arg.strip("<@!>") diff --git a/bot/interpreter.py b/bot/interpreter.py index 76a3fc293..8b7268746 100644 --- a/bot/interpreter.py +++ b/bot/interpreter.py @@ -2,7 +2,9 @@ from code import InteractiveInterpreter from io import StringIO from typing import Any -from discord.ext.commands import Bot, Context +from discord.ext.commands import Context + +from bot.bot import Bot CODE_TEMPLATE = """ async def _func(): diff --git a/bot/rules/attachments.py b/bot/rules/attachments.py index c550aed76..00bb2a949 100644 --- a/bot/rules/attachments.py +++ b/bot/rules/attachments.py @@ -7,14 +7,14 @@ async def apply( last_message: Message, recent_messages: List[Message], config: Dict[str, int] ) -> Optional[Tuple[str, Iterable[Member], Iterable[Message]]]: """Detects total attachments exceeding the limit sent by a single user.""" - relevant_messages = [last_message] + [ + relevant_messages = tuple( msg for msg in recent_messages if ( msg.author == last_message.author and len(msg.attachments) > 0 ) - ] + ) total_recent_attachments = sum(len(msg.attachments) for msg in relevant_messages) if total_recent_attachments > config['max']: diff --git a/bot/utils/time.py b/bot/utils/time.py index a024674ac..7416f36e0 100644 --- a/bot/utils/time.py +++ b/bot/utils/time.py @@ -113,7 +113,11 @@ def format_infraction(timestamp: str) -> str: return dateutil.parser.isoparse(timestamp).strftime(INFRACTION_FORMAT) -def format_infraction_with_duration(expiry: str, date_from: datetime.datetime = None, max_units: int = 2) -> str: +def format_infraction_with_duration( + expiry: Optional[str], + date_from: Optional[datetime.datetime] = None, + max_units: int = 2 +) -> Optional[str]: """ Format an infraction timestamp to a more readable ISO 8601 format WITH the duration. @@ -134,3 +138,28 @@ def format_infraction_with_duration(expiry: str, date_from: datetime.datetime = duration_formatted = f" ({duration})" if duration else '' return f"{expiry_formatted}{duration_formatted}" + + +def until_expiration( + expiry: Optional[str], + now: Optional[datetime.datetime] = None, + max_units: int = 2 +) -> Optional[str]: + """ + Get the remaining time until infraction's expiration, in a human-readable version of the relativedelta. + + Returns a human-readable version of the remaining duration between datetime.utcnow() and an expiry. + Unlike `humanize_delta`, this function will force the `precision` to be `seconds` by not passing it. + `max_units` specifies the maximum number of units of time to include (e.g. 1 may include days but not hours). + By default, max_units is 2. + """ + if not expiry: + return None + + now = now or datetime.datetime.utcnow() + since = dateutil.parser.isoparse(expiry).replace(tzinfo=None, microsecond=0) + + if since < now: + return None + + return humanize_delta(relativedelta(since, now), max_units=max_units) diff --git a/config-default.yml b/config-default.yml index ae09db66a..f842cf606 100644 --- a/config-default.yml +++ b/config-default.yml @@ -27,6 +27,8 @@ style: status_dnd: "<:status_dnd:470326272082313216>" status_offline: "<:status_offline:470326266537705472>" + failmail: "<:failmail:633660039931887616>" + bullet: "\u2022" pencil: "\u270F" new: "\U0001F195" @@ -39,6 +41,12 @@ style: ducky_ninja: &DUCKY_NINJA 637923502535606293 ducky_devil: &DUCKY_DEVIL 637925314982576139 ducky_tube: &DUCKY_TUBE 637881368008851456 + ducky_hunt: &DUCKY_HUNT 639355090909528084 + ducky_wizard: &DUCKY_WIZARD 639355996954689536 + ducky_party: &DUCKY_PARTY 639468753440210977 + ducky_angel: &DUCKY_ANGEL 640121935610511361 + ducky_maul: &DUCKY_MAUL 640137724958867467 + ducky_santa: &DUCKY_SANTA 655360331002019870 upvotes: "<:upvotes:638729835245731840>" comments: "<:comments:638729835073765387>" @@ -153,7 +161,7 @@ guild: contributor: 295488872404484098 core_developer: 587606783669829632 helpers: 267630620367257601 - jammer: 423054537079783434 + jammer: 591786436651646989 moderator: &MOD_ROLE 267629731250176001 muted: &MUTED_ROLE 277914926603829249 owner: &OWNER_ROLE 267627879762755584 @@ -200,6 +208,12 @@ filter: - 544525886180032552 # kennethreitz.org - 590806733924859943 # Discord Hack Week - 423249981340778496 # Kivy + - 197038439483310086 # Discord Testers + - 286633898581164032 # Ren'Py + - 349505959032389632 # PyGame + - 438622377094414346 # Pyglet + - 524691714909274162 # Panda3D + - 336642139381301249 # discord.py domain_blacklist: - pornhub.com @@ -367,11 +381,21 @@ anti_malware: - '.png' - '.tiff' - '.wmv' + - '.svg' + - '.psd' # Photoshop + - '.ai' # Illustrator + - '.aep' # After Effects + - '.xcf' # GIMP + - '.mp3' + - '.wav' + - '.ogg' reddit: subreddits: - 'r/Python' + client_id: !ENV "REDDIT_CLIENT_ID" + secret: !ENV "REDDIT_SECRET" wolfram: @@ -403,7 +427,7 @@ redirect_output: duck_pond: threshold: 5 - custom_emojis: [*DUCKY_YELLOW, *DUCKY_BLURPLE, *DUCKY_CAMO, *DUCKY_DEVIL, *DUCKY_NINJA, *DUCKY_REGAL, *DUCKY_TUBE] + custom_emojis: [*DUCKY_YELLOW, *DUCKY_BLURPLE, *DUCKY_CAMO, *DUCKY_DEVIL, *DUCKY_NINJA, *DUCKY_REGAL, *DUCKY_TUBE, *DUCKY_HUNT, *DUCKY_WIZARD, *DUCKY_PARTY, *DUCKY_ANGEL, *DUCKY_MAUL, *DUCKY_SANTA] config: required_keys: ['bot.token'] diff --git a/docker-compose.yml b/docker-compose.yml index f79fdba58..7281c7953 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,3 +42,5 @@ services: environment: BOT_TOKEN: ${BOT_TOKEN} BOT_API_KEY: badbot13m0n8f570f942013fc818f234916ca531 + REDDIT_CLIENT_ID: ${REDDIT_CLIENT_ID} + REDDIT_SECRET: ${REDDIT_SECRET} diff --git a/tests/bot/cogs/test_duck_pond.py b/tests/bot/cogs/test_duck_pond.py index b801e86f1..d07b2bce1 100644 --- a/tests/bot/cogs/test_duck_pond.py +++ b/tests/bot/cogs/test_duck_pond.py @@ -578,15 +578,7 @@ class DuckPondSetupTests(unittest.TestCase): """Tests setup of the `DuckPond` cog.""" def test_setup(self): - """Setup of the cog should log a message at `INFO` level.""" + """Setup of the extension should call add_cog.""" bot = helpers.MockBot() - log = logging.getLogger('bot.cogs.duck_pond') - - with self.assertLogs(logger=log, level=logging.INFO) as log_watcher: - duck_pond.setup(bot) - - self.assertEqual(len(log_watcher.records), 1) - record = log_watcher.records[0] - self.assertEqual(record.levelno, logging.INFO) - + duck_pond.setup(bot) bot.add_cog.assert_called_once() diff --git a/tests/bot/cogs/test_security.py b/tests/bot/cogs/test_security.py index efa7a50b1..9d1a62f7e 100644 --- a/tests/bot/cogs/test_security.py +++ b/tests/bot/cogs/test_security.py @@ -1,4 +1,3 @@ -import logging import unittest from unittest.mock import MagicMock @@ -49,11 +48,7 @@ class SecurityCogLoadTests(unittest.TestCase): """Tests loading the `Security` cog.""" def test_security_cog_load(self): - """Cog loading logs a message at `INFO` level.""" + """Setup of the extension should call add_cog.""" bot = MagicMock() - with self.assertLogs(logger='bot.cogs.security', level=logging.INFO) as cm: - security.setup(bot) - bot.add_cog.assert_called_once() - - [line] = cm.output - self.assertIn("Cog loaded: Security", line) + security.setup(bot) + bot.add_cog.assert_called_once() diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 3276cf5a5..a54b839d7 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -125,11 +125,7 @@ class TokenRemoverSetupTests(unittest.TestCase): """Tests setup of the `TokenRemover` cog.""" def test_setup(self): - """Setup of the cog should log a message at `INFO` level.""" + """Setup of the extension should call add_cog.""" bot = MockBot() - with self.assertLogs(logger='bot.cogs.token_remover', level=logging.INFO) as cm: - setup_cog(bot) - - [line] = cm.output + setup_cog(bot) bot.add_cog.assert_called_once() - self.assertIn("Cog loaded: TokenRemover", line) diff --git a/tests/bot/rules/test_attachments.py b/tests/bot/rules/test_attachments.py index 4bb0acf7c..d7187f315 100644 --- a/tests/bot/rules/test_attachments.py +++ b/tests/bot/rules/test_attachments.py @@ -1,52 +1,98 @@ -import asyncio import unittest -from dataclasses import dataclass -from typing import Any, List +from typing import List, NamedTuple, Tuple from bot.rules import attachments +from tests.helpers import MockMessage, async_test -# Using `MagicMock` sadly doesn't work for this usecase -# since it's __eq__ compares the MagicMock's ID. We just -# want to compare the actual attributes we set. -@dataclass -class FakeMessage: - author: str - attachments: List[Any] +class Case(NamedTuple): + recent_messages: List[MockMessage] + culprit: Tuple[str] + total_attachments: int -def msg(total_attachments: int) -> FakeMessage: - return FakeMessage(author='lemon', attachments=list(range(total_attachments))) +def msg(author: str, total_attachments: int) -> MockMessage: + """Builds a message with `total_attachments` attachments.""" + return MockMessage(author=author, attachments=list(range(total_attachments))) class AttachmentRuleTests(unittest.TestCase): - """Tests applying the `attachment` antispam rule.""" + """Tests applying the `attachments` antispam rule.""" - def test_allows_messages_without_too_many_attachments(self): + def setUp(self): + self.config = {"max": 5} + + @async_test + async def test_allows_messages_without_too_many_attachments(self): """Messages without too many attachments are allowed as-is.""" cases = ( - (msg(0), msg(0), msg(0)), - (msg(2), msg(2)), - (msg(0),), + [msg("bob", 0), msg("bob", 0), msg("bob", 0)], + [msg("bob", 2), msg("bob", 2)], + [msg("bob", 2), msg("alice", 2), msg("bob", 2)], ) - for last_message, *recent_messages in cases: - with self.subTest(last_message=last_message, recent_messages=recent_messages): - coro = attachments.apply(last_message, recent_messages, {'max': 5}) - self.assertIsNone(asyncio.run(coro)) + for recent_messages in cases: + last_message = recent_messages[0] + + with self.subTest( + last_message=last_message, + recent_messages=recent_messages, + config=self.config + ): + self.assertIsNone( + await attachments.apply(last_message, recent_messages, self.config) + ) - def test_disallows_messages_with_too_many_attachments(self): + @async_test + async def test_disallows_messages_with_too_many_attachments(self): """Messages with too many attachments trigger the rule.""" cases = ( - ((msg(4), msg(0), msg(6)), [msg(4), msg(6)], 10), - ((msg(6),), [msg(6)], 6), - ((msg(1),) * 6, [msg(1)] * 6, 6), + Case( + [msg("bob", 4), msg("bob", 0), msg("bob", 6)], + ("bob",), + 10 + ), + Case( + [msg("bob", 4), msg("alice", 6), msg("bob", 2)], + ("bob",), + 6 + ), + Case( + [msg("alice", 6)], + ("alice",), + 6 + ), + ( + [msg("alice", 1) for _ in range(6)], + ("alice",), + 6 + ), ) - for messages, relevant_messages, total in cases: - with self.subTest(messages=messages, relevant_messages=relevant_messages, total=total): - last_message, *recent_messages = messages - coro = attachments.apply(last_message, recent_messages, {'max': 5}) - self.assertEqual( - asyncio.run(coro), - (f"sent {total} attachments in 5s", ('lemon',), relevant_messages) + + for recent_messages, culprit, total_attachments in cases: + last_message = recent_messages[0] + relevant_messages = tuple( + msg + for msg in recent_messages + if ( + msg.author == last_message.author + and len(msg.attachments) > 0 + ) + ) + + with self.subTest( + last_message=last_message, + recent_messages=recent_messages, + relevant_messages=relevant_messages, + total_attachments=total_attachments, + config=self.config + ): + desired_output = ( + f"sent {total_attachments} attachments in {self.config['max']}s", + culprit, + relevant_messages + ) + self.assertTupleEqual( + await attachments.apply(last_message, recent_messages, self.config), + desired_output ) diff --git a/tests/bot/rules/test_links.py b/tests/bot/rules/test_links.py index be832843b..02a5d5501 100644 --- a/tests/bot/rules/test_links.py +++ b/tests/bot/rules/test_links.py @@ -2,25 +2,19 @@ import unittest from typing import List, NamedTuple, Tuple from bot.rules import links -from tests.helpers import async_test - - -class FakeMessage(NamedTuple): - author: str - content: str +from tests.helpers import MockMessage, async_test class Case(NamedTuple): - recent_messages: List[FakeMessage] - relevant_messages: Tuple[FakeMessage] + recent_messages: List[MockMessage] culprit: Tuple[str] total_links: int -def msg(author: str, total_links: int) -> FakeMessage: - """Makes a message with *total_links* links.""" +def msg(author: str, total_links: int) -> MockMessage: + """Makes a message with `total_links` links.""" content = " ".join(["https://pydis.com"] * total_links) - return FakeMessage(author=author, content=content) + return MockMessage(author=author, content=content) class LinksTests(unittest.TestCase): @@ -61,26 +55,28 @@ class LinksTests(unittest.TestCase): cases = ( Case( [msg("bob", 1), msg("bob", 2)], - (msg("bob", 1), msg("bob", 2)), ("bob",), 3 ), Case( [msg("alice", 1), msg("alice", 1), msg("alice", 1)], - (msg("alice", 1), msg("alice", 1), msg("alice", 1)), ("alice",), 3 ), Case( [msg("alice", 2), msg("bob", 3), msg("alice", 1)], - (msg("alice", 2), msg("alice", 1)), ("alice",), 3 ) ) - for recent_messages, relevant_messages, culprit, total_links in cases: + for recent_messages, culprit, total_links in cases: last_message = recent_messages[0] + relevant_messages = tuple( + msg + for msg in recent_messages + if msg.author == last_message.author + ) with self.subTest( last_message=last_message, diff --git a/tests/bot/rules/test_mentions.py b/tests/bot/rules/test_mentions.py new file mode 100644 index 000000000..ad49ead32 --- /dev/null +++ b/tests/bot/rules/test_mentions.py @@ -0,0 +1,95 @@ +import unittest +from typing import List, NamedTuple, Tuple + +from bot.rules import mentions +from tests.helpers import MockMessage, async_test + + +class Case(NamedTuple): + recent_messages: List[MockMessage] + culprit: Tuple[str] + total_mentions: int + + +def msg(author: str, total_mentions: int) -> MockMessage: + """Makes a message with `total_mentions` mentions.""" + return MockMessage(author=author, mentions=list(range(total_mentions))) + + +class TestMentions(unittest.TestCase): + """Tests applying the `mentions` antispam rule.""" + + def setUp(self): + self.config = { + "max": 2, + "interval": 10 + } + + @async_test + async def test_mentions_within_limit(self): + """Messages with an allowed amount of mentions.""" + cases = ( + [msg("bob", 0)], + [msg("bob", 2)], + [msg("bob", 1), msg("bob", 1)], + [msg("bob", 1), msg("alice", 2)] + ) + + for recent_messages in cases: + last_message = recent_messages[0] + + with self.subTest( + last_message=last_message, + recent_messages=recent_messages, + config=self.config + ): + self.assertIsNone( + await mentions.apply(last_message, recent_messages, self.config) + ) + + @async_test + async def test_mentions_exceeding_limit(self): + """Messages with a higher than allowed amount of mentions.""" + cases = ( + Case( + [msg("bob", 3)], + ("bob",), + 3 + ), + Case( + [msg("alice", 2), msg("alice", 0), msg("alice", 1)], + ("alice",), + 3 + ), + Case( + [msg("bob", 2), msg("alice", 3), msg("bob", 2)], + ("bob",), + 4 + ) + ) + + for recent_messages, culprit, total_mentions in cases: + last_message = recent_messages[0] + relevant_messages = tuple( + msg + for msg in recent_messages + if msg.author == last_message.author + ) + + with self.subTest( + last_message=last_message, + recent_messages=recent_messages, + relevant_messages=relevant_messages, + culprit=culprit, + total_mentions=total_mentions, + cofig=self.config + ): + desired_output = ( + f"sent {total_mentions} mentions in {self.config['interval']}s", + culprit, + relevant_messages + ) + self.assertTupleEqual( + await mentions.apply(last_message, recent_messages, self.config), + desired_output + ) diff --git a/tests/bot/utils/test_time.py b/tests/bot/utils/test_time.py new file mode 100644 index 000000000..69f35f2f5 --- /dev/null +++ b/tests/bot/utils/test_time.py @@ -0,0 +1,162 @@ +import asyncio +import unittest +from datetime import datetime, timezone +from unittest.mock import patch + +from dateutil.relativedelta import relativedelta + +from bot.utils import time +from tests.helpers import AsyncMock + + +class TimeTests(unittest.TestCase): + """Test helper functions in bot.utils.time.""" + + def test_humanize_delta_handle_unknown_units(self): + """humanize_delta should be able to handle unknown units, and will not abort.""" + # Does not abort for unknown units, as the unit name is checked + # against the attribute of the relativedelta instance. + self.assertEqual(time.humanize_delta(relativedelta(days=2, hours=2), 'elephants', 2), '2 days and 2 hours') + + def test_humanize_delta_handle_high_units(self): + """humanize_delta should be able to handle very high units.""" + # Very high maximum units, but it only ever iterates over + # each value the relativedelta might have. + self.assertEqual(time.humanize_delta(relativedelta(days=2, hours=2), 'hours', 20), '2 days and 2 hours') + + def test_humanize_delta_should_normal_usage(self): + """Testing humanize delta.""" + test_cases = ( + (relativedelta(days=2), 'seconds', 1, '2 days'), + (relativedelta(days=2, hours=2), 'seconds', 2, '2 days and 2 hours'), + (relativedelta(days=2, hours=2), 'seconds', 1, '2 days'), + (relativedelta(days=2, hours=2), 'days', 2, '2 days'), + ) + + for delta, precision, max_units, expected in test_cases: + with self.subTest(delta=delta, precision=precision, max_units=max_units, expected=expected): + self.assertEqual(time.humanize_delta(delta, precision, max_units), expected) + + def test_humanize_delta_raises_for_invalid_max_units(self): + """humanize_delta should raises ValueError('max_units must be positive') for invalid max_units.""" + test_cases = (-1, 0) + + for max_units in test_cases: + with self.subTest(max_units=max_units), self.assertRaises(ValueError) as error: + time.humanize_delta(relativedelta(days=2, hours=2), 'hours', max_units) + self.assertEqual(str(error), 'max_units must be positive') + + def test_parse_rfc1123(self): + """Testing parse_rfc1123.""" + self.assertEqual( + time.parse_rfc1123('Sun, 15 Sep 2019 12:00:00 GMT'), + datetime(2019, 9, 15, 12, 0, 0, tzinfo=timezone.utc) + ) + + def test_format_infraction(self): + """Testing format_infraction.""" + self.assertEqual(time.format_infraction('2019-12-12T00:01:00Z'), '2019-12-12 00:01') + + @patch('asyncio.sleep', new_callable=AsyncMock) + def test_wait_until(self, mock): + """Testing wait_until.""" + start = datetime(2019, 1, 1, 0, 0) + then = datetime(2019, 1, 1, 0, 10) + + # No return value + self.assertIs(asyncio.run(time.wait_until(then, start)), None) + + mock.assert_called_once_with(10 * 60) + + def test_format_infraction_with_duration_none_expiry(self): + """format_infraction_with_duration should work for None expiry.""" + test_cases = ( + (None, None, None, None), + + # To make sure that date_from and max_units are not touched + (None, 'Why hello there!', None, None), + (None, None, float('inf'), None), + (None, 'Why hello there!', float('inf'), None), + ) + + for expiry, date_from, max_units, expected in test_cases: + with self.subTest(expiry=expiry, date_from=date_from, max_units=max_units, expected=expected): + self.assertEqual(time.format_infraction_with_duration(expiry, date_from, max_units), expected) + + def test_format_infraction_with_duration_custom_units(self): + """format_infraction_with_duration should work for custom max_units.""" + test_cases = ( + ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 5, 5), 6, + '2019-12-12 00:01 (11 hours, 55 minutes and 55 seconds)'), + ('2019-11-23T20:09:00Z', datetime(2019, 4, 25, 20, 15), 20, + '2019-11-23 20:09 (6 months, 28 days, 23 hours and 54 minutes)') + ) + + for expiry, date_from, max_units, expected in test_cases: + with self.subTest(expiry=expiry, date_from=date_from, max_units=max_units, expected=expected): + self.assertEqual(time.format_infraction_with_duration(expiry, date_from, max_units), expected) + + def test_format_infraction_with_duration_normal_usage(self): + """format_infraction_with_duration should work for normal usage, across various durations.""" + test_cases = ( + ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 0, 5), 2, '2019-12-12 00:01 (12 hours and 55 seconds)'), + ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 0, 5), 1, '2019-12-12 00:01 (12 hours)'), + ('2019-12-12T00:00:00Z', datetime(2019, 12, 11, 23, 59), 2, '2019-12-12 00:00 (1 minute)'), + ('2019-11-23T20:09:00Z', datetime(2019, 11, 15, 20, 15), 2, '2019-11-23 20:09 (7 days and 23 hours)'), + ('2019-11-23T20:09:00Z', datetime(2019, 4, 25, 20, 15), 2, '2019-11-23 20:09 (6 months and 28 days)'), + ('2019-11-23T20:58:00Z', datetime(2019, 11, 23, 20, 53), 2, '2019-11-23 20:58 (5 minutes)'), + ('2019-11-24T00:00:00Z', datetime(2019, 11, 23, 23, 59, 0), 2, '2019-11-24 00:00 (1 minute)'), + ('2019-11-23T23:59:00Z', datetime(2017, 7, 21, 23, 0), 2, '2019-11-23 23:59 (2 years and 4 months)'), + ('2019-11-23T23:59:00Z', datetime(2019, 11, 23, 23, 49, 5), 2, + '2019-11-23 23:59 (9 minutes and 55 seconds)'), + (None, datetime(2019, 11, 23, 23, 49, 5), 2, None), + ) + + for expiry, date_from, max_units, expected in test_cases: + with self.subTest(expiry=expiry, date_from=date_from, max_units=max_units, expected=expected): + self.assertEqual(time.format_infraction_with_duration(expiry, date_from, max_units), expected) + + def test_until_expiration_with_duration_none_expiry(self): + """until_expiration should work for None expiry.""" + test_cases = ( + (None, None, None, None), + + # To make sure that now and max_units are not touched + (None, 'Why hello there!', None, None), + (None, None, float('inf'), None), + (None, 'Why hello there!', float('inf'), None), + ) + + for expiry, now, max_units, expected in test_cases: + with self.subTest(expiry=expiry, now=now, max_units=max_units, expected=expected): + self.assertEqual(time.until_expiration(expiry, now, max_units), expected) + + def test_until_expiration_with_duration_custom_units(self): + """until_expiration should work for custom max_units.""" + test_cases = ( + ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 5, 5), 6, '11 hours, 55 minutes and 55 seconds'), + ('2019-11-23T20:09:00Z', datetime(2019, 4, 25, 20, 15), 20, '6 months, 28 days, 23 hours and 54 minutes') + ) + + for expiry, now, max_units, expected in test_cases: + with self.subTest(expiry=expiry, now=now, max_units=max_units, expected=expected): + self.assertEqual(time.until_expiration(expiry, now, max_units), expected) + + def test_until_expiration_normal_usage(self): + """until_expiration should work for normal usage, across various durations.""" + test_cases = ( + ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 0, 5), 2, '12 hours and 55 seconds'), + ('2019-12-12T00:01:00Z', datetime(2019, 12, 11, 12, 0, 5), 1, '12 hours'), + ('2019-12-12T00:00:00Z', datetime(2019, 12, 11, 23, 59), 2, '1 minute'), + ('2019-11-23T20:09:00Z', datetime(2019, 11, 15, 20, 15), 2, '7 days and 23 hours'), + ('2019-11-23T20:09:00Z', datetime(2019, 4, 25, 20, 15), 2, '6 months and 28 days'), + ('2019-11-23T20:58:00Z', datetime(2019, 11, 23, 20, 53), 2, '5 minutes'), + ('2019-11-24T00:00:00Z', datetime(2019, 11, 23, 23, 59, 0), 2, '1 minute'), + ('2019-11-23T23:59:00Z', datetime(2017, 7, 21, 23, 0), 2, '2 years and 4 months'), + ('2019-11-23T23:59:00Z', datetime(2019, 11, 23, 23, 49, 5), 2, '9 minutes and 55 seconds'), + (None, datetime(2019, 11, 23, 23, 49, 5), 2, None), + ) + + for expiry, now, max_units, expected in test_cases: + with self.subTest(expiry=expiry, now=now, max_units=max_units, expected=expected): + self.assertEqual(time.until_expiration(expiry, now, max_units), expected) diff --git a/tests/helpers.py b/tests/helpers.py index b2daae92d..5df796c23 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -10,7 +10,9 @@ import unittest.mock from typing import Any, Iterable, Optional import discord -from discord.ext.commands import Bot, Context +from discord.ext.commands import Context + +from bot.bot import Bot for logger in logging.Logger.manager.loggerDict.values(): |