diff options
49 files changed, 2509 insertions, 2493 deletions
diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..864b4e557 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM python:3.7-alpine3.7 + +RUN apk add --no-cache \ + build-base \ + freetype-dev \ + git \ + jpeg-dev \ + libffi-dev \ + libxml2 \ + libxml2-dev \ + libxslt-dev \ + tini \ + zlib \ + zlib-dev + +ENV \ + LIBRARY_PATH=/lib:/usr/lib \ + PIPENV_HIDE_EMOJIS=1 \ + PIPENV_HIDE_EMOJIS=1 \ + PIPENV_IGNORE_VIRTUALENVS=1 \ + PIPENV_IGNORE_VIRTUALENVS=1 \ + PIPENV_NOSPIN=1 \ + PIPENV_NOSPIN=1 \ + PIPENV_VENV_IN_PROJECT=1 \ + PIPENV_VENV_IN_PROJECT=1 + +RUN pip install -U pipenv + +WORKDIR /bot +COPY . . + +RUN pipenv install --deploy --system + +ENTRYPOINT ["/sbin/tini", "--"] +CMD ["pipenv", "run", "start"] @@ -17,6 +17,7 @@ aio-pika = "*" python-dateutil = "*" deepdiff = "*" requests = "*" +dateparser = "*" urllib3 = ">=1.24.2,<1.25" [dev-packages] @@ -29,9 +30,10 @@ urllib3 = ">=1.24.2,<1.25" safety = "*" dodgy = "*" pre-commit = "*" +pytest = "*" [requires] -python_version = "3.6" +python_version = "3.7" [scripts] start = "python -m bot" diff --git a/Pipfile.lock b/Pipfile.lock index 735d7cd96..e2585756f 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "ab3b63b74dbf35fb960913a91e10282121a2776e935d98f0b4c3d780715f7a6b" + "sha256": "ad3b645e777f7b21a2bfb472e182361f904ae5f1f41df59300c4c68c89bd2fd1" }, "pipfile-spec": 6, "requires": { - "python_version": "3.6" + "python_version": "3.7" }, "sources": [ { @@ -18,11 +18,11 @@ "default": { "aio-pika": { "hashes": [ - "sha256:300474d8b0e9ccde17b2d1e71c3b4f7ba86559cc0842b9355b9eccb12be4a02a", - "sha256:3bc547600344beba8f36edfd1b1ec1c8b30f803ea7c11eaf249683099d07c98b" + "sha256:47b12535897117b9876db2e2c0506b6c89bd4dbd90617cd8b20163d4196137ed", + "sha256:821ee9f652ba472919ebffdc37c661fc740c24309a2291b37ac8a160b4003ce6" ], "index": "pypi", - "version": "==5.5.2" + "version": "==5.5.3" }, "aiodns": { "hashes": [ @@ -62,10 +62,10 @@ }, "aiormq": { "hashes": [ - "sha256:79b41e51481fb7617279414e4428a644a944beb4dea8ea0febd67a8902976250", - "sha256:f134cc91ac111b0135c97539272579b1d15b69f25c75a935f6ee39e5194df231" + "sha256:038bd43d68f8e77bf79c7cc362da9df5ca6497c23c3bf20ee43ce1622448ef8a", + "sha256:f36be480de4009ddb621a8795c52f0c146813799f56d79e126dfa60e13e41dd9" ], - "version": "==2.5.2" + "version": "==2.5.5" }, "alabaster": { "hashes": [ @@ -90,10 +90,10 @@ }, "babel": { "hashes": [ - "sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669", - "sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23" + "sha256:af92e6106cb7c55286b25b38ad7695f8b4efb36a90ba483d7f7a6628c46158ab", + "sha256:e86135ae101e31e2c8ec20a4e0c5220f4eed12487d5cf3f78be7e98d3a57fc28" ], - "version": "==2.6.0" + "version": "==2.7.0" }, "beautifulsoup4": { "hashes": [ @@ -105,10 +105,10 @@ }, "certifi": { "hashes": [ - "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", - "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" + "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", + "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" ], - "version": "==2019.3.9" + "version": "==2019.6.16" }, "cffi": { "hashes": [ @@ -150,6 +150,14 @@ ], "version": "==3.0.4" }, + "dateparser": { + "hashes": [ + "sha256:42d51be54e74a8e80a4d76d1fa6e4edd997098fce24ad2d94a2eab5ef247193e", + "sha256:78124c458c461ea7198faa3c038f6381f37588b84bb42740e91a4cbd260b1d09" + ], + "index": "pypi", + "version": "==0.7.1" + }, "deepdiff": { "hashes": [ "sha256:55e461f56dcae3dc540746b84434562fb7201e5c27ecf28800e4cfdd17f61e56", @@ -205,10 +213,10 @@ }, "jsonpickle": { "hashes": [ - "sha256:0231d6f7ebc4723169310141352d9c9b7bbbd6f3be110cf634575d2bf2af91f0", - "sha256:625098cc8e5854b8c23b587aec33bc8e33e0e597636bfaca76152249c78fe5c1" + "sha256:d0c5a4e6cb4e58f6d5406bdded44365c2bcf9c836c4f52910cc9ba7245a59dc2", + "sha256:d3e922d781b1d0096df2dad89a2e1f47177d7969b596aea806a9d91b4626b29b" ], - "version": "==1.1" + "version": "==1.2" }, "logmatic-python": { "hashes": [ @@ -219,35 +227,33 @@ }, "lxml": { "hashes": [ - "sha256:03984196d00670b2ab14ae0ea83d5cc0cfa4f5a42558afa9ab5fa745995328f5", - "sha256:0815b0c9f897468de6a386dc15917a0becf48cc92425613aa8bbfc7f0f82951f", - "sha256:175f3825f075cf02d15099eb52658457cf0ff103dcf11512b5d2583e1d40f58b", - "sha256:30e14c62d88d1e01a26936ecd1c6e784d4afc9aa002bba4321c5897937112616", - "sha256:3210da6f36cf4b835ff1be853962b22cc354d506f493b67a4303c88bbb40d57b", - "sha256:40f60819fbd5bad6e191ba1329bfafa09ab7f3f174b3d034d413ef5266963294", - "sha256:43b26a865a61549919f8a42e094dfdb62847113cf776d84bd6b60e4e3fc20ea3", - "sha256:4a03dd682f8e35a10234904e0b9508d705ff98cf962c5851ed052e9340df3d90", - "sha256:62f382cddf3d2e52cf266e161aa522d54fd624b8cc567bc18f573d9d50d40e8e", - "sha256:7b98f0325be8450da70aa4a796c4f06852949fe031878b4aa1d6c417a412f314", - "sha256:846a0739e595871041385d86d12af4b6999f921359b38affb99cdd6b54219a8f", - "sha256:a3080470559938a09a5d0ec558c005282e99ac77bf8211fb7b9a5c66390acd8d", - "sha256:ad841b78a476623955da270ab8d207c3c694aa5eba71f4792f65926dc46c6ee8", - "sha256:afdd75d9735e44c639ffd6258ce04a2de3b208f148072c02478162d0944d9da3", - "sha256:b4fbf9b552faff54742bcd0791ab1da5863363fb19047e68f6592be1ac2dab33", - "sha256:b90c4e32d6ec089d3fa3518436bdf5ce4d902a0787dbd9bb09f37afe8b994317", - "sha256:b91cfe4438c741aeff662d413fd2808ac901cc6229c838236840d11de4586d63", - "sha256:bdb0593a42070b0a5f138b79b872289ee73c8e25b3f0bea6564e795b55b6bcdd", - "sha256:c4e4bca2bb68ce22320297dfa1a7bf070a5b20bcbaec4ee023f83d2f6e76496f", - "sha256:cec4ab14af9eae8501be3266ff50c3c2aecc017ba1e86c160209bb4f0423df6a", - "sha256:e83b4b2bf029f5104bc1227dbb7bf5ace6fd8fabaebffcd4f8106fafc69fc45f", - "sha256:e995b3734a46d41ae60b6097f7c51ba9958648c6d1e0935b7e0ee446ee4abe22", - "sha256:f679d93dec7f7210575c85379a31322df4c46496f184ef650d3aba1484b38a2d", - "sha256:fd213bb5166e46974f113c8228daaef1732abc47cb561ce9c4c8eaed4bd3b09b", - "sha256:fdcb57b906dbc1f80666e6290e794ab8fb959a2e17aa5aee1758a85d1da4533f", - "sha256:ff424b01d090ffe1947ec7432b07f536912e0300458f9a7f48ea217dd8362b86" + "sha256:06c7616601430aa140a69f97e3116308fffe0848f543b639a5ec2e8920ae72fd", + "sha256:177202792f9842374a8077735c69c41a4282183f7851443d2beb8ee310720819", + "sha256:19317ad721ceb9e39847d11131903931e2794e447d4751ebb0d9236f1b349ff2", + "sha256:36d206e62f3e5dbaafd4ec692b67157e271f5da7fd925fda8515da675eace50d", + "sha256:387115b066c797c85f9861a9613abf50046a15aac16759bc92d04f94acfad082", + "sha256:3ce1c49d4b4a7bc75fb12acb3a6247bb7a91fe420542e6d671ba9187d12a12c2", + "sha256:4d2a5a7d6b0dbb8c37dab66a8ce09a8761409c044017721c21718659fa3365a1", + "sha256:58d0a1b33364d1253a88d18df6c0b2676a1746d27c969dc9e32d143a3701dda5", + "sha256:62a651c618b846b88fdcae0533ec23f185bb322d6c1845733f3123e8980c1d1b", + "sha256:69ff21064e7debc9b1b1e2eee8c2d686d042d4257186d70b338206a80c5bc5ea", + "sha256:7060453eba9ba59d821625c6af6a266bd68277dce6577f754d1eb9116c094266", + "sha256:7d26b36a9c4bce53b9cfe42e67849ae3c5c23558bc08363e53ffd6d94f4ff4d2", + "sha256:83b427ad2bfa0b9705e02a83d8d607d2c2f01889eb138168e462a3a052c42368", + "sha256:923d03c84534078386cf50193057aae98fa94cace8ea7580b74754493fda73ad", + "sha256:b773715609649a1a180025213f67ffdeb5a4878c784293ada300ee95a1f3257b", + "sha256:baff149c174e9108d4a2fee192c496711be85534eab63adb122f93e70aa35431", + "sha256:bca9d118b1014b4c2d19319b10a3ebed508ff649396ce1855e1c96528d9b2fa9", + "sha256:ce580c28845581535dc6000fc7c35fdadf8bea7ccb57d6321b044508e9ba0685", + "sha256:d34923a569e70224d88e6682490e24c842907ba2c948c5fd26185413cbe0cd96", + "sha256:dd9f0e531a049d8b35ec5e6c68a37f1ba6ec3a591415e6804cbdf652793d15d7", + "sha256:ecb805cbfe9102f3fd3d2ef16dfe5ae9e2d7a7dfbba92f4ff1e16ac9784dbfb0", + "sha256:ede9aad2197a0202caff35d417b671f5f91a3631477441076082a17c94edd846", + "sha256:ef2d1fc370400e0aa755aab0b20cf4f1d0e934e7fd5244f3dd4869078e4942b9", + "sha256:f2fec194a49bfaef42a548ee657362af5c7a640da757f6f452a35da7dd9f923c" ], "index": "pypi", - "version": "==4.3.3" + "version": "==4.3.4" }, "markdownify": { "hashes": [ @@ -363,16 +369,17 @@ }, "pycparser": { "hashes": [ - "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3", + "sha256:b360ff0cd21cdecd07372020a2d7f3234e1acc8c31ab4b4d3a6fa6e5bc6259cd" ], "version": "==2.19" }, "pygments": { "hashes": [ - "sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a", - "sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d" + "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", + "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" ], - "version": "==2.3.1" + "version": "==2.4.2" }, "pynacl": { "hashes": [ @@ -440,28 +447,44 @@ }, "pyyaml": { "hashes": [ - "sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c", - "sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95", - "sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2", - "sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4", - "sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad", - "sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba", - "sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1", - "sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e", - "sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673", - "sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13", - "sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19" + "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3", + "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043", + "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7", + "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265", + "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391", + "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778", + "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225", + "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955", + "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e", + "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190", + "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd" ], "index": "pypi", - "version": "==5.1" + "version": "==5.1.1" + }, + "regex": { + "hashes": [ + "sha256:1c70ccb8bf4ded0cbe53092e9f56dcc9d6b0efcf6e80b6ef9b0ece8a557d6635", + "sha256:2948310c01535ccb29bb600dd033b07b91f36e471953889b7f3a1e66b39d0c19", + "sha256:2ab13db0411cb308aa590d33c909ea4efeced40188d8a4a7d3d5970657fe73bc", + "sha256:38e6486c7e14683cd1b17a4218760f0ea4c015633cf1b06f7c190fb882a51ba7", + "sha256:80dde4ff10b73b823da451687363cac93dd3549e059d2dc19b72a02d048ba5aa", + "sha256:84daedefaa56320765e9c4d43912226d324ef3cc929f4d75fa95f8c579a08211", + "sha256:b98e5876ca1e63b41c4aa38d7d5cc04a736415d4e240e9ae7ebc4f780083c7d5", + "sha256:ca4f47131af28ef168ff7c80d4b4cad019cb4cabb5fa26143f43aa3dbd60389c", + "sha256:cf7838110d3052d359da527372666429b9485ab739286aa1a11ed482f037a88c", + "sha256:dd4e8924915fa748e128864352875d3d0be5f4597ab1b1d475988b8e3da10dd7", + "sha256:f2c65530255e4010a5029eb11138f5ecd5aa70363f57a3444d83b3253b0891be" + ], + "version": "==2019.6.8" }, "requests": { "hashes": [ - "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", - "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" ], "index": "pypi", - "version": "==2.21.0" + "version": "==2.22.0" }, "six": { "hashes": [ @@ -486,11 +509,11 @@ }, "sphinx": { "hashes": [ - "sha256:423280646fb37944dd3c85c58fb92a20d745793a9f6c511f59da82fa97cd404b", - "sha256:de930f42600a4fef993587633984cc5027dedba2464bcf00ddace26b40f8d9ce" + "sha256:15143166e786c7faa76fa990d3b6b38ebffe081ef81cffd1d656b07f3b28a1fa", + "sha256:5fd62ba64235d77a81554d47ff6b17578171b6dbbc992221e9ebc684898fff59" ], "index": "pypi", - "version": "==2.0.1" + "version": "==2.1.1" }, "sphinxcontrib-applehelp": { "hashes": [ @@ -534,13 +557,19 @@ ], "version": "==1.1.3" }, + "tzlocal": { + "hashes": [ + "sha256:4ebeb848845ac898da6519b9b31879cf13b6626f7184c496037b818e238f2c4e" + ], + "version": "==1.5.1" + }, "urllib3": { "hashes": [ - "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", - "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" + "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4", + "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb" ], "index": "pypi", - "version": "==1.24.2" + "version": "==1.24.3" }, "websockets": { "hashes": [ @@ -588,10 +617,17 @@ "develop": { "aspy.yaml": { "hashes": [ - "sha256:ae249074803e8b957c83fdd82a99160d0d6d26dff9ba81ba608b42eebd7d8cd3", - "sha256:c7390d79f58eb9157406966201abf26da0d56c07e0ff0deadc39c8f4dbc13482" + "sha256:463372c043f70160a9ec950c3f1e4c3a82db5fca01d334b6bc89c7164d744bdc", + "sha256:e7c742382eff2caed61f87a39d13f99109088e5e93f04d76eb8d4b28aa143f45" + ], + "version": "==1.3.0" + }, + "atomicwrites": { + "hashes": [ + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" ], - "version": "==1.2.0" + "version": "==1.3.0" }, "attrs": { "hashes": [ @@ -602,17 +638,17 @@ }, "certifi": { "hashes": [ - "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", - "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" + "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", + "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" ], - "version": "==2019.3.9" + "version": "==2019.6.16" }, "cfgv": { "hashes": [ - "sha256:6e9f2feea5e84bc71e56abd703140d7a2c250fc5ba38b8702fd6a68ed4e3b2ef", - "sha256:e7f186d4a36c099a9e20b04ac3108bd8bb9b9257e692ce18c8c3764d5cb12172" + "sha256:32edbe09de6f4521224b87822103a8c16a614d31a894735f7a5b3bcf0eb3c37e", + "sha256:3bd31385cd2bebddbba8012200aaf15aa208539f1b33973759b4d02fc2148da5" ], - "version": "==1.6.0" + "version": "==2.0.0" }, "chardet": { "hashes": [ @@ -698,10 +734,10 @@ }, "identify": { "hashes": [ - "sha256:443f419ca6160773cbaf22dbb302b1e436a386f23129dbb5482b68a147c2eca9", - "sha256:bd7f15fe07112b713fb68fbdde3a34dd774d9062128f2c398104889f783f989d" + "sha256:0a11379b46d06529795442742a043dc2fa14cd8c995ae81d1febbc5f1c014c87", + "sha256:43a5d24ffdb07bc7e21faf68b08e9f526a1f41f0056073f480291539ef961dfd" ], - "version": "==1.4.2" + "version": "==1.4.5" }, "idna": { "hashes": [ @@ -712,10 +748,10 @@ }, "importlib-metadata": { "hashes": [ - "sha256:46fc60c34b6ed7547e2a723fc8de6dc2e3a1173f8423246b3ce497f064e9c3de", - "sha256:bc136180e961875af88b1ab85b4009f4f1278f8396a60526c0009f503a1a96ca" + "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7", + "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db" ], - "version": "==0.9" + "version": "==0.18" }, "mccabe": { "hashes": [ @@ -724,6 +760,14 @@ ], "version": "==0.6.1" }, + "more-itertools": { + "hashes": [ + "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7", + "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a" + ], + "markers": "python_version > '2.7'", + "version": "==7.0.0" + }, "nodeenv": { "hashes": [ "sha256:ad8259494cf1c9034539f6cced78a1da4840a4b157e23640bc4a0c0546b0cb7a" @@ -737,13 +781,27 @@ ], "version": "==19.0" }, + "pluggy": { + "hashes": [ + "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc", + "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c" + ], + "version": "==0.12.0" + }, "pre-commit": { "hashes": [ - "sha256:2576a2776098f3902ef9540a84696e8e06bf18a337ce43a6a889e7fa5d26c4c5", - "sha256:82f2f2d657d7f9280de9f927ae56886d60b9ef7f3714eae92d12713cd9cb9e11" + "sha256:92e406d556190503630fd801958379861c94884693a032ba66629d0351fdccd4", + "sha256:cccc39051bc2457b0c0f7152a411f8e05e3ba2fe1a5613e4ee0833c1c1985ce3" ], "index": "pypi", - "version": "==1.15.2" + "version": "==1.17.0" + }, + "py": { + "hashes": [ + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" + ], + "version": "==1.8.0" }, "pycodestyle": { "hashes": [ @@ -766,30 +824,38 @@ ], "version": "==2.4.0" }, + "pytest": { + "hashes": [ + "sha256:4a784f1d4f2ef198fe9b7aef793e9fa1a3b2f84e822d9b3a64a181293a572d45", + "sha256:926855726d8ae8371803f7b2e6ec0a69953d9c6311fa7c3b6c1b929ff92d27da" + ], + "index": "pypi", + "version": "==4.6.3" + }, "pyyaml": { "hashes": [ - "sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c", - "sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95", - "sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2", - "sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4", - "sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad", - "sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba", - "sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1", - "sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e", - "sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673", - "sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13", - "sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19" + "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3", + "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043", + "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7", + "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265", + "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391", + "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778", + "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225", + "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955", + "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e", + "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190", + "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd" ], "index": "pypi", - "version": "==5.1" + "version": "==5.1.1" }, "requests": { "hashes": [ - "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", - "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" ], "index": "pypi", - "version": "==2.21.0" + "version": "==2.22.0" }, "safety": { "hashes": [ @@ -815,25 +881,32 @@ }, "urllib3": { "hashes": [ - "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", - "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" + "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4", + "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb" ], "index": "pypi", - "version": "==1.24.2" + "version": "==1.24.3" }, "virtualenv": { "hashes": [ - "sha256:15ee248d13e4001a691d9583948ad3947bcb8a289775102e4c4aa98a8b7a6d73", - "sha256:bfc98bb9b42a3029ee41b96dc00a34c2f254cbf7716bec824477b2c82741a5c4" + "sha256:b7335cddd9260a3dd214b73a2521ffc09647bde3e9457fcca31dc3be3999d04a", + "sha256:d28ca64c0f3f125f59cabf13e0a150e1c68e5eea60983cc4395d88c584495783" ], - "version": "==16.5.0" + "version": "==16.6.1" + }, + "wcwidth": { + "hashes": [ + "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", + "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + ], + "version": "==0.1.7" }, "zipp": { "hashes": [ - "sha256:139391b239594fd8b91d856bc530fbd2df0892b17dd8d98a91f018715954185f", - "sha256:8047e4575ce8d700370a3301bbfc972896a5845eb62dd535da395b86be95dfad" + "sha256:8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d", + "sha256:ca943a7e809cc12257001ccfb99e3563da9af99d52f261725e96dfe0f9275bc3" ], - "version": "==0.4.0" + "version": "==0.5.1" } } } diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a14364881..19df35c11 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,7 +10,7 @@ jobs: displayName: 'Lint & Test' pool: - vmImage: 'Ubuntu 16.04' + vmImage: ubuntu-16.04 variables: PIPENV_CACHE_DIR: ".cache/pipenv" @@ -18,10 +18,9 @@ jobs: PIP_SRC: ".cache/src" steps: - - script: sudo apt-get update - displayName: 'Updating package list' - - - script: sudo apt-get install build-essential curl docker libffi-dev libfreetype6-dev libxml2 libxml2-dev libxslt1-dev zlib1g zlib1g-dev + - script: | + sudo apt-get update + sudo apt-get install build-essential curl docker libffi-dev libfreetype6-dev libxml2 libxml2-dev libxslt1-dev zlib1g zlib1g-dev displayName: 'Install base dependencies' - task: UsePythonVersion@0 @@ -39,6 +38,9 @@ jobs: - script: python -m flake8 displayName: 'Run linter' + - script: BOT_TOKEN=foobar python -m pytest tests + displayName: Run tests + - job: build displayName: 'Build Containers' dependsOn: 'test' @@ -54,7 +56,5 @@ jobs: - task: ShellScript@2 displayName: 'Build and deploy containers' - inputs: scriptPath: scripts/deploy-azure.sh - args: '$(AUTODEPLOY_TOKEN) $(AUTODEPLOY_WEBHOOK)' diff --git a/bot/__init__.py b/bot/__init__.py index a088138a0..8efa5e53c 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -55,7 +55,7 @@ else: logging.basicConfig( - format="%(asctime)s pd.beardfist.com Bot: | %(name)30s | %(levelname)8s | %(message)s", + format="%(asctime)s pd.beardfist.com Bot: | %(name)33s | %(levelname)8s | %(message)s", datefmt="%b %d %H:%M:%S", level=logging.TRACE if DEBUG_MODE else logging.INFO, handlers=logging_handlers diff --git a/bot/__main__.py b/bot/__main__.py index ead6d287a..b3f80ef55 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -1,3 +1,4 @@ +import asyncio import logging import socket @@ -5,8 +6,8 @@ from aiohttp import AsyncResolver, ClientSession, TCPConnector from discord import Game from discord.ext.commands import Bot, when_mentioned_or +from bot.api import APIClient from bot.constants import Bot as BotConfig, DEBUG_MODE -from bot.utils.service_discovery import wait_for_rmq log = logging.getLogger(__name__) @@ -27,18 +28,9 @@ bot.http_session = ClientSession( family=socket.AF_INET, ) ) - -log.info("Waiting for RabbitMQ...") - -has_rmq = wait_for_rmq() - -if has_rmq: - log.info("RabbitMQ found") -else: - log.warning("Timed out while waiting for RabbitMQ") +bot.api_client = APIClient(loop=asyncio.get_event_loop()) # Internal/debug -bot.load_extension("bot.cogs.events") bot.load_extension("bot.cogs.filtering") bot.load_extension("bot.cogs.logging") bot.load_extension("bot.cogs.modlog") @@ -46,12 +38,10 @@ bot.load_extension("bot.cogs.security") # Commands, etc bot.load_extension("bot.cogs.antispam") -bot.load_extension("bot.cogs.bigbrother") bot.load_extension("bot.cogs.bot") bot.load_extension("bot.cogs.clean") bot.load_extension("bot.cogs.cogs") bot.load_extension("bot.cogs.help") -bot.load_extension("bot.cogs.rules") # Only load this in production if not DEBUG_MODE: @@ -74,14 +64,13 @@ bot.load_extension("bot.cogs.reminders") bot.load_extension("bot.cogs.site") bot.load_extension("bot.cogs.snekbox") bot.load_extension("bot.cogs.superstarify") +bot.load_extension("bot.cogs.sync") bot.load_extension("bot.cogs.tags") bot.load_extension("bot.cogs.token_remover") bot.load_extension("bot.cogs.utils") +bot.load_extension("bot.cogs.watchchannels") bot.load_extension("bot.cogs.wolfram") -if has_rmq: - bot.load_extension("bot.cogs.rmq") - bot.run(BotConfig.token) bot.http_session.close() # Close the aiohttp session when the bot finishes running diff --git a/bot/api.py b/bot/api.py new file mode 100644 index 000000000..2e1a239ba --- /dev/null +++ b/bot/api.py @@ -0,0 +1,48 @@ +from urllib.parse import quote as quote_url + +import aiohttp + +from .constants import Keys, URLs + + +class APIClient: + def __init__(self, **kwargs): + auth_headers = { + 'Authorization': f"Token {Keys.site_api}" + } + + if 'headers' in kwargs: + kwargs['headers'].update(auth_headers) + else: + kwargs['headers'] = auth_headers + + self.session = aiohttp.ClientSession( + **kwargs, + raise_for_status=True + ) + + @staticmethod + def _url_for(endpoint: str): + return f"{URLs.site_schema}{URLs.site_api}/{quote_url(endpoint)}" + + async def get(self, endpoint: str, *args, **kwargs): + async with self.session.get(self._url_for(endpoint), *args, **kwargs) as resp: + return await resp.json() + + async def patch(self, endpoint: str, *args, **kwargs): + async with self.session.patch(self._url_for(endpoint), *args, **kwargs) as resp: + return await resp.json() + + async def post(self, endpoint: str, *args, **kwargs): + async with self.session.post(self._url_for(endpoint), *args, **kwargs) as resp: + return await resp.json() + + async def put(self, endpoint: str, *args, **kwargs): + async with self.session.put(self._url_for(endpoint), *args, **kwargs) as resp: + return await resp.json() + + async def delete(self, endpoint: str, *args, **kwargs): + async with self.session.delete(self._url_for(endpoint), *args, **kwargs) as resp: + if resp.status == 204: + return None + return await resp.json() diff --git a/bot/cogs/alias.py b/bot/cogs/alias.py index bf40fe409..85d101448 100644 --- a/bot/cogs/alias.py +++ b/bot/cogs/alias.py @@ -1,11 +1,13 @@ import inspect import logging +from typing import Union -from discord import Colour, Embed, User +from discord import Colour, Embed, Member, User from discord.ext.commands import ( Command, Context, clean_content, command, group ) +from bot.cogs.watchchannels.watchchannel import proxy_user from bot.converters import TagNameConverter from bot.pagination import LinePaginator @@ -70,9 +72,7 @@ class Alias: await self.invoke(ctx, "site resources") @command(name="watch", hidden=True) - async def bigbrother_watch_alias( - self, ctx: Context, user: User, *, reason: str - ): + async def bigbrother_watch_alias(self, ctx, user: Union[Member, User, proxy_user], *, reason: str): """ Alias for invoking <prefix>bigbrother watch [user] [reason]. """ @@ -80,7 +80,7 @@ class Alias: await self.invoke(ctx, "bigbrother watch", user, reason=reason) @command(name="unwatch", hidden=True) - async def bigbrother_unwatch_alias(self, ctx, user: User, *, reason: str): + async def bigbrother_unwatch_alias(self, ctx, user: Union[User, proxy_user], *, reason: str): """ Alias for invoking <prefix>bigbrother unwatch [user] [reason]. """ @@ -103,6 +103,14 @@ class Alias: await self.invoke(ctx, "site faq") + @command(name="rules", hidden=True) + async def site_rules_alias(self, ctx): + """ + Alias for invoking <prefix>site rules. + """ + + await self.invoke(ctx, "site rules") + @command(name="reload", hidden=True) async def cogs_reload_alias(self, ctx, *, cog_name: str): """ @@ -173,6 +181,22 @@ class Alias: await self.invoke(ctx, "docs get", symbol) + @command(name="nominate", hidden=True) + async def nomination_add_alias(self, ctx, user: Union[Member, User, proxy_user], *, reason: str): + """ + Alias for invoking <prefix>talentpool add [user] [reason]. + """ + + await self.invoke(ctx, "talentpool add", user, reason=reason) + + @command(name="unnominate", hidden=True) + async def nomination_end_alias(self, ctx, user: Union[User, proxy_user], *, reason: str): + """ + Alias for invoking <prefix>nomination end [user] [reason]. + """ + + await self.invoke(ctx, "nomination end", user, reason=reason) + def setup(bot): bot.add_cog(Alias(bot)) diff --git a/bot/cogs/antispam.py b/bot/cogs/antispam.py index 03551e806..0c6a02bf9 100644 --- a/bot/cogs/antispam.py +++ b/bot/cogs/antispam.py @@ -110,7 +110,7 @@ class AntiSpam: # For multiple messages or those with excessive newlines, use the logs API if len(messages) > 1 or rule_name == 'newlines': - url = await self.mod_log.upload_log(messages) + url = await self.mod_log.upload_log(messages, msg.guild.me.id) mod_alert_message += f"A complete log of the offending messages can be found [here]({url})" else: mod_alert_message += "Message:\n" diff --git a/bot/cogs/bigbrother.py b/bot/cogs/bigbrother.py deleted file mode 100644 index 97655812b..000000000 --- a/bot/cogs/bigbrother.py +++ /dev/null @@ -1,501 +0,0 @@ -import asyncio -import logging -import re -from collections import defaultdict, deque -from time import strptime, struct_time -from typing import List, NamedTuple, Optional, Union - -from aiohttp import ClientError -from discord import Color, Embed, Guild, Member, Message, TextChannel, User, errors -from discord.ext.commands import Bot, Context, command, group - -from bot.constants import ( - BigBrother as BigBrotherConfig, Channels, Emojis, - Guild as GuildConfig, Keys, - MODERATION_ROLES, STAFF_ROLES, URLs -) -from bot.decorators import with_role -from bot.pagination import LinePaginator -from bot.utils import messages -from bot.utils.moderation import post_infraction -from bot.utils.time import parse_rfc1123, time_since - -log = logging.getLogger(__name__) - -URL_RE = re.compile(r"(https?://[^\s]+)") - - -class WatchInformation(NamedTuple): - reason: str - actor_id: Optional[int] - inserted_at: Optional[str] - - -class BigBrother: - """User monitoring to assist with moderation.""" - - HEADERS = {'X-API-Key': Keys.site_api} - - def __init__(self, bot: Bot): - self.bot = bot - self.watched_users = {} # { user_id: log_channel_id } - self.watch_reasons = {} # { user_id: watch_reason } - self.channel_queues = defaultdict(lambda: defaultdict(deque)) # { user_id: { channel_id: queue(messages) } - self.last_log = [None, None, 0] # [user_id, channel_id, message_count] - self.consuming = False - self.consume_task = None - self.infraction_watch_prefix = "bb watch: " # Please do not change or we won't be able to find old reasons - self.nomination_prefix = "Helper nomination: " - - self.bot.loop.create_task(self.get_watched_users()) - - def update_cache(self, api_response: List[dict]): - """ - Updates the internal cache of watched users from the given `api_response`. - This function will only add (or update) existing keys, it will not delete - keys that were not present in the API response. - A user is only added if the bot can find a channel - with the given `channel_id` in its channel cache. - """ - - for entry in api_response: - user_id = int(entry['user_id']) - channel_id = int(entry['channel_id']) - channel = self.bot.get_channel(channel_id) - - if channel is not None: - self.watched_users[user_id] = channel - else: - log.error( - f"Site specified to relay messages by `{user_id}` in `{channel_id}`, " - "but the given channel could not be found. Ignoring." - ) - - async def get_watched_users(self): - """Retrieves watched users from the API.""" - - await self.bot.wait_until_ready() - async with self.bot.http_session.get(URLs.site_bigbrother_api, headers=self.HEADERS) as response: - data = await response.json() - self.update_cache(data) - - async def update_watched_users(self): - async with self.bot.http_session.get(URLs.site_bigbrother_api, headers=self.HEADERS) as response: - if response.status == 200: - data = await response.json() - self.update_cache(data) - log.trace("Updated Big Brother watchlist cache") - return True - else: - return False - - async def get_watch_information(self, user_id: int, prefix: str) -> WatchInformation: - """ Fetches and returns the latest watch reason for a user using the infraction API """ - - re_bb_watch = rf"^{prefix}" - user_id = str(user_id) - - try: - response = await self.bot.http_session.get( - URLs.site_infractions_user_type.format( - user_id=user_id, - infraction_type="note", - ), - params={"search": re_bb_watch, "hidden": "True", "active": "False"}, - headers=self.HEADERS - ) - infraction_list = await response.json() - except ClientError: - log.exception(f"Failed to retrieve bb watch reason for {user_id}.") - return WatchInformation(reason="(error retrieving bb reason)", actor_id=None, inserted_at=None) - - if infraction_list: - # Get the latest watch reason - latest_reason_infraction = max(infraction_list, key=self._parse_infraction_time) - - # Get the actor of the watch/nominate action - actor_id = int(latest_reason_infraction["actor"]["user_id"]) - - # Get the date the watch was set - date = latest_reason_infraction["inserted_at"] - - # Get the latest reason without the prefix - latest_reason = latest_reason_infraction['reason'][len(prefix):] - - log.trace(f"The latest bb watch reason for {user_id}: {latest_reason}") - return WatchInformation(reason=latest_reason, actor_id=actor_id, inserted_at=date) - - log.trace(f"No bb watch reason found for {user_id}; returning defaults") - return WatchInformation(reason="(no reason specified)", actor_id=None, inserted_at=None) - - @staticmethod - def _parse_infraction_time(infraction: dict) -> struct_time: - """ - Helper function that retrieves the insertion time from the infraction dictionary, - converts the retrieved RFC1123 date_time string to a time object, and returns it - so infractions can be sorted by their insertion time. - """ - - date_string = infraction["inserted_at"] - return strptime(date_string, "%a, %d %b %Y %H:%M:%S %Z") - - async def on_member_ban(self, guild: Guild, user: Union[User, Member]): - if guild.id == GuildConfig.id and user.id in self.watched_users: - url = f"{URLs.site_bigbrother_api}?user_id={user.id}" - channel = self.watched_users[user.id] - - async with self.bot.http_session.delete(url, headers=self.HEADERS) as response: - del self.watched_users[user.id] - del self.channel_queues[user.id] - del self.watch_reasons[user.id] - if response.status == 204: - await channel.send( - f"{Emojis.bb_message}:hammer: {user} got banned, so " - f"`BigBrother` will no longer relay their messages to {channel}" - ) - - else: - data = await response.json() - reason = data.get('error_message', "no message provided") - await channel.send( - f"{Emojis.bb_message}:x: {user} got banned, but trying to remove them from" - f"BigBrother's user dictionary on the API returned an error: {reason}" - ) - - async def on_message(self, msg: Message): - """Queues up messages sent by watched users.""" - - if msg.author.id in self.watched_users: - if not self.consuming: - self.consume_task = self.bot.loop.create_task(self.consume_messages()) - - if self.consuming and self.consume_task.done(): - # This should never happen, so something went wrong - - log.error("The consume_task has finished, but did not reset the self.consuming boolean") - e = self.consume_task.exception() - if e: - log.exception("The Exception for the Task:", exc_info=e) - else: - log.error("However, an Exception was not found.") - - self.consume_task = self.bot.loop.create_task(self.consume_messages()) - - log.trace(f"Received message: {msg.content} ({len(msg.attachments)} attachments)") - self.channel_queues[msg.author.id][msg.channel.id].append(msg) - - async def consume_messages(self): - """Consumes the message queues to log watched users' messages.""" - - if not self.consuming: - self.consuming = True - log.trace("Sleeping before consuming...") - await asyncio.sleep(BigBrotherConfig.log_delay) - - log.trace("Begin consuming messages.") - channel_queues = self.channel_queues.copy() - self.channel_queues.clear() - for user_id, queues in channel_queues.items(): - for _, queue in queues.items(): - channel = self.watched_users[user_id] - while queue: - msg = queue.popleft() - log.trace(f"Consuming message: {msg.clean_content} ({len(msg.attachments)} attachments)") - - self.last_log[2] += 1 # Increment message count. - await self.send_header(msg, channel) - await self.log_message(msg, channel) - - if self.channel_queues: - log.trace("Queue not empty; continue consumption.") - self.consume_task = self.bot.loop.create_task(self.consume_messages()) - else: - log.trace("Done consuming messages.") - self.consuming = False - - async def send_header(self, message: Message, destination: TextChannel): - """ - Sends a log message header to the given channel. - - A header is only sent if the user or channel are different than the previous, or if the configured message - limit for a single header has been exceeded. - - :param message: the first message in the queue - :param destination: the channel in which to send the header - """ - - last_user, last_channel, msg_count = self.last_log - limit = BigBrotherConfig.header_message_limit - - # Send header if user/channel are different or if message limit exceeded. - if message.author.id != last_user or message.channel.id != last_channel or msg_count > limit: - # Retrieve watch reason from API if it's not already in the cache - if message.author.id not in self.watch_reasons: - log.trace(f"No watch information for {message.author.id} found in cache; retrieving from API") - if destination == self.bot.get_channel(Channels.talent_pool): - prefix = self.nomination_prefix - else: - prefix = self.infraction_watch_prefix - user_watch_information = await self.get_watch_information(message.author.id, prefix) - self.watch_reasons[message.author.id] = user_watch_information - - self.last_log = [message.author.id, message.channel.id, 0] - - # Get reason, actor, inserted_at - reason, actor_id, inserted_at = self.watch_reasons[message.author.id] - - # Setting up the default author_field - author_field = message.author.nick or message.author.name - - # When we're dealing with a talent-pool header, add nomination info to the author field - if destination == self.bot.get_channel(Channels.talent_pool): - log.trace("We're sending a header to the talent-pool; let's add nomination info") - # If a reason was provided, both should be known - if actor_id and inserted_at: - # Parse actor name - guild: GuildConfig = self.bot.get_guild(GuildConfig.id) - actor_as_member = guild.get_member(actor_id) - actor = actor_as_member.nick or actor_as_member.name - - # Get time delta since insertion - date_time = parse_rfc1123(inserted_at).replace(tzinfo=None) - time_delta = time_since(date_time, precision="minutes", max_units=1) - - # Adding nomination info to author_field - author_field = f"{author_field} (nominated {time_delta} by {actor})" - else: - if inserted_at: - # Get time delta since insertion - date_time = parse_rfc1123(inserted_at).replace(tzinfo=None) - time_delta = time_since(date_time, precision="minutes", max_units=1) - - author_field = f"{author_field} (added {time_delta})" - - embed = Embed(description=f"{message.author.mention} in [#{message.channel.name}]({message.jump_url})") - embed.set_author(name=author_field, icon_url=message.author.avatar_url) - embed.set_footer(text=f"Reason: {reason}") - await destination.send(embed=embed) - - @staticmethod - async def log_message(message: Message, destination: TextChannel): - """ - Logs a watched user's message in the given channel. - - Attachments are also sent. All non-image or non-video URLs are put in inline code blocks to prevent preview - embeds from being automatically generated. - - :param message: the message to log - :param destination: the channel in which to log the message - """ - - content = message.clean_content - if content: - # Put all non-media URLs in inline code blocks. - media_urls = {embed.url for embed in message.embeds if embed.type in ("image", "video")} - for url in URL_RE.findall(content): - if url not in media_urls: - content = content.replace(url, f"`{url}`") - - await destination.send(content) - - try: - await messages.send_attachments(message, destination) - except (errors.Forbidden, errors.NotFound): - e = Embed( - description=":x: **This message contained an attachment, but it could not be retrieved**", - color=Color.red() - ) - await destination.send(embed=e) - - async def _watch_user(self, ctx: Context, user: User, reason: str, channel_id: int): - post_data = { - 'user_id': str(user.id), - 'channel_id': str(channel_id) - } - - async with self.bot.http_session.post( - URLs.site_bigbrother_api, - headers=self.HEADERS, - json=post_data - ) as response: - if response.status == 204: - if channel_id == Channels.talent_pool: - await ctx.send(f":ok_hand: added {user} to the <#{channel_id}>!") - else: - await ctx.send(f":ok_hand: will now relay messages sent by {user} in <#{channel_id}>") - - channel = self.bot.get_channel(channel_id) - if channel is None: - log.error( - f"could not update internal cache, failed to find a channel with ID {channel_id}" - ) - else: - self.watched_users[user.id] = channel - - # Add a note (shadow warning) with the reason for watching - await post_infraction(ctx, user, type="warning", reason=reason, hidden=True) - else: - data = await response.json() - error_reason = data.get('error_message', "no message provided") - await ctx.send(f":x: the API returned an error: {error_reason}") - - @group(name='bigbrother', aliases=('bb',), invoke_without_command=True) - @with_role(*MODERATION_ROLES) - async def bigbrother_group(self, ctx: Context): - """Monitor users, NSA-style.""" - - await ctx.invoke(self.bot.get_command("help"), "bigbrother") - - @bigbrother_group.command(name='watched', aliases=('all',)) - @with_role(*MODERATION_ROLES) - async def watched_command(self, ctx: Context, from_cache: bool = True): - """ - Shows all users that are currently monitored and in which channel. - By default, the users are returned from the cache. - If this is not desired, `from_cache` can be given as a falsy value, e.g. e.g. 'no'. - """ - if not from_cache: - updated = await self.update_watched_users() - if not updated: - await ctx.send(f":x: Failed to update cache: non-200 response from the API") - return - title = "Watched users (updated cache)" - else: - title = "Watched users (from cache)" - - lines = tuple( - f"• <@{user_id}> in <#{self.watched_users[user_id].id}>" - for user_id in self.watched_users - ) - await LinePaginator.paginate( - lines or ("There's nothing here yet.",), - ctx, - Embed(title=title, color=Color.blue()), - empty=False - ) - - @bigbrother_group.command(name='watch', aliases=('w',)) - @with_role(*MODERATION_ROLES) - async def watch_command(self, ctx: Context, user: User, *, reason: str): - """ - Relay messages sent by the given `user` to the `#big-brother-logs` channel - - A `reason` for watching is required, which is added for the user to be watched as a - note (aka: shadow warning) - """ - - # Update cache to avoid double watching of a user - await self.update_watched_users() - - if user.id in self.watched_users: - message = f":x: User is already being watched in {self.watched_users[user.id].name}" - await ctx.send(message) - return - - channel_id = Channels.big_brother_logs - - reason = f"{self.infraction_watch_prefix}{reason}" - - await self._watch_user(ctx, user, reason, channel_id) - - @bigbrother_group.command(name='unwatch', aliases=('uw',)) - @with_role(*MODERATION_ROLES) - async def unwatch_command(self, ctx: Context, user: User, *, reason: str): - """ - Stop relaying messages by the given `user`. - - A `reason` for unwatching is required, which will be added as a note to the user. - """ - - url = f"{URLs.site_bigbrother_api}?user_id={user.id}" - async with self.bot.http_session.delete(url, headers=self.HEADERS) as response: - if response.status == 204: - await ctx.send(f":ok_hand: will no longer relay messages sent by {user}") - - if user.id in self.watched_users: - channel = self.watched_users[user.id] - - del self.watched_users[user.id] - if user.id in self.channel_queues: - del self.channel_queues[user.id] - if user.id in self.watch_reasons: - del self.watch_reasons[user.id] - else: - channel = None - log.warning(f"user {user.id} was unwatched but was not found in the cache") - - reason = f"Unwatched ({channel.name if channel else 'unknown channel'}): {reason}" - await post_infraction(ctx, user, type="warning", reason=reason, hidden=True) - - else: - data = await response.json() - reason = data.get('error_message', "no message provided") - await ctx.send(f":x: the API returned an error: {reason}") - - @bigbrother_group.command(name='nominate', aliases=('n',)) - @with_role(*MODERATION_ROLES) - async def nominate_command(self, ctx: Context, user: User, *, reason: str): - """ - Nominates a user for the helper role by adding them to the talent-pool channel - - A `reason` for the nomination is required and will be added as a note to - the user's records. - """ - - # Note: This function is called from HelperNomination.nominate_command so that the - # !nominate command does not show up under "BigBrother" in the help embed, but under - # the header HelperNomination for users with the helper role. - - member = ctx.guild.get_member(user.id) - - if member and any(role.id in STAFF_ROLES for role in member.roles): - await ctx.send(f":x: {user.mention} is already a staff member!") - return - - channel_id = Channels.talent_pool - - # Update watch cache to avoid overwriting active nomination reason - await self.update_watched_users() - - if user.id in self.watched_users: - if self.watched_users[user.id].id == Channels.talent_pool: - prefix = "Additional nomination: " - else: - # If the user is being watched in big-brother, don't add them to talent-pool - message = ( - f":x: {user.mention} can't be added to the talent-pool " - "as they are currently being watched in big-brother." - ) - await ctx.send(message) - return - else: - prefix = self.nomination_prefix - - reason = f"{prefix}{reason}" - - await self._watch_user(ctx, user, reason, channel_id) - - -class HelperNomination: - def __init__(self, bot): - self.bot = bot - - @command(name='nominate', aliases=('n',)) - @with_role(*STAFF_ROLES) - async def nominate_command(self, ctx: Context, user: User, *, reason: str): - """ - Nominates a user for the helper role by adding them to the talent-pool channel - - A `reason` for the nomination is required and will be added as a note to - the user's records. - """ - - cmd = self.bot.get_command("bigbrother nominate") - - await ctx.invoke(cmd, user, reason=reason) - - -def setup(bot: Bot): - bot.add_cog(BigBrother(bot)) - bot.add_cog(HelperNomination(bot)) - log.info("Cog loaded: BigBrother") diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index b34d1118b..e7b6bac85 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -165,7 +165,7 @@ class Clean: # Reverse the list to restore chronological order if messages: messages = list(reversed(messages)) - log_url = await self.mod_log.upload_log(messages) + log_url = await self.mod_log.upload_log(messages, ctx.author.id) else: # Can't build an embed, nothing to clean! embed = Embed( diff --git a/bot/cogs/defcon.py b/bot/cogs/defcon.py index f07d9df9f..c67fa2807 100644 --- a/bot/cogs/defcon.py +++ b/bot/cogs/defcon.py @@ -5,7 +5,7 @@ from discord import Colour, Embed, Member from discord.ext.commands import Bot, Context, group from bot.cogs.modlog import ModLog -from bot.constants import Channels, Colours, Emojis, Event, Icons, Keys, Roles, URLs +from bot.constants import Channels, Colours, Emojis, Event, Icons, Keys, Roles from bot.decorators import with_role log = logging.getLogger(__name__) @@ -40,13 +40,8 @@ class Defcon: async def on_ready(self): try: - response = await self.bot.http_session.get( - URLs.site_settings_api, - headers=self.headers, - params={"keys": "defcon_enabled,defcon_days"} - ) - - data = await response.json() + response = await self.bot.api_client.get('bot/bot-settings/defcon') + data = response['data'] except Exception: # Yikes! log.exception("Unable to get DEFCON settings!") @@ -55,9 +50,9 @@ class Defcon: ) else: - if data["defcon_enabled"]: + if data["enabled"]: self.enabled = True - self.days = timedelta(days=data["defcon_days"]) + self.days = timedelta(days=data["days"]) log.warning(f"DEFCON enabled: {self.days.days} days") else: @@ -118,13 +113,18 @@ class Defcon: self.enabled = True try: - response = await self.bot.http_session.put( - URLs.site_settings_api, - headers=self.headers, - json={"defcon_enabled": True} + await self.bot.api_client.put( + 'bot/bot-settings/defcon', + json={ + 'name': 'defcon', + 'data': { + 'enabled': True, + # TODO: retrieve old days count + 'days': 0 + } + } ) - await response.json() except Exception as e: log.exception("Unable to update DEFCON settings.") await ctx.send( @@ -142,6 +142,7 @@ class Defcon: "restarted.\n\n" f"```py\n{e}\n```" ) + else: await ctx.send(f"{Emojis.defcon_enabled} DEFCON enabled.") @@ -163,13 +164,16 @@ class Defcon: self.enabled = False try: - response = await self.bot.http_session.put( - URLs.site_settings_api, - headers=self.headers, - json={"defcon_enabled": False} + await self.bot.api_client.put( + 'bot/bot-settings/defcon', + json={ + 'data': { + 'days': 0, + 'enabled': False + }, + 'name': 'defcon' + } ) - - await response.json() except Exception as e: log.exception("Unable to update DEFCON settings.") await ctx.send( @@ -221,13 +225,16 @@ class Defcon: self.days = timedelta(days=days) try: - response = await self.bot.http_session.put( - URLs.site_settings_api, - headers=self.headers, - json={"defcon_days": days} + await self.bot.api_client.put( + 'bot/bot-settings/defcon', + json={ + 'data': { + 'days': days, + 'enabled': True + }, + 'name': 'defcon' + } ) - - await response.json() except Exception as e: log.exception("Unable to update DEFCON settings.") await ctx.send( diff --git a/bot/cogs/doc.py b/bot/cogs/doc.py index 2f2cf8000..aa49b0c25 100644 --- a/bot/cogs/doc.py +++ b/bot/cogs/doc.py @@ -1,11 +1,10 @@ import asyncio import functools import logging -import random import re import textwrap from collections import OrderedDict -from typing import Dict, List, Optional, Tuple +from typing import Optional, Tuple import discord from bs4 import BeautifulSoup @@ -14,7 +13,7 @@ from markdownify import MarkdownConverter from requests import ConnectionError from sphinx.ext import intersphinx -from bot.constants import ERROR_REPLIES, Keys, MODERATION_ROLES, URLs +from bot.constants import MODERATION_ROLES from bot.converters import ValidPythonIdentifier, ValidURL from bot.decorators import with_role from bot.pagination import LinePaginator @@ -126,7 +125,6 @@ class Doc: self.base_urls = {} self.bot = bot self.inventories = {} - self.headers = {"X-API-KEY": Keys.site_api} async def on_ready(self): await self.refresh_inventory() @@ -179,7 +177,7 @@ class Doc: coros = [ self.update_single( package["package"], package["base_url"], package["inventory_url"], config - ) for package in await self.get_all_packages() + ) for package in await self.bot.api_client.get('bot/documentation-links') ] await asyncio.gather(*coros) @@ -267,95 +265,6 @@ class Doc: description=f"```py\n{signature}```{description}" ) - async def get_all_packages(self) -> List[Dict[str, str]]: - """ - Performs HTTP GET to get all packages from the website. - - :return: - A list of packages, in the following format: - [ - { - "package": "example-package", - "base_url": "https://example.readthedocs.io", - "inventory_url": "https://example.readthedocs.io/objects.inv" - }, - ... - ] - `package` specifies the package name, for example 'aiohttp'. - `base_url` specifies the documentation root URL, used to build absolute links. - `inventory_url` specifies the location of the Intersphinx inventory. - """ - - async with self.bot.http_session.get(URLs.site_docs_api, headers=self.headers) as resp: - return await resp.json() - - async def get_package(self, package_name: str) -> Optional[Dict[str, str]]: - """ - Performs HTTP GET to get the specified package from the documentation database. - - :param package_name: The package name for which information should be returned. - :return: - Either a dictionary with information in the following format: - { - "package": "example-package", - "base_url": "https://example.readthedocs.io", - "inventory_url": "https://example.readthedocs.io/objects.inv" - } - or `None` if the site didn't returned no results for the given name. - """ - - params = {"package": package_name} - - async with self.bot.http_session.get(URLs.site_docs_api, - headers=self.headers, - params=params) as resp: - package_data = await resp.json() - if not package_data: - return None - return package_data[0] - - async def set_package(self, name: str, base_url: str, inventory_url: str) -> Dict[str, bool]: - """ - Performs HTTP POST to add a new package to the website's documentation database. - - :param name: The name of the package, for example `aiohttp`. - :param base_url: The documentation root URL, used to build absolute links. - :param inventory_url: The absolute URl to the intersphinx inventory of the package. - - :return: The JSON response of the server, which is always: - { - "success": True - } - """ - - package_json = { - 'package': name, - 'base_url': base_url, - 'inventory_url': inventory_url - } - - async with self.bot.http_session.post(URLs.site_docs_api, - headers=self.headers, - json=package_json) as resp: - return await resp.json() - - async def delete_package(self, name: str) -> bool: - """ - Performs HTTP DELETE to delete the specified package from the documentation database. - - :param name: The package to delete. - - :return: `True` if successful, `False` if the package is unknown. - """ - - package_json = {'package': name} - - async with self.bot.http_session.delete(URLs.site_docs_api, - headers=self.headers, - json=package_json) as resp: - changes = await resp.json() - return changes["deleted"] == 1 # Did the package delete successfully? - @commands.group(name='docs', aliases=('doc', 'd'), invoke_without_command=True) async def docs_group(self, ctx, symbol: commands.clean_content = None): """Lookup documentation for Python symbols.""" @@ -386,7 +295,12 @@ class Doc: ) lines = sorted(f"• [`{name}`]({url})" for name, url in self.base_urls.items()) - await LinePaginator.paginate(lines, ctx, inventory_embed, max_size=400, empty=False) + if self.base_urls: + await LinePaginator.paginate(lines, ctx, inventory_embed, max_size=400, empty=False) + + else: + inventory_embed.description = "Hmmm, seems like there's nothing here yet." + await ctx.send(embed=inventory_embed) else: # Fetching documentation for a symbol (at least for the first time, since @@ -427,7 +341,13 @@ class Doc: https://discordpy.readthedocs.io/en/rewrite/objects.inv """ - await self.set_package(package_name, base_url, inventory_url) + body = { + 'package': package_name, + 'base_url': base_url, + 'inventory_url': inventory_url + } + await self.bot.api_client.post('bot/documentation-links', json=body) + log.info( f"User @{ctx.author.name}#{ctx.author.discriminator} ({ctx.author.id}) " "added a new documentation package:\n" @@ -455,42 +375,13 @@ class Doc: !docs delete aiohttp """ - success = await self.delete_package(package_name) - if success: + await self.bot.api_client.delete(f'bot/documentation-links/{package_name}') - async with ctx.typing(): - # Rebuild the inventory to ensure that everything - # that was from this package is properly deleted. - await self.refresh_inventory() - await ctx.send(f"Successfully deleted `{package_name}` and refreshed inventory.") - - else: - await ctx.send( - f"Can't find any package named `{package_name}` in the database. " - "View all known packages by using `docs.get()`." - ) - - @get_command.error - @delete_command.error - @set_command.error - async def general_command_error(self, ctx, error: commands.CommandError): - """ - Handle the `BadArgument` error caused by - the commands when argument validation fails. - - :param ctx: Discord message context of the message creating the error - :param error: The error raised, usually `BadArgument` - """ - - if isinstance(error, commands.BadArgument): - embed = discord.Embed( - title=random.choice(ERROR_REPLIES), - description=f"Error: {error}", - colour=discord.Colour.red() - ) - await ctx.send(embed=embed) - else: - log.exception(f"Unhandled error: {error}") + async with ctx.typing(): + # Rebuild the inventory to ensure that everything + # that was from this package is properly deleted. + await self.refresh_inventory() + await ctx.send(f"Successfully deleted `{package_name}` and refreshed inventory.") def setup(bot): diff --git a/bot/cogs/events.py b/bot/cogs/events.py deleted file mode 100644 index 2819b7dcc..000000000 --- a/bot/cogs/events.py +++ /dev/null @@ -1,311 +0,0 @@ -import logging -from functools import partial -from typing import List - -from discord import Colour, Embed, Member, Object -from discord.ext.commands import ( - BadArgument, Bot, BotMissingPermissions, - CommandError, CommandInvokeError, CommandNotFound, - Context, NoPrivateMessage, UserInputError -) - -from bot.constants import ( - Channels, Colours, DEBUG_MODE, - Guild, Icons, Keys, - Roles, URLs -) -from bot.utils import chunks - -log = logging.getLogger(__name__) - -RESTORE_ROLES = (str(Roles.muted), str(Roles.announcements)) - - -class Events: - """No commands, just event handlers.""" - - def __init__(self, bot: Bot): - self.bot = bot - self.headers = {"X-API-KEY": Keys.site_api} - - @property - def send_log(self) -> partial: - cog = self.bot.get_cog("ModLog") - return partial(cog.send_log_message, channel_id=Channels.userlog) - - async def send_updated_users(self, *users, replace_all=False): - users = list(filter(lambda user: str(Roles.verified) in user["roles"], users)) - - for chunk in chunks(users, 1000): - response = None - - try: - if replace_all: - response = await self.bot.http_session.post( - url=URLs.site_user_api, - json=chunk, - headers={"X-API-Key": Keys.site_api} - ) - else: - response = await self.bot.http_session.put( - url=URLs.site_user_api, - json=chunk, - headers={"X-API-Key": Keys.site_api} - ) - - await response.json() # We do this to ensure we got a proper response from the site - except Exception: - if not response: - log.exception(f"Failed to send {len(chunk)} users") - else: - text = await response.text() - log.exception(f"Failed to send {len(chunk)} users", extra={"body": text}) - break # Stop right now, thank you very much - - result = {} - - if replace_all: - response = None - - try: - response = await self.bot.http_session.post( - url=URLs.site_user_complete_api, - headers={"X-API-Key": Keys.site_api} - ) - - result = await response.json() - except Exception: - if not response: - log.exception(f"Failed to send {len(chunk)} users") - else: - text = await response.text() - log.exception(f"Failed to send {len(chunk)} users", extra={"body": text}) - - return result - - async def send_delete_users(self, *users): - try: - response = await self.bot.http_session.delete( - url=URLs.site_user_api, - json=list(users), - headers={"X-API-Key": Keys.site_api} - ) - - return await response.json() - except Exception: - log.exception(f"Failed to delete {len(users)} users") - return {} - - async def get_user(self, user_id): - response = await self.bot.http_session.get( - url=URLs.site_user_api, - params={"user_id": user_id}, - headers={"X-API-Key": Keys.site_api} - ) - - resp = await response.json() - return resp["data"] - - async def has_active_mute(self, user_id: str) -> bool: - """ - Check whether a user has any active mute infractions - """ - - response = await self.bot.http_session.get( - URLs.site_infractions_user.format( - user_id=user_id - ), - params={"hidden": "True"}, - headers=self.headers - ) - infraction_list = await response.json() - - # Check for active mute infractions - if not infraction_list: - # Short circuit - return False - - return any( - infraction["active"] for infraction in infraction_list if infraction["type"].lower() == "mute" - ) - - async def on_command_error(self, ctx: Context, e: CommandError): - command = ctx.command - parent = None - - if command is not None: - parent = command.parent - - if parent and command: - help_command = (self.bot.get_command("help"), parent.name, command.name) - elif command: - help_command = (self.bot.get_command("help"), command.name) - else: - help_command = (self.bot.get_command("help"),) - - if hasattr(command, "on_error"): - log.debug(f"Command {command} has a local error handler, ignoring.") - return - - if isinstance(e, CommandNotFound) and not hasattr(ctx, "invoked_from_error_handler"): - tags_get_command = self.bot.get_command("tags get") - ctx.invoked_from_error_handler = True - - # Return to not raise the exception - return await ctx.invoke(tags_get_command, tag_name=ctx.invoked_with) - elif isinstance(e, BadArgument): - await ctx.send(f"Bad argument: {e}\n") - await ctx.invoke(*help_command) - elif isinstance(e, UserInputError): - await ctx.invoke(*help_command) - elif isinstance(e, NoPrivateMessage): - await ctx.send("Sorry, this command can't be used in a private message!") - elif isinstance(e, BotMissingPermissions): - await ctx.send( - f"Sorry, it looks like I don't have the permissions I need to do that.\n\n" - f"Here's what I'm missing: **{e.missing_perms}**" - ) - elif isinstance(e, CommandInvokeError): - await ctx.send( - f"Sorry, an unexpected error occurred. Please let us know!\n\n```{e}```" - ) - raise e.original - raise e - - async def on_ready(self): - users = [] - - for member in self.bot.get_guild(Guild.id).members: # type: Member - roles: List[int] = [str(r.id) for r in member.roles] - - users.append({ - "avatar": member.avatar_url_as(format="png"), - "user_id": str(member.id), - "roles": roles, - "username": member.name, - "discriminator": member.discriminator - }) - - if users: - log.info(f"{len(users)} user roles to be updated") - - done = await self.send_updated_users(*users, replace_all=True) - - if any(done.values()): - embed = Embed( - title="Users updated" - ) - - for key, value in done.items(): - if value: - if key == "deleted_oauth": - key = "Deleted (OAuth)" - elif key == "deleted_jam_profiles": - key = "Deleted (Jammer Profiles)" - elif key == "deleted_responses": - key = "Deleted (Jam Form Responses)" - elif key == "jam_bans": - key = "Ex-Jammer Bans" - else: - key = key.title() - - embed.add_field( - name=key, value=str(value) - ) - - if not DEBUG_MODE: - await self.bot.get_channel(Channels.devlog).send( - embed=embed - ) - - async def on_member_update(self, before: Member, after: Member): - if ( - before.roles == after.roles - and before.name == after.name - and before.discriminator == after.discriminator - and before.avatar == after.avatar): - return - - before_role_names: List[str] = [role.name for role in before.roles] - after_role_names: List[str] = [role.name for role in after.roles] - role_ids: List[str] = [str(r.id) for r in after.roles] - - log.debug(f"{before.display_name} roles changing from {before_role_names} to {after_role_names}") - - changes = await self.send_updated_users({ - "avatar": after.avatar_url_as(format="png"), - "user_id": str(after.id), - "roles": role_ids, - "username": after.name, - "discriminator": after.discriminator - }) - - log.debug(f"User {after.id} updated; changes: {changes}") - - async def on_member_join(self, member: Member): - role_ids: List[str] = [str(r.id) for r in member.roles] - new_roles = [] - - try: - user_objs = await self.get_user(str(member.id)) - except Exception as e: - log.exception("Failed to persist roles") - - await self.send_log( - Icons.crown_red, Colour(Colours.soft_red), "Failed to persist roles", - f"```py\n{e}\n```", - member.avatar_url_as(static_format="png") - ) - else: - if user_objs: - old_roles = user_objs[0].get("roles", []) - - for role in RESTORE_ROLES: - if role in old_roles: - # Check for mute roles that were not able to be removed and skip if present - if role == str(Roles.muted) and not await self.has_active_mute(str(member.id)): - log.debug( - f"User {member.id} has no active mute infraction, " - "their leftover muted role will not be persisted" - ) - continue - - new_roles.append(Object(int(role))) - - for role in new_roles: - if str(role) not in role_ids: - role_ids.append(str(role.id)) - - changes = await self.send_updated_users({ - "avatar": member.avatar_url_as(format="png"), - "user_id": str(member.id), - "roles": role_ids, - "username": member.name, - "discriminator": member.discriminator - }) - - log.debug(f"User {member.id} joined; changes: {changes}") - - if new_roles: - await member.add_roles( - *new_roles, - reason="Roles restored" - ) - - await self.send_log( - Icons.crown_blurple, Colour.blurple(), "Roles restored", - f"Restored {len(new_roles)} roles", - member.avatar_url_as(static_format="png") - ) - - async def on_member_remove(self, member: Member): - changes = await self.send_delete_users({ - "user_id": str(member.id) - }) - - log.debug(f"User {member.id} left; changes: {changes}") - - -def setup(bot): - bot.add_cog(Events(bot)) - log.info("Cog loaded: Events") diff --git a/bot/cogs/information.py b/bot/cogs/information.py index 469999c00..a2585f395 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -3,11 +3,12 @@ import random import textwrap from discord import CategoryChannel, Colour, Embed, Member, TextChannel, VoiceChannel -from discord.ext.commands import BadArgument, Bot, CommandError, Context, MissingPermissions, command +from discord.ext.commands import ( + BadArgument, Bot, CommandError, Context, MissingPermissions, command +) from bot.constants import ( - Channels, Emojis, Keys, MODERATION_ROLES, - NEGATIVE_REPLIES, STAFF_ROLES, URLs + Channels, Emojis, Keys, MODERATION_ROLES, NEGATIVE_REPLIES, STAFF_ROLES ) from bot.decorators import with_role from bot.utils.checks import with_role_check @@ -166,14 +167,14 @@ class Information: ) # Infractions - api_response = await self.bot.http_session.get( - url=URLs.site_infractions_user.format(user_id=user.id), - params={"hidden": hidden}, - headers=self.headers + infractions = await self.bot.api_client.get( + 'bot/infractions', + params={ + 'hidden': hidden, + 'user__id': str(user.id) + } ) - infractions = await api_response.json() - infr_total = 0 infr_active = 0 diff --git a/bot/cogs/moderation.py b/bot/cogs/moderation.py index 73359c88c..1dc2c70d6 100644 --- a/bot/cogs/moderation.py +++ b/bot/cogs/moderation.py @@ -1,9 +1,9 @@ import asyncio import logging import textwrap +from datetime import datetime from typing import Union -from aiohttp import ClientError from discord import ( Colour, Embed, Forbidden, Guild, HTTPException, Member, Object, User ) @@ -13,13 +13,13 @@ from discord.ext.commands import ( from bot import constants from bot.cogs.modlog import ModLog -from bot.constants import Colours, Event, Icons, Keys, MODERATION_ROLES, URLs -from bot.converters import InfractionSearchQuery +from bot.constants import Colours, Event, Icons, MODERATION_ROLES +from bot.converters import ExpirationDate, InfractionSearchQuery from bot.decorators import with_role from bot.pagination import LinePaginator from bot.utils.moderation import post_infraction from bot.utils.scheduling import Scheduler, create_task -from bot.utils.time import parse_rfc1123, wait_until +from bot.utils.time import wait_until log = logging.getLogger(__name__) @@ -53,7 +53,6 @@ class Moderation(Scheduler): def __init__(self, bot: Bot): self.bot = bot - self.headers = {"X-API-KEY": Keys.site_api} self._muted_role = Object(constants.Roles.muted) super().__init__() @@ -63,15 +62,12 @@ class Moderation(Scheduler): async def on_ready(self): # Schedule expiration for previous infractions - response = await self.bot.http_session.get( - URLs.site_infractions, - params={"dangling": "true"}, - headers=self.headers + infractions = await self.bot.api_client.get( + 'bot/infractions', params={'active': 'true'} ) - infraction_list = await response.json() - for infraction_object in infraction_list: - if infraction_object["expires_at"] is not None: - self.schedule_task(self.bot.loop, infraction_object["id"], infraction_object) + for infraction in infractions: + if infraction["expires_at"] is not None: + self.schedule_task(self.bot.loop, infraction["id"], infraction) # region: Permanent infractions @@ -200,6 +196,20 @@ class Moderation(Scheduler): # Warning is sent to ctx by the helper method return + active_bans = await self.bot.api_client.get( + 'bot/infractions', + params={ + 'active': 'true', + 'type': 'ban', + 'user__id': str(user.id) + } + ) + if active_bans: + return await ctx.send( + ":x: According to my records, this user is already banned. " + f"See infraction **#{active_bans[0]['id']}**." + ) + response_object = await post_infraction(ctx, user, type="ban", reason=reason) if response_object is None: return @@ -207,7 +217,6 @@ class Moderation(Scheduler): notified = await self.notify_infraction( user=user, infr_type="Ban", - duration="Permanent", reason=reason ) @@ -259,6 +268,20 @@ class Moderation(Scheduler): **`reason`:** The reason for the mute. """ + active_mutes = await self.bot.api_client.get( + 'bot/infractions', + params={ + 'active': 'true', + 'type': 'mute', + 'user__id': str(user.id) + } + ) + if active_mutes: + return await ctx.send( + ":x: According to my records, this user is already muted. " + f"See infraction **#{active_mutes[0]['id']}**." + ) + response_object = await post_infraction(ctx, user, type="mute", reason=reason) if response_object is None: return @@ -269,7 +292,7 @@ class Moderation(Scheduler): notified = await self.notify_infraction( user=user, infr_type="Mute", - duration="Permanent", + expires_at="Permanent", reason=reason ) @@ -308,7 +331,10 @@ class Moderation(Scheduler): @with_role(*MODERATION_ROLES) @command() - async def tempmute(self, ctx: Context, user: Member, duration: str, *, reason: str = None): + async def tempmute( + self, ctx: Context, user: Member, expiration: ExpirationDate, + *, reason: str = None + ): """ Create a temporary mute infraction in the database for a user. @@ -317,11 +343,25 @@ class Moderation(Scheduler): **`reason`:** The reason for the temporary mute. """ - response_object = await post_infraction( - ctx, user, type="mute", reason=reason, duration=duration + active_mutes = await self.bot.api_client.get( + 'bot/infractions', + params={ + 'active': 'true', + 'type': 'mute', + 'user__id': str(user.id) + } + ) + if active_mutes: + return await ctx.send( + ":x: According to my records, this user is already muted. " + f"See infraction **#{active_mutes[0]['id']}**." + ) + + infraction = await post_infraction( + ctx, user, + type="mute", reason=reason, + expires_at=expiration ) - if response_object is None: - return self.mod_log.ignore(Event.member_update, user.id) await user.add_roles(self._muted_role, reason=reason) @@ -329,14 +369,17 @@ class Moderation(Scheduler): notified = await self.notify_infraction( user=user, infr_type="Mute", - duration=duration, + expires_at=expiration, reason=reason ) - infraction_object = response_object["infraction"] - infraction_expiration = infraction_object["expires_at"] + infraction_expiration = ( + datetime + .fromisoformat(infraction["expires_at"][:-1]) + .strftime('%c') + ) - self.schedule_task(ctx.bot.loop, infraction_object["id"], infraction_object) + self.schedule_task(ctx.bot.loop, infraction["id"], infraction) dm_result = ":incoming_envelope: " if notified else "" action = f"{dm_result}:ok_hand: muted {user.mention} until {infraction_expiration}" @@ -363,24 +406,23 @@ class Moderation(Scheduler): Actor: {ctx.message.author} DM: {dm_status} Reason: {reason} - Duration: {duration} Expires: {infraction_expiration} """), content=log_content, - footer=f"ID {response_object['infraction']['id']}" + footer=f"ID {infraction['id']}" ) @with_role(*MODERATION_ROLES) @command() async def tempban( - self, ctx: Context, user: UserTypes, duration: str, *, reason: str = None + self, ctx: Context, user: UserTypes, expiry: ExpirationDate, *, reason: str = None ): """ Create a temporary ban infraction in the database for a user. - **`user`:** Accepts user mention, ID, etc. - **`duration`:** The duration for the temporary ban infraction - **`reason`:** The reason for the temporary ban. + **`user`:** Accepts user mention, ID, etc. + **`expiry`:** The duration for the temporary ban infraction + **`reason`:** The reason for the temporary ban. """ if not await self.respect_role_hierarchy(ctx, user, 'tempban'): @@ -388,16 +430,31 @@ class Moderation(Scheduler): # Warning is sent to ctx by the helper method return - response_object = await post_infraction( - ctx, user, type="ban", reason=reason, duration=duration + active_bans = await self.bot.api_client.get( + 'bot/infractions', + params={ + 'active': 'true', + 'type': 'ban', + 'user__id': str(user.id) + } ) - if response_object is None: + if active_bans: + return await ctx.send( + ":x: According to my records, this user is already banned. " + f"See infraction **#{active_bans[0]['id']}**." + ) + + infraction = await post_infraction( + ctx, user, type="ban", + reason=reason, expires_at=expiry + ) + if infraction is None: return notified = await self.notify_infraction( user=user, infr_type="Ban", - duration=duration, + expires_at=expiry, reason=reason ) @@ -410,10 +467,13 @@ class Moderation(Scheduler): except Forbidden: action_result = False - infraction_object = response_object["infraction"] - infraction_expiration = infraction_object["expires_at"] + infraction_expiration = ( + datetime + .fromisoformat(infraction["expires_at"][:-1]) + .strftime('%c') + ) - self.schedule_task(ctx.bot.loop, infraction_object["id"], infraction_object) + self.schedule_task(ctx.bot.loop, infraction["id"], infraction) dm_result = ":incoming_envelope: " if notified else "" action = f"{dm_result}:ok_hand: banned {user.mention} until {infraction_expiration}" @@ -439,11 +499,10 @@ class Moderation(Scheduler): Actor: {ctx.message.author} DM: {dm_status} Reason: {reason} - Duration: {duration} Expires: {infraction_expiration} """), content=log_content, - footer=f"ID {response_object['infraction']['id']}" + footer=f"ID {infraction['id']}" ) # endregion @@ -462,6 +521,7 @@ class Moderation(Scheduler): response_object = await post_infraction( ctx, user, type="warning", reason=reason, hidden=True ) + if response_object is None: return @@ -760,31 +820,27 @@ class Moderation(Scheduler): try: # check the current active infraction - response = await self.bot.http_session.get( - URLs.site_infractions_user_type_current.format( - user_id=user.id, - infraction_type="mute" - ), - headers=self.headers + response = await self.bot.api_client.get( + 'bot/infractions', + params={ + 'active': 'true', + 'type': 'mute', + 'user__id': user.id + } ) + if len(response) > 1: + log.warning("Found more than one active mute infraction for user `%d`", user.id) - response_object = await response.json() - if "error_code" in response_object: - return await ctx.send( - ":x: There was an error removing the infraction: " - f"{response_object['error_message']}" - ) - - infraction_object = response_object["infraction"] - if infraction_object is None: + if not response: # no active infraction return await ctx.send( f":x: There is no active mute infraction for user {user.mention}." ) - await self._deactivate_infraction(infraction_object) - if infraction_object["expires_at"] is not None: - self.cancel_expiration(infraction_object["id"]) + infraction = response[0] + await self._deactivate_infraction(infraction) + if infraction["expires_at"] is not None: + self.cancel_expiration(infraction["id"]) notified = await self.notify_pardon( user=user, @@ -813,15 +869,14 @@ class Moderation(Scheduler): text=textwrap.dedent(f""" Member: {user.mention} (`{user.id}`) Actor: {ctx.message.author} - Intended expiry: {infraction_object['expires_at']} + Intended expiry: {infraction['expires_at']} DM: {dm_status} """), - footer=infraction_object["id"], + footer=infraction["id"], content=log_content ) - - except Exception as e: - log.exception("There was an error removing an infraction.", exc_info=e) + except Exception: + log.exception("There was an error removing an infraction.") await ctx.send(":x: There was an error removing the infraction.") @with_role(*MODERATION_ROLES) @@ -835,30 +890,30 @@ class Moderation(Scheduler): try: # check the current active infraction - response = await self.bot.http_session.get( - URLs.site_infractions_user_type_current.format( - user_id=user.id, - infraction_type="ban" - ), - headers=self.headers + response = await self.bot.api_client.get( + 'bot/infractions', + params={ + 'active': 'true', + 'type': 'ban', + 'user__id': str(user.id) + } ) - response_object = await response.json() - if "error_code" in response_object: - return await ctx.send( - ":x: There was an error removing the infraction: " - f"{response_object['error_message']}" + if len(response) > 1: + log.warning( + "More than one active ban infraction found for user `%d`.", + user.id ) - infraction_object = response_object["infraction"] - if infraction_object is None: + if not response: # no active infraction return await ctx.send( f":x: There is no active ban infraction for user {user.mention}." ) - await self._deactivate_infraction(infraction_object) - if infraction_object["expires_at"] is not None: - self.cancel_expiration(infraction_object["id"]) + infraction = response[0] + await self._deactivate_infraction(infraction) + if infraction["expires_at"] is not None: + self.cancel_expiration(infraction["id"]) await ctx.send(f":ok_hand: Un-banned {user.mention}.") @@ -871,7 +926,7 @@ class Moderation(Scheduler): text=textwrap.dedent(f""" Member: {user.mention} (`{user.id}`) Actor: {ctx.message.author} - Intended expiry: {infraction_object['expires_at']} + Intended expiry: {infraction['expires_at']} """) ) except Exception: @@ -897,56 +952,59 @@ class Moderation(Scheduler): @with_role(*MODERATION_ROLES) @infraction_edit_group.command(name="duration") - async def edit_duration(self, ctx: Context, infraction_id: str, duration: str): + async def edit_duration( + self, ctx: Context, + infraction_id: int, expires_at: Union[ExpirationDate, str] + ): """ - Sets the duration of the given infraction, relative to the time of - updating. + Sets the duration of the given infraction, relative to the time of updating. - **`infraction_id`:** The ID (UUID) of the infraction. - **`duration`:** The new duration of the infraction, relative to the - time of updating. Use "permanent" to the infraction as permanent. + **`infraction_id`:** the id of the infraction + **`expires_at`:** the new expiration date of the infraction. + Use "permanent" to mark the infraction as permanent. """ - try: - previous = await self.bot.http_session.get( - URLs.site_infractions_by_id.format( - infraction_id=infraction_id - ), - headers=self.headers + if isinstance(expires_at, str) and expires_at != 'permanent': + raise BadArgument( + "If `expires_at` is given as a non-datetime, " + "it must be `permanent`." ) + if expires_at == 'permanent': + expires_at = None - previous_object = await previous.json() + try: + previous_infraction = await self.bot.api_client.get( + 'bot/infractions/' + str(infraction_id) + ) - if duration == "permanent": - duration = None # check the current active infraction - response = await self.bot.http_session.patch( - URLs.site_infractions, + infraction = await self.bot.api_client.patch( + 'bot/infractions/' + str(infraction_id), json={ - "id": infraction_id, - "duration": duration - }, - headers=self.headers + 'expires_at': ( + expires_at.isoformat() + if expires_at is not None + else None + ) + } ) - response_object = await response.json() - if "error_code" in response_object or response_object.get("success") is False: - return await ctx.send( - ":x: There was an error updating the infraction: " - f"{response_object['error_message']}" - ) - infraction_object = response_object["infraction"] # Re-schedule - self.cancel_task(infraction_id) + self.cancel_task(infraction['id']) loop = asyncio.get_event_loop() - self.schedule_task(loop, infraction_object["id"], infraction_object) + self.schedule_task(loop, infraction['id'], infraction) - if duration is None: + if expires_at is None: await ctx.send(f":ok_hand: Updated infraction: marked as permanent.") else: + human_expiry = ( + datetime + .fromisoformat(infraction['expires_at'][:-1]) + .strftime('%c') + ) await ctx.send( ":ok_hand: Updated infraction: set to expire on " - f"{infraction_object['expires_at']}." + f"{human_expiry}." ) except Exception: @@ -954,10 +1012,8 @@ class Moderation(Scheduler): await ctx.send(":x: There was an error updating the infraction.") return - prev_infraction = previous_object["infraction"] - # Get information about the infraction's user - user_id = int(infraction_object["user"]["user_id"]) + user_id = infraction["user"] user = ctx.guild.get_member(user_id) if user: @@ -968,7 +1024,7 @@ class Moderation(Scheduler): thumbnail = None # The infraction's actor - actor_id = int(infraction_object["actor"]["user_id"]) + actor_id = infraction["actor"] actor = ctx.guild.get_member(actor_id) or f"`{actor_id}`" await self.mod_log.send_log_message( @@ -980,55 +1036,37 @@ class Moderation(Scheduler): Member: {member_text} Actor: {actor} Edited by: {ctx.message.author} - Previous expiry: {prev_infraction['expires_at']} - New expiry: {infraction_object['expires_at']} + Previous expiry: {previous_infraction['expires_at']} + New expiry: {infraction['expires_at']} """) ) @with_role(*MODERATION_ROLES) @infraction_edit_group.command(name="reason") - async def edit_reason(self, ctx: Context, infraction_id: str, *, reason: str): + async def edit_reason(self, ctx: Context, infraction_id: int, *, reason: str): """ Sets the reason of the given infraction. - **`infraction_id`:** The ID (UUID) of the infraction. - **`reason`:** The new reason of the infraction. + **`infraction_id`:** the id of the infraction + **`reason`:** The new reason of the infraction """ try: - previous = await self.bot.http_session.get( - URLs.site_infractions_by_id.format( - infraction_id=infraction_id - ), - headers=self.headers + old_infraction = await self.bot.api_client.get( + 'bot/infractions/' + str(infraction_id) ) - previous_object = await previous.json() - - response = await self.bot.http_session.patch( - URLs.site_infractions, - json={ - "id": infraction_id, - "reason": reason - }, - headers=self.headers + updated_infraction = await self.bot.api_client.patch( + 'bot/infractions/' + str(infraction_id), + json={'reason': reason} ) - response_object = await response.json() - if "error_code" in response_object or response_object.get("success") is False: - return await ctx.send( - ":x: There was an error updating the infraction: " - f"{response_object['error_message']}" - ) - await ctx.send(f":ok_hand: Updated infraction: set reason to \"{reason}\".") + except Exception: log.exception("There was an error updating an infraction.") return await ctx.send(":x: There was an error updating the infraction.") - new_infraction = response_object["infraction"] - prev_infraction = previous_object["infraction"] - # Get information about the infraction's user - user_id = int(new_infraction["user"]["user_id"]) + user_id = updated_infraction['user'] user = ctx.guild.get_member(user_id) if user: @@ -1039,7 +1077,7 @@ class Moderation(Scheduler): thumbnail = None # The infraction's actor - actor_id = int(new_infraction["actor"]["user_id"]) + actor_id = updated_infraction['actor'] actor = ctx.guild.get_member(actor_id) or f"`{actor_id}`" await self.mod_log.send_log_message( @@ -1051,8 +1089,8 @@ class Moderation(Scheduler): Member: {user_text} Actor: {actor} Edited by: {ctx.message.author} - Previous reason: {prev_infraction['reason']} - New reason: {new_infraction['reason']} + Previous reason: {old_infraction['reason']} + New reason: {updated_infraction['reason']} """) ) @@ -1079,25 +1117,14 @@ class Moderation(Scheduler): Search for infractions by member. """ - try: - response = await self.bot.http_session.get( - URLs.site_infractions_user.format( - user_id=user.id - ), - params={"hidden": "True"}, - headers=self.headers - ) - infraction_list = await response.json() - except ClientError: - log.exception(f"Failed to fetch infractions for user {user} ({user.id}).") - await ctx.send(":x: An error occurred while fetching infractions.") - return - + infraction_list = await self.bot.api_client.get( + 'bot/infractions', + params={'user__id': str(user.id)} + ) embed = Embed( title=f"Infractions for {user} ({len(infraction_list)} total)", colour=Colour.orange() ) - await self.send_infraction_list(ctx, embed, infraction_list) @with_role(*MODERATION_ROLES) @@ -1107,23 +1134,13 @@ class Moderation(Scheduler): Search for infractions by their reason. Use Re2 for matching. """ - try: - response = await self.bot.http_session.get( - URLs.site_infractions, - params={"search": reason, "hidden": "True"}, - headers=self.headers - ) - infraction_list = await response.json() - except ClientError: - log.exception(f"Failed to fetch infractions matching reason `{reason}`.") - await ctx.send(":x: An error occurred while fetching infractions.") - return - + infraction_list = await self.bot.api_client.get( + 'bot/infractions', params={'search': reason} + ) embed = Embed( title=f"Infractions matching `{reason}` ({len(infraction_list)} total)", colour=Colour.orange() ) - await self.send_infraction_list(ctx, embed, infraction_list) # endregion @@ -1135,11 +1152,10 @@ class Moderation(Scheduler): await ctx.send(f":warning: No infractions could be found for that query.") return - lines = [] - for infraction in infractions: - lines.append( - self._infraction_to_string(infraction) - ) + lines = tuple( + self._infraction_to_string(infraction) + for infraction in infractions + ) await LinePaginator.paginate( lines, @@ -1195,7 +1211,7 @@ class Moderation(Scheduler): infraction_id = infraction_object["id"] # transform expiration to delay in seconds - expiration_datetime = parse_rfc1123(infraction_object["expires_at"]) + expiration_datetime = datetime.fromisoformat(infraction_object["expires_at"][:-1]) await wait_until(expiration_datetime) log.debug(f"Marking infraction {infraction_id} as inactive (expired).") @@ -1204,7 +1220,7 @@ class Moderation(Scheduler): self.cancel_task(infraction_object["id"]) # Notify the user that they've been unmuted. - user_id = int(infraction_object["user"]["user_id"]) + user_id = infraction_object["user"] guild = self.bot.get_guild(constants.Guild.id) await self.notify_pardon( user=guild.get_member(user_id), @@ -1215,14 +1231,14 @@ class Moderation(Scheduler): async def _deactivate_infraction(self, infraction_object): """ - A co-routine which marks an infraction as inactive on the website. This co-routine does - not cancel or un-schedule an expiration task. + A co-routine which marks an infraction as inactive on the website. + This co-routine does not cancel or un-schedule an expiration task. :param infraction_object: the infraction in question """ guild: Guild = self.bot.get_guild(constants.Guild.id) - user_id = int(infraction_object["user"]["user_id"]) + user_id = infraction_object["user"] infraction_type = infraction_object["type"] if infraction_type == "mute": @@ -1237,22 +1253,18 @@ class Moderation(Scheduler): user: Object = Object(user_id) await guild.unban(user) - await self.bot.http_session.patch( - URLs.site_infractions, - headers=self.headers, - json={ - "id": infraction_object["id"], - "active": False - } + await self.bot.api_client.patch( + 'bot/infractions/' + str(infraction_object['id']), + json={"active": False} ) def _infraction_to_string(self, infraction_object): - actor_id = int(infraction_object["actor"]["user_id"]) + actor_id = infraction_object["actor"] guild: Guild = self.bot.get_guild(constants.Guild.id) actor = guild.get_member(actor_id) - active = infraction_object["active"] is True - user_id = int(infraction_object["user"]["user_id"]) - hidden = infraction_object.get("hidden", False) is True + active = infraction_object["active"] + user_id = infraction_object["user"] + hidden = infraction_object["hidden"] lines = textwrap.dedent(f""" {"**===============**" if active else "==============="} @@ -1271,8 +1283,8 @@ class Moderation(Scheduler): return lines.strip() async def notify_infraction( - self, user: Union[User, Member], infr_type: str, duration: str = None, - reason: str = None + self, user: Union[User, Member], infr_type: str, + expires_at: Union[datetime, str] = 'N/A', reason: str = "No reason provided." ): """ Notify a user of their fresh infraction :) @@ -1283,16 +1295,13 @@ class Moderation(Scheduler): :param reason: The reason for the infraction. """ - if duration is None: - duration = "N/A" - - if reason is None: - reason = "No reason provided." + if isinstance(expires_at, datetime): + expires_at = expires_at.strftime('%c') embed = Embed( description=textwrap.dedent(f""" **Type:** {infr_type} - **Duration:** {duration} + **Expires:** {expires_at} **Reason:** {reason} """), colour=Colour(Colours.soft_red) diff --git a/bot/cogs/modlog.py b/bot/cogs/modlog.py index b3094321e..9f0c88424 100644 --- a/bot/cogs/modlog.py +++ b/bot/cogs/modlog.py @@ -1,9 +1,8 @@ import asyncio -import datetime import logging +from datetime import datetime from typing import List, Optional, Union -from aiohttp import ClientResponseError from dateutil.relativedelta import relativedelta from deepdiff import DeepDiff from discord import ( @@ -15,9 +14,7 @@ from discord.abc import GuildChannel from discord.ext.commands import Bot from bot.constants import ( - Channels, Colours, Emojis, - Event, Guild as GuildConstant, Icons, - Keys, Roles, URLs + Channels, Colours, Emojis, Event, Guild as GuildConstant, Icons, URLs ) from bot.utils.time import humanize_delta @@ -38,13 +35,12 @@ class ModLog: def __init__(self, bot: Bot): self.bot = bot - self.headers = {"X-API-KEY": Keys.site_api} self._ignored = {event: [] for event in Event} self._cached_deletes = [] self._cached_edits = [] - async def upload_log(self, messages: List[Message]) -> Optional[str]: + async def upload_log(self, messages: List[Message], actor_id: int) -> Optional[str]: """ Uploads the log data to the database via an API endpoint for uploading logs. @@ -54,48 +50,25 @@ class ModLog: Returns a URL that can be used to view the log. """ - log_data = [] - - for message in messages: - author = f"{message.author.name}#{message.author.discriminator}" - - # message.author may return either a User or a Member. Users don't have roles. - if type(message.author) is User: - role_id = Roles.developer - else: - role_id = message.author.top_role.id - - content = message.content - embeds = [embed.to_dict() for embed in message.embeds] - attachments = ["<Attachment>" for _ in message.attachments] - - log_data.append({ - "content": content, - "author": author, - "user_id": str(message.author.id), - "role_id": str(role_id), - "timestamp": message.created_at.strftime("%D %H:%M"), - "attachments": attachments, - "embeds": embeds, - }) - - response = await self.bot.http_session.post( - URLs.site_logs_api, - headers=self.headers, - json={"log_data": log_data} + response = await self.bot.api_client.post( + 'bot/deleted-messages', + json={ + 'actor': actor_id, + 'creation': datetime.utcnow().isoformat(), + 'deletedmessage_set': [ + { + 'id': message.id, + 'author': message.author.id, + 'channel_id': message.channel.id, + 'content': message.content, + 'embeds': [embed.to_dict() for embed in message.embeds] + } + for message in messages + ] + } ) - try: - data = await response.json() - log_id = data["log_id"] - except (KeyError, ClientResponseError): - log.debug( - "API returned an unexpected result:\n" - f"{response.text}" - ) - return - - return f"{URLs.site_logs_view}/{log_id}" + return f"{URLs.site_logs_view}/{response['id']}" def ignore(self, event: Event, *items: int): for item in items: @@ -115,7 +88,7 @@ class ModLog: content: Optional[str] = None, additional_embeds: Optional[List[Embed]] = None, additional_embeds_msg: Optional[str] = None, - timestamp_override: Optional[datetime.datetime] = None, + timestamp_override: Optional[datetime] = None, footer: Optional[str] = None, ): embed = Embed(description=text) @@ -124,8 +97,7 @@ class ModLog: embed.set_author(name=title, icon_url=icon_url) embed.colour = colour - - embed.timestamp = timestamp_override or datetime.datetime.utcnow() + embed.timestamp = timestamp_override or datetime.utcnow() if footer: embed.set_footer(text=footer) @@ -391,7 +363,7 @@ class ModLog: return message = f"{member.name}#{member.discriminator} (`{member.id}`)" - now = datetime.datetime.utcnow() + now = datetime.utcnow() difference = abs(relativedelta(now, member.created_at)) message += "\n\n**Account age:** " + humanize_delta(difference) diff --git a/bot/cogs/off_topic_names.py b/bot/cogs/off_topic_names.py index 9b0f5d6c5..c0d2e5dc5 100644 --- a/bot/cogs/off_topic_names.py +++ b/bot/cogs/off_topic_names.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta from discord import Colour, Embed from discord.ext.commands import BadArgument, Bot, Context, Converter, group -from bot.constants import Channels, Keys, MODERATION_ROLES, URLs +from bot.constants import Channels, Keys, MODERATION_ROLES from bot.decorators import with_role from bot.pagination import LinePaginator @@ -44,7 +44,7 @@ async def update_names(bot: Bot, headers: dict): Args: bot (Bot): The running bot instance, used for fetching data from the - website via the bot's `http_session`. + website via the bot's `api_client`. """ while True: @@ -55,11 +55,9 @@ async def update_names(bot: Bot, headers: dict): seconds_to_sleep = (next_midnight - datetime.utcnow()).seconds + 1 await asyncio.sleep(seconds_to_sleep) - response = await bot.http_session.get( - f'{URLs.site_off_topic_names_api}?random_items=3', - headers=headers + channel_0_name, channel_1_name, channel_2_name = await bot.api_client.get( + 'bot/off-topic-channel-names', params={'random_items': 3} ) - channel_0_name, channel_1_name, channel_2_name = await response.json() channel_0, channel_1, channel_2 = (bot.get_channel(channel_id) for channel_id in CHANNELS) await channel_0.edit(name=f'ot0-{channel_0_name}') @@ -100,49 +98,24 @@ class OffTopicNames: async def add_command(self, ctx, name: OffTopicName): """Adds a new off-topic name to the rotation.""" - result = await self.bot.http_session.post( - URLs.site_off_topic_names_api, - headers=self.headers, - params={'name': name} + await self.bot.api_client.post(f'bot/off-topic-channel-names', params={'name': name}) + log.info( + f"{ctx.author.name}#{ctx.author.discriminator}" + f" added the off-topic channel name '{name}" ) - - response = await result.json() - - if result.status == 200: - log.info( - f"{ctx.author.name}#{ctx.author.discriminator}" - f" added the off-topic channel name '{name}" - ) - await ctx.send(":ok_hand:") - else: - error_reason = response.get('message', "No reason provided.") - await ctx.send(f":warning: got non-200 from the API: {error_reason}") + await ctx.send(":ok_hand:") @otname_group.command(name='delete', aliases=('remove', 'rm', 'del', 'd')) @with_role(*MODERATION_ROLES) async def delete_command(self, ctx, name: OffTopicName): """Removes a off-topic name from the rotation.""" - result = await self.bot.http_session.delete( - URLs.site_off_topic_names_api, - headers=self.headers, - params={'name': name} + await self.bot.api_client.delete(f'bot/off-topic-channel-names/{name}') + log.info( + f"{ctx.author.name}#{ctx.author.discriminator}" + f" deleted the off-topic channel name '{name}" ) - - response = await result.json() - - if result.status == 200: - if response['deleted'] == 0: - await ctx.send(f":warning: No name matching `{name}` was found in the database.") - else: - log.info( - f"{ctx.author.name}#{ctx.author.discriminator}" - f" deleted the off-topic channel name '{name}" - ) - await ctx.send(":ok_hand:") - else: - error_reason = response.get('message', "No reason provided.") - await ctx.send(f":warning: got non-200 from the API: {error_reason}") + await ctx.send(":ok_hand:") @otname_group.command(name='list', aliases=('l',)) @with_role(*MODERATION_ROLES) @@ -152,18 +125,17 @@ class OffTopicNames: Restricted to Moderator and above to not spoil the surprise. """ - result = await self.bot.http_session.get( - URLs.site_off_topic_names_api, - headers=self.headers - ) - response = await result.json() - lines = sorted(f"• {name}" for name in response) - + result = await self.bot.api_client.get('bot/off-topic-channel-names') + lines = sorted(f"• {name}" for name in result) embed = Embed( - title=f"Known off-topic names (`{len(response)}` total)", + title=f"Known off-topic names (`{len(result)}` total)", colour=Colour.blue() ) - await LinePaginator.paginate(lines, ctx, embed, max_size=400, empty=False) + if result: + await LinePaginator.paginate(lines, ctx, embed, max_size=400, empty=False) + else: + embed.description = "Hmmm, seems like there's nothing here yet." + await ctx.send(embed=embed) def setup(bot: Bot): diff --git a/bot/cogs/reminders.py b/bot/cogs/reminders.py index e8177107b..03ea00de8 100644 --- a/bot/cogs/reminders.py +++ b/bot/cogs/reminders.py @@ -1,22 +1,20 @@ import asyncio -import datetime import logging import random import textwrap +from datetime import datetime +from operator import itemgetter -from aiohttp import ClientResponseError from dateutil.relativedelta import relativedelta from discord import Colour, Embed from discord.ext.commands import Bot, Context, group -from bot.constants import ( - Channels, Icons, Keys, NEGATIVE_REPLIES, - POSITIVE_REPLIES, STAFF_ROLES, URLs -) +from bot.constants import Channels, Icons, NEGATIVE_REPLIES, POSITIVE_REPLIES, STAFF_ROLES +from bot.converters import ExpirationDate from bot.pagination import LinePaginator from bot.utils.checks import without_role_check from bot.utils.scheduling import Scheduler -from bot.utils.time import humanize_delta, parse_rfc1123, wait_until +from bot.utils.time import humanize_delta, wait_until log = logging.getLogger(__name__) @@ -28,24 +26,20 @@ class Reminders(Scheduler): def __init__(self, bot: Bot): self.bot = bot - self.headers = {"X-API-Key": Keys.site_api} super().__init__() async def on_ready(self): # Get all the current reminders for re-scheduling - response = await self.bot.http_session.get( - url=URLs.site_reminders_api, - headers=self.headers + response = await self.bot.api_client.get( + 'bot/reminders', + params={'active': 'true'} ) - response_data = await response.json() - - # Find the current time, timezone-aware. - now = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc) + now = datetime.utcnow() loop = asyncio.get_event_loop() - for reminder in response_data["reminders"]: - remind_at = parse_rfc1123(reminder["remind_at"]) + for reminder in response: + remind_at = datetime.fromisoformat(reminder['expiration'][:-1]) # If the reminder is already overdue ... if remind_at < now: @@ -56,32 +50,16 @@ class Reminders(Scheduler): self.schedule_task(loop, reminder["id"], reminder) @staticmethod - async def _send_confirmation(ctx: Context, response: dict, on_success: str): + async def _send_confirmation(ctx: Context, on_success: str): """ - Send an embed confirming whether or not a change was made successfully. - - :return: A Boolean value indicating whether it failed (True) or passed (False) + Send an embed confirming the change was made successfully. """ embed = Embed() - - if not response.get("success"): - embed.colour = Colour.red() - embed.title = random.choice(NEGATIVE_REPLIES) - embed.description = response.get("error_message", "An unexpected error occurred.") - - log.warn(f"Unable to create/edit/delete a reminder. Response: {response}") - failed = True - - else: - embed.colour = Colour.green() - embed.title = random.choice(POSITIVE_REPLIES) - embed.description = on_success - - failed = False - + embed.colour = Colour.green() + embed.title = random.choice(POSITIVE_REPLIES) + embed.description = on_success await ctx.send(embed=embed) - return failed async def _scheduled_task(self, reminder: dict): """ @@ -92,7 +70,7 @@ class Reminders(Scheduler): """ reminder_id = reminder["id"] - reminder_datetime = parse_rfc1123(reminder["remind_at"]) + reminder_datetime = datetime.fromisoformat(reminder['expiration'][:-1]) # Send the reminder message once the desired duration has passed await wait_until(reminder_datetime) @@ -111,18 +89,7 @@ class Reminders(Scheduler): :param reminder_id: The ID of the reminder. """ - # The API requires a list, so let's give it one :) - json_data = { - "reminders": [ - reminder_id - ] - } - - await self.bot.http_session.delete( - url=URLs.site_reminders_api, - headers=self.headers, - json=json_data - ) + await self.bot.api_client.delete('bot/reminders/' + str(reminder_id)) # Now we can remove it from the schedule list self.cancel_task(reminder_id) @@ -147,8 +114,8 @@ class Reminders(Scheduler): :param late: How late the reminder is (if at all) """ - channel = self.bot.get_channel(int(reminder["channel_id"])) - user = self.bot.get_user(int(reminder["user_id"])) + channel = self.bot.get_channel(reminder["channel_id"]) + user = self.bot.get_user(reminder["author"]) embed = Embed() embed.colour = Colour.blurple() @@ -173,15 +140,15 @@ class Reminders(Scheduler): await self._delete_reminder(reminder["id"]) @group(name="remind", aliases=("reminder", "reminders"), invoke_without_command=True) - async def remind_group(self, ctx: Context, duration: str, *, content: str): + async def remind_group(self, ctx: Context, expiration: ExpirationDate, *, content: str): """ Commands for managing your reminders. """ - await ctx.invoke(self.new_reminder, duration=duration, content=content) + await ctx.invoke(self.new_reminder, expiration=expiration, content=content) @remind_group.command(name="new", aliases=("add", "create")) - async def new_reminder(self, ctx: Context, duration: str, *, content: str): + async def new_reminder(self, ctx: Context, expiration: ExpirationDate, *, content: str): """ Set yourself a simple reminder. """ @@ -200,13 +167,13 @@ class Reminders(Scheduler): return await ctx.send(embed=embed) # Get their current active reminders - response = await self.bot.http_session.get( - url=URLs.site_reminders_user_api.format(user_id=ctx.author.id), - headers=self.headers + active_reminders = await self.bot.api_client.get( + 'bot/reminders', + params={ + 'user__id': str(ctx.author.id) + } ) - active_reminders = await response.json() - # Let's limit this, so we don't get 10 000 # reminders from kip or something like that :P if len(active_reminders) > MAXIMUM_REMINDERS: @@ -217,45 +184,23 @@ class Reminders(Scheduler): return await ctx.send(embed=embed) # Now we can attempt to actually set the reminder. - try: - response = await self.bot.http_session.post( - url=URLs.site_reminders_api, - headers=self.headers, - json={ - "user_id": str(ctx.author.id), - "duration": duration, - "content": content, - "channel_id": str(ctx.channel.id) - } - ) - - response_data = await response.json() - - # AFAIK only happens if the user enters, like, a quintillion weeks - except ClientResponseError: - embed.colour = Colour.red() - embed.title = random.choice(NEGATIVE_REPLIES) - embed.description = ( - "An error occurred while adding your reminder to the database. " - "Did you enter a reasonable duration?" - ) - - log.warn(f"User {ctx.author} attempted to create a reminder for {duration}, but failed.") - - return await ctx.send(embed=embed) - - # Confirm to the user whether or not it worked. - failed = await self._send_confirmation( - ctx, response_data, - on_success="Your reminder has been created successfully!" + reminder = await self.bot.api_client.post( + 'bot/reminders', + json={ + 'author': ctx.author.id, + 'channel_id': ctx.message.channel.id, + 'content': content, + 'expiration': expiration.isoformat() + } ) - # If it worked, schedule the reminder. - if not failed: - loop = asyncio.get_event_loop() - reminder = response_data["reminder"] + # Confirm to the user that it worked. + await self._send_confirmation( + ctx, on_success="Your reminder has been created successfully!" + ) - self.schedule_task(loop, reminder["id"], reminder) + loop = asyncio.get_event_loop() + self.schedule_task(loop, reminder["id"], reminder) @remind_group.command(name="list") async def list_reminders(self, ctx: Context): @@ -264,31 +209,31 @@ class Reminders(Scheduler): """ # Get all the user's reminders from the database. - response = await self.bot.http_session.get( - url=URLs.site_reminders_user_api, - params={"user_id": str(ctx.author.id)}, - headers=self.headers + data = await self.bot.api_client.get( + 'bot/reminders', + params={'user__id': str(ctx.author.id)} ) - data = await response.json() - now = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc) + now = datetime.utcnow() # Make a list of tuples so it can be sorted by time. - reminders = [ - (rem["content"], rem["remind_at"], rem["friendly_id"]) for rem in data["reminders"] - ] - - reminders.sort(key=lambda rem: rem[1]) + reminders = sorted( + ( + (rem['content'], rem['expiration'], rem['id']) + for rem in data + ), + key=itemgetter(1) + ) lines = [] - for index, (content, remind_at, friendly_id) in enumerate(reminders): + for content, remind_at, id_ in reminders: # Parse and humanize the time, make it pretty :D - remind_datetime = parse_rfc1123(remind_at) + remind_datetime = datetime.fromisoformat(remind_at[:-1]) time = humanize_delta(relativedelta(remind_datetime, now)) text = textwrap.dedent(f""" - **Reminder #{index}:** *expires in {time}* (ID: {friendly_id}) + **Reminder #{id_}:** *expires in {time}* (ID: {id_}) {content} """).strip() @@ -322,84 +267,53 @@ class Reminders(Scheduler): await ctx.invoke(self.bot.get_command("help"), "reminders", "edit") @edit_reminder_group.command(name="duration", aliases=("time",)) - async def edit_reminder_duration(self, ctx: Context, friendly_id: str, duration: str): + async def edit_reminder_duration(self, ctx: Context, id_: int, expiration: ExpirationDate): """ - Edit one of your reminders' duration. + Edit one of your reminders' expiration. """ # Send the request to update the reminder in the database - response = await self.bot.http_session.patch( - url=URLs.site_reminders_user_api, - headers=self.headers, - json={ - "user_id": str(ctx.author.id), - "friendly_id": friendly_id, - "duration": duration - } + reminder = await self.bot.api_client.patch( + 'bot/reminders/' + str(id_), + json={'expiration': expiration.isoformat()} ) # Send a confirmation message to the channel - response_data = await response.json() - failed = await self._send_confirmation( - ctx, response_data, - on_success="That reminder has been edited successfully!" + await self._send_confirmation( + ctx, on_success="That reminder has been edited successfully!" ) - if not failed: - await self._reschedule_reminder(response_data["reminder"]) + await self._reschedule_reminder(reminder) @edit_reminder_group.command(name="content", aliases=("reason",)) - async def edit_reminder_content(self, ctx: Context, friendly_id: str, *, content: str): + async def edit_reminder_content(self, ctx: Context, id_: int, *, content: str): """ Edit one of your reminders' content. """ # Send the request to update the reminder in the database - response = await self.bot.http_session.patch( - url=URLs.site_reminders_user_api, - headers=self.headers, - json={ - "user_id": str(ctx.author.id), - "friendly_id": friendly_id, - "content": content - } + reminder = await self.bot.api_client.patch( + 'bot/reminders/' + str(id_), + json={'content': content} ) # Send a confirmation message to the channel - response_data = await response.json() - failed = await self._send_confirmation( - ctx, response_data, - on_success="That reminder has been edited successfully!" + await self._send_confirmation( + ctx, on_success="That reminder has been edited successfully!" ) - - if not failed: - await self._reschedule_reminder(response_data["reminder"]) + await self._reschedule_reminder(reminder) @remind_group.command("delete", aliases=("remove",)) - async def delete_reminder(self, ctx: Context, friendly_id: str): + async def delete_reminder(self, ctx: Context, id_: int): """ Delete one of your active reminders. """ - # Send the request to delete the reminder from the database - response = await self.bot.http_session.delete( - url=URLs.site_reminders_user_api, - headers=self.headers, - json={ - "user_id": str(ctx.author.id), - "friendly_id": friendly_id - } - ) - - response_data = await response.json() - failed = await self._send_confirmation( - ctx, response_data, - on_success="That reminder has been deleted successfully!" + await self._delete_reminder(id_) + await self._send_confirmation( + ctx, on_success="That reminder has been deleted successfully!" ) - if not failed: - await self._delete_reminder(response_data["reminder_id"]) - def setup(bot: Bot): bot.add_cog(Reminders(bot)) diff --git a/bot/cogs/rmq.py b/bot/cogs/rmq.py deleted file mode 100644 index 585eacc25..000000000 --- a/bot/cogs/rmq.py +++ /dev/null @@ -1,229 +0,0 @@ -import asyncio -import datetime -import json -import logging -import pprint - -import aio_pika -from aio_pika import Message -from dateutil import parser as date_parser -from discord import Colour, Embed -from discord.ext.commands import Bot -from discord.utils import get - -from bot.constants import Channels, Guild, RabbitMQ - -log = logging.getLogger(__name__) - -LEVEL_COLOURS = { - "debug": Colour.blue(), - "info": Colour.green(), - "warning": Colour.gold(), - "error": Colour.red() -} - -DEFAULT_LEVEL_COLOUR = Colour.greyple() -EMBED_PARAMS = ( - "colour", "title", "url", "description", "timestamp" -) - -CONSUME_TIMEOUT = datetime.timedelta(seconds=10) - - -class RMQ: - """ - RabbitMQ event handling - """ - - rmq = None # type: aio_pika.Connection - channel = None # type: aio_pika.Channel - queue = None # type: aio_pika.Queue - - def __init__(self, bot: Bot): - self.bot = bot - - async def on_ready(self): - self.rmq = await aio_pika.connect_robust( - host=RabbitMQ.host, port=RabbitMQ.port, login=RabbitMQ.username, password=RabbitMQ.password - ) - - log.info("Connected to RabbitMQ") - - self.channel = await self.rmq.channel() - self.queue = await self.channel.declare_queue("bot_events", durable=True) - - log.debug("Channel opened, queue declared") - - async for message in self.queue: - with message.process(): - message.ack() - await self.handle_message(message, message.body.decode()) - - async def send_text(self, queue: str, data: str): - message = Message(data.encode("utf-8")) - await self.channel.default_exchange.publish(message, queue) - - async def send_json(self, queue: str, **data): - message = Message(json.dumps(data).encode("utf-8")) - await self.channel.default_exchange.publish(message, queue) - - async def consume(self, queue: str, **kwargs): - queue_obj = await self.channel.declare_queue(queue, **kwargs) - - result = None - start_time = datetime.datetime.now() - - while result is None: - if datetime.datetime.now() - start_time >= CONSUME_TIMEOUT: - result = "Timed out while waiting for a response." - else: - result = await queue_obj.get(timeout=5, fail=False) - await asyncio.sleep(0.5) - - if result: - result.ack() - - return result - - async def handle_message(self, message, data): - log.debug(f"Message: {message}") - log.debug(f"Data: {data}") - - try: - data = json.loads(data) - except Exception: - await self.do_mod_log("error", "Unable to parse event", data) - else: - event = data["event"] - event_data = data["data"] - - try: - func = getattr(self, f"do_{event}") - await func(**event_data) - except Exception as e: - await self.do_mod_log( - "error", f"Unable to handle event: {event}", - str(e) - ) - - async def do_mod_log(self, level: str, title: str, message: str): - colour = LEVEL_COLOURS.get(level, DEFAULT_LEVEL_COLOUR) - embed = Embed( - title=title, description=f"```\n{message}\n```", - colour=colour, timestamp=datetime.datetime.utcnow() - ) - - await self.bot.get_channel(Channels.devlog).send(embed=embed) - log.log(logging._nameToLevel[level.upper()], f"Modlog: {title} | {message}") - - async def do_send_message(self, target: int, message: str): - channel = self.bot.get_channel(target) - - if channel is None: - await self.do_mod_log( - "error", "Failed: Send Message", - f"Unable to find channel: {target}" - ) - else: - await channel.send(message) - - await self.do_mod_log( - "info", "Succeeded: Send Embed", - f"Message sent to channel {target}\n\n{message}" - ) - - async def do_send_embed(self, target: int, **embed_params): - for param, value in list(embed_params.items()): # To keep a full copy - if param not in EMBED_PARAMS: - await self.do_mod_log( - "warning", "Warning: Send Embed", - f"Unknown embed parameter: {param}" - ) - del embed_params[param] - - if param == "timestamp": - embed_params[param] = date_parser.parse(value) - elif param == "colour": - embed_params[param] = Colour(value) - - channel = self.bot.get_channel(target) - - if channel is None: - await self.do_mod_log( - "error", "Failed: Send Embed", - f"Unable to find channel: {target}" - ) - else: - await channel.send(embed=Embed(**embed_params)) - - await self.do_mod_log( - "info", "Succeeded: Send Embed", - f"Embed sent to channel {target}\n\n{pprint.pformat(embed_params, 4)}" - ) - - async def do_add_role(self, target: int, role_id: int, reason: str): - guild = self.bot.get_guild(Guild.id) - member = guild.get_member(int(target)) - - if member is None: - return await self.do_mod_log( - "error", "Failed: Add Role", - f"Unable to find member: {target}" - ) - - role = get(guild.roles, id=int(role_id)) - - if role is None: - return await self.do_mod_log( - "error", "Failed: Add Role", - f"Unable to find role: {role_id}" - ) - - try: - await member.add_roles(role, reason=reason) - except Exception as e: - await self.do_mod_log( - "error", "Failed: Add Role", - f"Error while adding role {role.name}: {e}" - ) - else: - await self.do_mod_log( - "info", "Succeeded: Add Role", - f"Role {role.name} added to member {target}" - ) - - async def do_remove_role(self, target: int, role_id: int, reason: str): - guild = self.bot.get_guild(Guild.id) - member = guild.get_member(int(target)) - - if member is None: - return await self.do_mod_log( - "error", "Failed: Remove Role", - f"Unable to find member: {target}" - ) - - role = get(guild.roles, id=int(role_id)) - - if role is None: - return await self.do_mod_log( - "error", "Failed: Remove Role", - f"Unable to find role: {role_id}" - ) - - try: - await member.remove_roles(role, reason=reason) - except Exception as e: - await self.do_mod_log( - "error", "Failed: Remove Role", - f"Error while adding role {role.name}: {e}" - ) - else: - await self.do_mod_log( - "info", "Succeeded: Remove Role", - f"Role {role.name} removed from member {target}" - ) - - -def setup(bot): - bot.add_cog(RMQ(bot)) - log.info("Cog loaded: RMQ") diff --git a/bot/cogs/rules.py b/bot/cogs/rules.py deleted file mode 100644 index b8a26ff76..000000000 --- a/bot/cogs/rules.py +++ /dev/null @@ -1,104 +0,0 @@ -import re -from typing import Optional - -from discord import Colour, Embed -from discord.ext.commands import Bot, Context, command - -from bot.constants import Channels, STAFF_ROLES -from bot.decorators import redirect_output -from bot.pagination import LinePaginator - - -class Rules: - - def __init__(self, bot: Bot): - self.bot = bot - - # We'll get the rules from the API when the - # site has been updated to the Django Framework. - # Hard-code the rules for now until the new RulesView is released. - - self.rules = ( - "Be polite, and do not spam.", - - "Follow the [Discord Community Guidelines](https://discordapp.com/guidelines).", - - "Don't intentionally make other people uncomfortable - if someone asks you to stop " - "discussing something, you should stop.", - - "Be patient both with users asking questions, and the users answering them.", - - "We will not help you with anything that might break a law or the terms of service " - "of any other community, site, service, or otherwise - No piracy, brute-forcing, " - "captcha circumvention, sneaker bots, or anything else of that nature.", - - "Listen to and respect the staff members - we're here to help, but we're all human " - "beings.", - - "All discussion should be kept within the relevant channels for the subject - See the " - "[channels page](https://pythondiscord.com/about/channels) for more information.", - - "This is an English-speaking server, so please speak English to the best of your " - "ability - [Google Translate](https://translate.google.com/) should be fine if you're " - "not sure.", - - "Keep all discussions safe for work - No gore, nudity, sexual soliciting, references " - "to suicide, or anything else of that nature", - - "We do not allow advertisements for communities (including other Discord servers) or " - "commercial projects - Contact us directly if you want to discuss a partnership!" - ) - self.default_desc = ("The rules and guidelines that apply to this community can be found on" - " our [rules page](https://pythondiscord.com/about/rules). We expect" - " all members of the community to have read and understood these." - ) - self.title_link = 'https://pythondiscord.com/about/rules' - - @command(aliases=['r', 'rule'], name='rules') - @redirect_output(destination_channel=Channels.bot, bypass_roles=STAFF_ROLES) - async def rules_command(self, ctx: Context, *, rules: Optional[str] = None): - """ - Provides a link to the `rules` endpoint of the website, or displays - specific rules, if they are requested. - - **`ctx`:** The Discord message context - **`rules`:** The rules a user wants to get. - """ - rules_embed = Embed(title='Rules', color=Colour.blurple()) - - if not rules: - # Rules were not submitted. Return the default description. - rules_embed.description = self.default_desc - rules_embed.url = 'https://pythondiscord.com/about/rules' - return await ctx.send(embed=rules_embed) - - # Split the rules input by slash, comma or space - # Returns a list of ints if they're in range of rules index - rules_to_get = [] - split_rules = re.split(r'[/, ]', rules) - for item in split_rules: - if not item.isdigit(): - if not item: - continue - rule_match = re.search(r'\d?\d[:|-]1?\d', item) - if rule_match: - a, b = sorted([int(x)-1 for x in re.split(r'[:-]', rule_match.group())]) - rules_to_get.extend(range(a, b+1)) - else: - rules_to_get.append(int(item)-1) - final_rules = [ - f'**{i+1}.** {self.rules[i]}' for i in sorted(rules_to_get) if i < len(self.rules) - ] - - if not final_rules: - # No valid rules in rules input. Return the default description. - rules_embed.description = self.default_desc - return await ctx.send(embed=rules_embed) - await LinePaginator.paginate( - final_rules, ctx, rules_embed, - max_lines=3, url=self.title_link - ) - - -def setup(bot): - bot.add_cog(Rules(bot)) diff --git a/bot/cogs/site.py b/bot/cogs/site.py index e5fd645fb..c22beafd1 100644 --- a/bot/cogs/site.py +++ b/bot/cogs/site.py @@ -1,9 +1,12 @@ +import gettext import logging from discord import Colour, Embed from discord.ext.commands import Bot, Context, group -from bot.constants import URLs +from bot.constants import Channels, STAFF_ROLES, URLs +from bot.decorators import redirect_output +from bot.pagination import LinePaginator log = logging.getLogger(__name__) @@ -92,6 +95,46 @@ class Site: await ctx.send(embed=embed) + @site_group.command(aliases=['r', 'rule'], name='rules') + @redirect_output(destination_channel=Channels.bot, bypass_roles=STAFF_ROLES) + async def site_rules(self, ctx: Context, *rules: int): + """ + Provides a link to the `rules` endpoint of the website, or displays + specific rules, if they are requested. + + **`ctx`:** The Discord message context + **`rules`:** The rules a user wants to get. + """ + rules_embed = Embed(title='Rules', color=Colour.blurple()) + rules_embed.url = f"{URLs.site_schema}{URLs.site}/about/rules" + + if not rules: + # Rules were not submitted. Return the default description. + rules_embed.description = ( + "The rules and guidelines that apply to this community can be found on" + " our [rules page](https://pythondiscord.com/about/rules). We expect" + " all members of the community to have read and understood these." + ) + + await ctx.send(embed=rules_embed) + return + + full_rules = await self.bot.api_client.get('rules', params={'link_format': 'md'}) + invalid_indices = tuple( + pick + for pick in rules + if pick < 0 or pick >= len(full_rules) + ) + + if invalid_indices: + indices = ', '.join(map(str, invalid_indices)) + await ctx.send(f":x: Invalid rule indices {indices}") + return + + final_rules = tuple(f"**{pick}.** {full_rules[pick]}" for pick in rules) + + await LinePaginator.paginate(final_rules, ctx, rules_embed, max_lines=3) + def setup(bot): bot.add_cog(Site(bot)) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index cc18c0041..05834e421 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -3,13 +3,14 @@ import logging import random import re import textwrap +from signal import Signals +from typing import Optional, Tuple from discord import Colour, Embed from discord.ext.commands import ( Bot, CommandError, Context, NoPrivateMessage, command, guild_only ) -from bot.cogs.rmq import RMQ from bot.constants import Channels, ERROR_REPLIES, NEGATIVE_REPLIES, STAFF_ROLES, URLs from bot.decorators import InChannelCheckFailure, in_channel from bot.utils.messages import wait_for_deletion @@ -17,23 +18,6 @@ from bot.utils.messages import wait_for_deletion log = logging.getLogger(__name__) -RMQ_ARGS = { - "durable": False, - "arguments": {"x-message-ttl": 5000}, - "auto_delete": True -} - -CODE_TEMPLATE = """ -venv_file = "/snekbox/.venv/bin/activate_this.py" -exec(open(venv_file).read(), dict(__file__=venv_file)) - -try: -{CODE} -except: - import traceback - print(traceback.format_exc()) -""" - ESCAPE_REGEX = re.compile("[`\u202E\u200B]{3,}") FORMATTED_CODE_REGEX = re.compile( r"^\s*" # any leading whitespace from the beginning of the string @@ -53,42 +37,47 @@ RAW_CODE_REGEX = re.compile( re.DOTALL # "." also matches newlines ) +MAX_PASTE_LEN = 1000 + class Snekbox: """ - Safe evaluation using Snekbox + Safe evaluation of Python code using Snekbox """ def __init__(self, bot: Bot): self.bot = bot self.jobs = {} - @property - def rmq(self) -> RMQ: - return self.bot.get_cog("RMQ") - - @command(name='eval', aliases=('e',)) - @guild_only() - @in_channel(Channels.bot, bypass_roles=STAFF_ROLES) - async def eval_command(self, ctx: Context, *, code: str = None): - """ - Run some code. get the result back. We've done our best to make this safe, but do let us know if you - manage to find an issue with it! - - This command supports multiple lines of code, including code wrapped inside a formatted code block. - """ + async def post_eval(self, code: str) -> dict: + """Send a POST request to the Snekbox API to evaluate code and return the results.""" + url = URLs.snekbox_eval_api + data = {"input": code} + async with self.bot.http_session.post(url, json=data, raise_for_status=True) as resp: + return await resp.json() - if ctx.author.id in self.jobs: - await ctx.send(f"{ctx.author.mention} You've already got a job running - please wait for it to finish!") - return - - if not code: # None or empty string - return await ctx.invoke(self.bot.get_command("help"), "eval") + async def upload_output(self, output: str) -> Optional[str]: + """Upload the eval output to a paste service and return a URL to it if successful.""" + log.trace("Uploading full output to paste service...") - log.info(f"Received code from {ctx.author.name}#{ctx.author.discriminator} for evaluation:\n{code}") - self.jobs[ctx.author.id] = datetime.datetime.now() + if len(output) > MAX_PASTE_LEN: + log.info("Full output is too long to upload") + return "too long to upload" - # Strip whitespace and inline or block code markdown and extract the code and some formatting info + url = URLs.paste_service.format(key="documents") + try: + async with self.bot.http_session.post(url, data=output, raise_for_status=True) as resp: + data = await resp.json() + + if "key" in data: + return URLs.paste_service.format(key=data["key"]) + except Exception: + # 400 (Bad Request) means there are too many characters + log.exception("Failed to upload full output to paste service!") + + @staticmethod + def prepare_input(code: str) -> str: + """Extract code from the Markdown, format it, and insert it into the code template.""" match = FORMATTED_CODE_REGEX.fullmatch(code) if match: code, block, lang, delim = match.group("code", "block", "lang", "delim") @@ -100,87 +89,138 @@ class Snekbox: log.trace(f"Extracted {info} for evaluation:\n{code}") else: code = textwrap.dedent(RAW_CODE_REGEX.fullmatch(code).group("code")) - log.trace(f"Eval message contains not or badly formatted code, stripping whitespace only:\n{code}") + log.trace( + f"Eval message contains unformatted or badly formatted code, " + f"stripping whitespace only:\n{code}" + ) - try: - stripped_lines = [ln.strip() for ln in code.split('\n')] - if all(line.startswith('#') for line in stripped_lines): - return await ctx.send( - f"{ctx.author.mention} Your eval job has completed.\n\n```[No output]```" - ) + return code + + @staticmethod + def get_results_message(results: dict) -> Tuple[str, str]: + """Return a user-friendly message and error corresponding to the process's return code.""" + stdout, returncode = results["stdout"], results["returncode"] + msg = f"Your eval job has completed with return code {returncode}" + error = "" + + if returncode is None: + msg = "Your eval job has failed" + error = stdout.strip() + elif returncode == 128 + Signals.SIGKILL: + msg = "Your eval job timed out or ran out of memory" + elif returncode == 255: + msg = "Your eval job has failed" + error = "A fatal NsJail error occurred" + else: + # Try to append signal's name if one exists + try: + name = Signals(returncode - 128).name + msg = f"{msg} ({name})" + except ValueError: + pass + + return msg, error + + async def format_output(self, output: str) -> Tuple[str, Optional[str]]: + """ + Format the output and return a tuple of the formatted output and a URL to the full output. + + Prepend each line with a line number. Truncate if there are over 10 lines or 1000 characters + and upload the full output to a paste service. + """ + log.trace("Formatting output...") + + output = output.strip(" \n") + original_output = output # To be uploaded to a pasting service if needed + paste_link = None + + if "<@" in output: + output = output.replace("<@", "<@\u200B") # Zero-width space - code = textwrap.indent(code, " ") - code = CODE_TEMPLATE.replace("{CODE}", code) + if "<!@" in output: + output = output.replace("<!@", "<!@\u200B") # Zero-width space - await self.rmq.send_json( - "input", - snekid=str(ctx.author.id), message=code + if ESCAPE_REGEX.findall(output): + return "Code block escape attempt detected; will not output result", paste_link + + truncated = False + lines = output.count("\n") + + if lines > 0: + output = output.split("\n")[:10] # Only first 10 cause the rest is truncated anyway + output = (f"{i:03d} | {line}" for i, line in enumerate(output, 1)) + output = "\n".join(output) + + if lines > 10: + truncated = True + if len(output) >= 1000: + output = f"{output[:1000]}\n... (truncated - too long, too many lines)" + else: + output = f"{output}\n... (truncated - too many lines)" + elif len(output) >= 1000: + truncated = True + output = f"{output[:1000]}\n... (truncated - too long)" + + if truncated: + paste_link = await self.upload_output(original_output) + + output = output.strip() + if not output: + output = "[No output]" + + return output, paste_link + + @command(name="eval", aliases=("e",)) + @guild_only() + @in_channel(Channels.bot, bypass_roles=STAFF_ROLES) + async def eval_command(self, ctx: Context, *, code: str = None): + """ + Run Python code and get the results. + + This command supports multiple lines of code, including code wrapped inside a formatted code + block. We've done our best to make this safe, but do let us know if you manage to find an + issue with it! + """ + if ctx.author.id in self.jobs: + return await ctx.send( + f"{ctx.author.mention} You've already got a job running - " + f"please wait for it to finish!" ) - async with ctx.typing(): - message = await self.rmq.consume(str(ctx.author.id), **RMQ_ARGS) - paste_link = None + if not code: # None or empty string + return await ctx.invoke(self.bot.get_command("help"), "eval") - if isinstance(message, str): - output = str.strip(" \n") - else: - output = message.body.decode().strip(" \n") + log.info( + f"Received code from {ctx.author.name}#{ctx.author.discriminator} " + f"for evaluation:\n{code}" + ) - if "<@" in output: - output = output.replace("<@", "<@\u200B") # Zero-width space + self.jobs[ctx.author.id] = datetime.datetime.now() + code = self.prepare_input(code) - if "<!@" in output: - output = output.replace("<!@", "<!@\u200B") # Zero-width space + try: + async with ctx.typing(): + results = await self.post_eval(code) + msg, error = self.get_results_message(results) - if ESCAPE_REGEX.findall(output): - output = "Code block escape attempt detected; will not output result" + if error: + output, paste_link = error, None else: - # the original output, to send to a pasting service if needed - full_output = output - truncated = False - if output.count("\n") > 0: - output = [f"{i:03d} | {line}" for i, line in enumerate(output.split("\n"), start=1)] - output = "\n".join(output) - - if output.count("\n") > 10: - output = "\n".join(output.split("\n")[:10]) - - if len(output) >= 1000: - output = f"{output[:1000]}\n... (truncated - too long, too many lines)" - else: - output = f"{output}\n... (truncated - too many lines)" - truncated = True - - elif len(output) >= 1000: - output = f"{output[:1000]}\n... (truncated - too long)" - truncated = True - - if truncated: - try: - response = await self.bot.http_session.post( - URLs.paste_service.format(key="documents"), - data=full_output - ) - data = await response.json() - if "key" in data: - paste_link = URLs.paste_service.format(key=data["key"]) - except Exception: - log.exception("Failed to upload full output to paste service!") - - if output.strip(): - if paste_link: - msg = f"{ctx.author.mention} Your eval job has completed.\n\n```py\n{output}\n```" \ - f"\nFull output: {paste_link}" - else: - msg = f"{ctx.author.mention} Your eval job has completed.\n\n```py\n{output}\n```" - - response = await ctx.send(msg) - self.bot.loop.create_task(wait_for_deletion(response, user_ids=(ctx.author.id,), client=ctx.bot)) + output, paste_link = await self.format_output(results["stdout"]) - else: - await ctx.send( - f"{ctx.author.mention} Your eval job has completed.\n\n```[No output]```" - ) + msg = f"{ctx.author.mention} {msg}.\n\n```py\n{output}\n```" + if paste_link: + msg = f"{msg}\nFull output: {paste_link}" + + response = await ctx.send(msg) + self.bot.loop.create_task( + wait_for_deletion(response, user_ids=(ctx.author.id,), client=ctx.bot) + ) + + log.info( + f"{ctx.author.name}#{ctx.author.discriminator}'s job had a return code of " + f"{results['returncode']}" + ) finally: del self.jobs[ctx.author.id] diff --git a/bot/cogs/superstarify.py b/bot/cogs/superstarify/__init__.py index f46f62552..4b26f3f40 100644 --- a/bot/cogs/superstarify.py +++ b/bot/cogs/superstarify/__init__.py @@ -1,5 +1,6 @@ import logging import random +from datetime import datetime from discord import Colour, Embed, Member from discord.errors import Forbidden @@ -7,12 +8,11 @@ from discord.ext.commands import Bot, Context, command from bot.cogs.moderation import Moderation from bot.cogs.modlog import ModLog -from bot.constants import ( - Icons, Keys, - MODERATION_ROLES, NEGATIVE_REPLIES, - POSITIVE_REPLIES, URLs -) +from bot.cogs.superstarify.stars import get_nick +from bot.constants import Icons, MODERATION_ROLES, POSITIVE_REPLIES +from bot.converters import ExpirationDate from bot.decorators import with_role +from bot.utils.moderation import post_infraction log = logging.getLogger(__name__) NICKNAME_POLICY_URL = "https://pythondiscord.com/about/rules#nickname-policy" @@ -25,7 +25,6 @@ class Superstarify: def __init__(self, bot: Bot): self.bot = bot - self.headers = {"X-API-KEY": Keys.site_api} @property def moderation(self) -> Moderation: @@ -46,35 +45,43 @@ class Superstarify: if before.display_name == after.display_name: return # User didn't change their nickname. Abort! - log.debug( + log.trace( f"{before.display_name} is trying to change their nickname to {after.display_name}. " "Checking if the user is in superstar-prison..." ) - response = await self.bot.http_session.get( - URLs.site_superstarify_api, - headers=self.headers, - params={"user_id": str(before.id)} + active_superstarifies = await self.bot.api_client.get( + 'bot/infractions', + params={ + 'active': 'true', + 'type': 'superstar', + 'user__id': str(before.id) + } ) - response = await response.json() - - if response and response.get("end_timestamp") and not response.get("error_code"): - if after.display_name == response.get("forced_nick"): + if active_superstarifies: + [infraction] = active_superstarifies + forced_nick = get_nick(infraction['id'], before.id) + if after.display_name == forced_nick: return # Nick change was triggered by this event. Ignore. - log.debug( + log.info( f"{after.display_name} is currently in superstar-prison. " f"Changing the nick back to {before.display_name}." ) - await after.edit(nick=response.get("forced_nick")) + await after.edit(nick=forced_nick) + end_timestamp_human = ( + datetime.fromisoformat(infraction['expires_at'][:-1]) + .strftime('%c') + ) + try: await after.send( "You have tried to change your nickname on the **Python Discord** server " f"from **{before.display_name}** to **{after.display_name}**, but as you " "are currently in superstar-prison, you do not have permission to do so. " "You will be allowed to change your nickname again at the following time:\n\n" - f"**{response.get('end_timestamp')}**." + f"**{end_timestamp_human}**." ) except Forbidden: log.warning( @@ -92,23 +99,23 @@ class Superstarify: back to the forced nickname. """ - response = await self.bot.http_session.get( - URLs.site_superstarify_api, - headers=self.headers, - params={"user_id": str(member.id)} + active_superstarifies = await self.bot.api_client.get( + 'bot/infractions', + params={ + 'active': 'true', + 'type': 'superstarify', + 'user__id': member.id + } ) - response = await response.json() - - if response and response.get("end_timestamp") and not response.get("error_code"): - forced_nick = response.get("forced_nick") - end_timestamp = response.get("end_timestamp") - log.debug( - f"{member.name} rejoined but is currently in superstar-prison. " - f"Changing the nick back to {forced_nick}." + if active_superstarifies: + [infraction] = active_superstarifies + forced_nick = get_nick(infraction['id'], member.id) + await member.edit(nick=forced_nick) + end_timestamp_human = ( + datetime.fromisoformat(infraction['expires_at'][:-1]).strftime('%c') ) - await member.edit(nick=forced_nick) try: await member.send( "You have left and rejoined the **Python Discord** server, effectively resetting " @@ -116,7 +123,7 @@ class Superstarify: "but as you are currently in superstar-prison, you do not have permission to do so. " "Therefore your nickname was automatically changed back. You will be allowed to " "change your nickname again at the following time:\n\n" - f"**{end_timestamp}**." + f"**{end_timestamp_human}**." ) except Forbidden: log.warning( @@ -132,7 +139,7 @@ class Superstarify: f"**{member.name}#{member.discriminator}** (`{member.id}`)\n\n" f"Superstarified member potentially tried to escape the prison.\n" f"Restored enforced nickname: `{forced_nick}`\n" - f"Superstardom ends: **{end_timestamp}**" + f"Superstardom ends: **{end_timestamp_human}**" ) await self.modlog.send_log_message( icon_url=Icons.user_update, @@ -144,95 +151,74 @@ class Superstarify: @command(name='superstarify', aliases=('force_nick', 'star')) @with_role(*MODERATION_ROLES) - async def superstarify(self, ctx: Context, member: Member, duration: str, *, forced_nick: str = None): + async def superstarify( + self, ctx: Context, member: Member, expiration: ExpirationDate, reason: str = None + ): """ This command will force a random superstar name (like Taylor Swift) to be the user's - nickname for a specified duration. If a forced_nick is provided, it will use that instead. - - :param ctx: Discord message context - :param ta: - If provided, this function shows data for that specific tag. - If not provided, this function shows the caller a list of all tags. + nickname for a specified duration. An optional reason can be provided. + If no reason is given, the original name will be shown in a generated reason. """ - log.debug( - f"Attempting to superstarify {member.display_name} for {duration}. " - f"forced_nick is set to {forced_nick}." + active_superstarifies = await self.bot.api_client.get( + 'bot/infractions', + params={ + 'active': 'true', + 'type': 'superstar', + 'user__id': str(member.id) + } ) + if active_superstarifies: + return await ctx.send( + ":x: According to my records, this user is already superstarified. " + f"See infraction **#{active_superstarifies[0]['id']}**." + ) - embed = Embed() - embed.colour = Colour.blurple() - - params = { - "user_id": str(member.id), - "duration": duration - } - - if forced_nick: - params["forced_nick"] = forced_nick - - response = await self.bot.http_session.post( - URLs.site_superstarify_api, - headers=self.headers, - json=params + infraction = await post_infraction( + ctx, member, + type='superstar', reason=reason or ('old nick: ' + member.display_name), + expires_at=expiration ) + forced_nick = get_nick(infraction['id'], member.id) - response = await response.json() - - if "error_message" in response: - log.warning( - "Encountered the following error when trying to superstarify the user:\n" - f"{response.get('error_message')}" - ) - embed.colour = Colour.red() - embed.title = random.choice(NEGATIVE_REPLIES) - embed.description = response.get("error_message") - return await ctx.send(embed=embed) - - else: - forced_nick = response.get('forced_nick') - end_time = response.get("end_timestamp") - image_url = response.get("image_url") - old_nick = member.display_name - - embed.title = "Congratulations!" - embed.description = ( - f"Your previous nickname, **{old_nick}**, was so bad that we have decided to change it. " - f"Your new nickname will be **{forced_nick}**.\n\n" - f"You will be unable to change your nickname until \n**{end_time}**.\n\n" - "If you're confused by this, please read our " - f"[official nickname policy]({NICKNAME_POLICY_URL})." - ) - embed.set_image(url=image_url) + embed = Embed() + embed.title = "Congratulations!" + embed.description = ( + f"Your previous nickname, **{member.display_name}**, was so bad that we have decided to change it. " + f"Your new nickname will be **{forced_nick}**.\n\n" + f"You will be unable to change your nickname until \n**{expiration}**.\n\n" + "If you're confused by this, please read our " + f"[official nickname policy]({NICKNAME_POLICY_URL})." + ) - # Log to the mod_log channel - log.trace("Logging to the #mod-log channel. This could fail because of channel permissions.") - mod_log_message = ( - f"**{member.name}#{member.discriminator}** (`{member.id}`)\n\n" - f"Superstarified by **{ctx.author.name}**\n" - f"Old nickname: `{old_nick}`\n" - f"New nickname: `{forced_nick}`\n" - f"Superstardom ends: **{end_time}**" - ) - await self.modlog.send_log_message( - icon_url=Icons.user_update, - colour=Colour.gold(), - title="Member Achieved Superstardom", - text=mod_log_message, - thumbnail=member.avatar_url_as(static_format="png") - ) + # Log to the mod_log channel + log.trace("Logging to the #mod-log channel. This could fail because of channel permissions.") + mod_log_message = ( + f"**{member.name}#{member.discriminator}** (`{member.id}`)\n\n" + f"Superstarified by **{ctx.author.name}**\n" + f"Old nickname: `{member.display_name}`\n" + f"New nickname: `{forced_nick}`\n" + f"Superstardom ends: **{expiration}**" + ) + await self.modlog.send_log_message( + icon_url=Icons.user_update, + colour=Colour.gold(), + title="Member Achieved Superstardom", + text=mod_log_message, + thumbnail=member.avatar_url_as(static_format="png") + ) - await self.moderation.notify_infraction( - user=member, - infr_type="Superstarify", - duration=duration, - reason=f"Your nickname didn't comply with our [nickname policy]({NICKNAME_POLICY_URL})." - ) + await self.moderation.notify_infraction( + user=member, + infr_type="Superstarify", + expires_at=expiration, + reason=f"Your nickname didn't comply with our [nickname policy]({NICKNAME_POLICY_URL})." + ) - # Change the nick and return the embed - log.debug("Changing the users nickname and sending the embed.") - await member.edit(nick=forced_nick) - await ctx.send(embed=embed) + # Change the nick and return the embed + log.trace("Changing the users nickname and sending the embed.") + await member.edit(nick=forced_nick) + await ctx.send(embed=embed) @command(name='unsuperstarify', aliases=('release_nick', 'unstar')) @with_role(*MODERATION_ROLES) @@ -250,33 +236,35 @@ class Superstarify: embed = Embed() embed.colour = Colour.blurple() - response = await self.bot.http_session.delete( - URLs.site_superstarify_api, - headers=self.headers, - json={"user_id": str(member.id)} + active_superstarifies = await self.bot.api_client.get( + 'bot/infractions', + params={ + 'active': 'true', + 'type': 'superstar', + 'user__id': str(member.id) + } + ) + if not active_superstarifies: + return await ctx.send( + ":x: There is no active superstarify infraction for this user." + ) + + [infraction] = active_superstarifies + await self.bot.api_client.patch( + 'bot/infractions/' + str(infraction['id']), + json={'active': False} ) - response = await response.json() + embed = Embed() embed.description = "User has been released from superstar-prison." embed.title = random.choice(POSITIVE_REPLIES) - if "error_message" in response: - embed.colour = Colour.red() - embed.title = random.choice(NEGATIVE_REPLIES) - embed.description = response.get("error_message") - log.warning( - f"Error encountered when trying to unsuperstarify {member.display_name}:\n" - f"{response}" - ) - - else: - await self.moderation.notify_pardon( - user=member, - title="You are no longer superstarified.", - content="You may now change your nickname on the server." - ) - - log.debug(f"{member.display_name} was successfully released from superstar-prison.") + await self.moderation.notify_pardon( + user=member, + title="You are no longer superstarified.", + content="You may now change your nickname on the server." + ) + log.trace(f"{member.display_name} was successfully released from superstar-prison.") await ctx.send(embed=embed) diff --git a/bot/cogs/superstarify/stars.py b/bot/cogs/superstarify/stars.py new file mode 100644 index 000000000..9b49d7175 --- /dev/null +++ b/bot/cogs/superstarify/stars.py @@ -0,0 +1,86 @@ +import random + + +STAR_NAMES = ( + "Adele", + "Aerosmith", + "Aretha Franklin", + "Ayumi Hamasaki", + "B'z", + "Barbra Streisand", + "Barry Manilow", + "Barry White", + "Beyonce", + "Billy Joel", + "Bob Dylan", + "Bob Marley", + "Bob Seger", + "Bon Jovi", + "Britney Spears", + "Bruce Springsteen", + "Bruno Mars", + "Bryan Adams", + "Celine Dion", + "Cher", + "Christina Aguilera", + "David Bowie", + "Donna Summer", + "Drake", + "Ed Sheeran", + "Elton John", + "Elvis Presley", + "Eminem", + "Enya", + "Flo Rida", + "Frank Sinatra", + "Garth Brooks", + "George Michael", + "George Strait", + "James Taylor", + "Janet Jackson", + "Jay-Z", + "Johnny Cash", + "Johnny Hallyday", + "Julio Iglesias", + "Justin Bieber", + "Justin Timberlake", + "Kanye West", + "Katy Perry", + "Kenny G", + "Kenny Rogers", + "Lady Gaga", + "Lil Wayne", + "Linda Ronstadt", + "Lionel Richie", + "Madonna", + "Mariah Carey", + "Meat Loaf", + "Michael Jackson", + "Neil Diamond", + "Nicki Minaj", + "Olivia Newton-John", + "Paul McCartney", + "Phil Collins", + "Pink", + "Prince", + "Reba McEntire", + "Rihanna", + "Robbie Williams", + "Rod Stewart", + "Santana", + "Shania Twain", + "Stevie Wonder", + "Taylor Swift", + "Tim McGraw", + "Tina Turner", + "Tom Petty", + "Tupac Shakur", + "Usher", + "Van Halen", + "Whitney Houston", +) + + +def get_nick(infraction_id, member_id): + rng = random.Random(str(infraction_id) + str(member_id)) + return rng.choice(STAR_NAMES) diff --git a/bot/cogs/sync/__init__.py b/bot/cogs/sync/__init__.py new file mode 100644 index 000000000..e4f960620 --- /dev/null +++ b/bot/cogs/sync/__init__.py @@ -0,0 +1,10 @@ +import logging + +from .cog import Sync + +log = logging.getLogger(__name__) + + +def setup(bot): + bot.add_cog(Sync(bot)) + log.info("Cog loaded: Sync") diff --git a/bot/cogs/sync/cog.py b/bot/cogs/sync/cog.py new file mode 100644 index 000000000..ab591ebf8 --- /dev/null +++ b/bot/cogs/sync/cog.py @@ -0,0 +1,180 @@ +import logging +from typing import Callable, Iterable + +import aiohttp +from discord import Guild, Member, Role +from discord.ext import commands +from discord.ext.commands import Bot + +from bot import constants +from bot.cogs.sync import syncers + +log = logging.getLogger(__name__) + + +class Sync: + """Captures relevant events and sends them to the site.""" + + # The server to synchronize events on. + # Note that setting this wrongly will result in things getting deleted + # that possibly shouldn't be. + SYNC_SERVER_ID = constants.Guild.id + + # An iterable of callables that are called when the bot is ready. + ON_READY_SYNCERS: Iterable[Callable[[Bot, Guild], None]] = ( + syncers.sync_roles, + syncers.sync_users + ) + + def __init__(self, bot): + self.bot = bot + + async def on_ready(self): + guild = self.bot.get_guild(self.SYNC_SERVER_ID) + if guild is not None: + for syncer in self.ON_READY_SYNCERS: + syncer_name = syncer.__name__[5:] # drop off `sync_` + log.info("Starting `%s` syncer.", syncer_name) + total_created, total_updated = await syncer(self.bot, guild) + log.info( + "`%s` syncer finished, created `%d`, updated `%d`.", + syncer_name, total_created, total_updated + ) + + async def on_guild_role_create(self, role: Role): + await self.bot.api_client.post( + 'bot/roles', + json={ + 'colour': role.colour.value, + 'id': role.id, + 'name': role.name, + 'permissions': role.permissions.value + } + ) + + async def on_guild_role_delete(self, role: Role): + log.warning( + ( + "Attempted to delete role `%s` (`%d`), but role deletion " + "is currently not implementeed." + ), + role.name, role.id + ) + + async def on_guild_role_update(self, before: Role, after: Role): + if ( + before.name != after.name + or before.colour != after.colour + or before.permissions != after.permissions + ): + await self.bot.api_client.put( + 'bot/roles/' + str(after.id), + json={ + 'colour': after.colour.value, + 'id': after.id, + 'name': after.name, + 'permissions': after.permissions.value + } + ) + + async def on_member_join(self, member: Member): + packed = { + 'avatar_hash': member.avatar, + 'discriminator': int(member.discriminator), + 'id': member.id, + 'in_guild': True, + 'name': member.name, + 'roles': sorted(role.id for role in member.roles) + } + + got_error = False + + try: + # First try an update of the user to set the `in_guild` field and other + # fields that may have changed since the last time we've seen them. + await self.bot.api_client.put('bot/users/' + str(member.id), json=packed) + + except aiohttp.client_exceptions.ClientResponseError as e: + # If we didn't get 404, something else broke - propagate it up. + if e.status != 404: + raise + + got_error = True # yikes + + if got_error: + # If we got `404`, the user is new. Create them. + await self.bot.api_client.post('bot/users', json=packed) + + async def on_member_leave(self, member: Member): + await self.bot.api_client.put( + 'bot/users/' + str(member.id), + json={ + 'avatar_hash': member.avatar, + 'discriminator': int(member.discriminator), + 'id': member.id, + 'in_guild': True, + 'name': member.name, + 'roles': sorted(role.id for role in member.roles) + } + ) + + async def on_member_update(self, before: Member, after: Member): + if ( + before.name != after.name + or before.avatar != after.avatar + or before.discriminator != after.discriminator + or before.roles != after.roles + ): + try: + await self.bot.api_client.put( + 'bot/users/' + str(after.id), + json={ + 'avatar_hash': after.avatar, + 'discriminator': int(after.discriminator), + 'id': after.id, + 'in_guild': True, + 'name': after.name, + 'roles': sorted(role.id for role in after.roles) + } + ) + except aiohttp.client_exceptions.ClientResponseError as e: + if e.status != 404: + raise + + log.warning( + "Unable to update user, got 404. " + "Assuming race condition from join event." + ) + + @commands.group(name='sync') + @commands.has_permissions(administrator=True) + async def sync_group(self, ctx): + """Run synchronizations between the bot and site manually.""" + + @sync_group.command(name='roles') + @commands.has_permissions(administrator=True) + async def sync_roles_command(self, ctx): + """Manually synchronize the guild's roles with the roles on the site.""" + + initial_response = await ctx.send("📊 Synchronizing roles.") + total_created, total_updated = await syncers.sync_roles(self.bot, ctx.guild) + await initial_response.edit( + content=( + f"👌 Role synchronization complete, created **{total_created}** " + f"and updated **{total_created}** roles." + ) + ) + + @sync_group.command(name='users') + @commands.has_permissions(administrator=True) + async def sync_users_command(self, ctx): + """Manually synchronize the guild's users with the users on the site.""" + + initial_response = await ctx.send("📊 Synchronizing users.") + total_created, total_updated = await syncers.sync_users(self.bot, ctx.guild) + await initial_response.edit( + content=( + f"👌 User synchronization complete, created **{total_created}** " + f"and updated **{total_created}** users." + ) + ) diff --git a/bot/cogs/sync/syncers.py b/bot/cogs/sync/syncers.py new file mode 100644 index 000000000..3037d2e31 --- /dev/null +++ b/bot/cogs/sync/syncers.py @@ -0,0 +1,227 @@ +from collections import namedtuple +from typing import Dict, Set, Tuple + +from discord import Guild +from discord.ext.commands import Bot + +# These objects are declared as namedtuples because tuples are hashable, +# something that we make use of when diffing site roles against guild roles. +Role = namedtuple('Role', ('id', 'name', 'colour', 'permissions')) +User = namedtuple('User', ('id', 'name', 'discriminator', 'avatar_hash', 'roles', 'in_guild')) + + +def get_roles_for_sync( + guild_roles: Set[Role], api_roles: Set[Role] +) -> Tuple[Set[Role], Set[Role]]: + """ + Determine which roles should be created or updated on the site. + + Arguments: + guild_roles (Set[Role]): + Roles that were found on the guild at startup. + + api_roles (Set[Role]): + Roles that were retrieved from the API at startup. + + Returns: + Tuple[Set[Role], Set[Role]]: + A tuple with two elements. The first element represents + roles to be created on the site, meaning that they were + present on the cached guild but not on the API. The second + element represents roles to be updated, meaning they were + present on both the cached guild and the API but non-ID + fields have changed inbetween. + """ + + guild_role_ids = {role.id for role in guild_roles} + api_role_ids = {role.id for role in api_roles} + new_role_ids = guild_role_ids - api_role_ids + + # New roles are those which are on the cached guild but not on the + # API guild, going by the role ID. We need to send them in for creation. + roles_to_create = {role for role in guild_roles if role.id in new_role_ids} + roles_to_update = guild_roles - api_roles - roles_to_create + return roles_to_create, roles_to_update + + +async def sync_roles(bot: Bot, guild: Guild): + """ + Synchronize roles found on the given `guild` with the ones on the API. + + Arguments: + bot (discord.ext.commands.Bot): + The bot instance that we're running with. + + guild (discord.Guild): + The guild instance from the bot's cache + to synchronize roles with. + + Returns: + Tuple[int, int]: + A tuple with two integers representing how many roles were created + (element `0`) and how many roles were updated (element `1`). + """ + + roles = await bot.api_client.get('bot/roles') + + # Pack API roles and guild roles into one common format, + # which is also hashable. We need hashability to be able + # to compare these easily later using sets. + api_roles = {Role(**role_dict) for role_dict in roles} + guild_roles = { + Role( + id=role.id, name=role.name, + colour=role.colour.value, permissions=role.permissions.value + ) + for role in guild.roles + } + roles_to_create, roles_to_update = get_roles_for_sync(guild_roles, api_roles) + + for role in roles_to_create: + await bot.api_client.post( + 'bot/roles', + json={ + 'id': role.id, + 'name': role.name, + 'colour': role.colour, + 'permissions': role.permissions + } + ) + + for role in roles_to_update: + await bot.api_client.put( + 'bot/roles/' + str(role.id), + json={ + 'id': role.id, + 'name': role.name, + 'colour': role.colour, + 'permissions': role.permissions + } + ) + + return (len(roles_to_create), len(roles_to_update)) + + +def get_users_for_sync( + guild_users: Dict[int, User], api_users: Dict[int, User] +) -> Tuple[Set[User], Set[User]]: + """ + Determine which users should be created or updated on the website. + + Arguments: + guild_users (Dict[int, User]): + A mapping of user IDs to user data, populated from the + guild cached on the running bot instance. + + api_users (Dict[int, User]): + A mapping of user IDs to user data, populated from the API's + current inventory of all users. + + Returns: + Tuple[Set[User], Set[User]]: + Two user sets as a tuple. The first element represents users + to be created on the website, these are users that are present + in the cached guild data but not in the API at all, going by + their ID. The second element represents users to update. It is + populated by users which are present on both the API and the + guild, but where the attribute of a user on the API is not + equal to the attribute of the user on the guild. + """ + + users_to_create = set() + users_to_update = set() + + for api_user in api_users.values(): + guild_user = guild_users.get(api_user.id) + if guild_user is not None: + if api_user != guild_user: + users_to_update.add(guild_user) + + elif api_user.in_guild: + # The user is known on the API but not the guild, and the + # API currently specifies that the user is a member of the guild. + # This means that the user has left since the last sync. + # Update the `in_guild` attribute of the user on the site + # to signify that the user left. + new_api_user = api_user._replace(in_guild=False) + users_to_update.add(new_api_user) + + new_user_ids = set(guild_users.keys()) - set(api_users.keys()) + for user_id in new_user_ids: + # The user is known on the guild but not on the API. This means + # that the user has joined since the last sync. Create it. + new_user = guild_users[user_id] + users_to_create.add(new_user) + + return users_to_create, users_to_update + + +async def sync_users(bot: Bot, guild: Guild): + """ + Synchronize users found on the given + `guild` with the ones on the API. + + Arguments: + bot (discord.ext.commands.Bot): + The bot instance that we're running with. + + guild (discord.Guild): + The guild instance from the bot's cache + to synchronize roles with. + + Returns: + Tuple[int, int]: + A tuple with two integers representing how many users were created + (element `0`) and how many users were updated (element `1`). + """ + + current_users = await bot.api_client.get('bot/users') + + # Pack API users and guild users into one common format, + # which is also hashable. We need hashability to be able + # to compare these easily later using sets. + api_users = { + user_dict['id']: User( + roles=tuple(sorted(user_dict.pop('roles'))), + **user_dict + ) + for user_dict in current_users + } + guild_users = { + member.id: User( + id=member.id, name=member.name, + discriminator=int(member.discriminator), avatar_hash=member.avatar, + roles=tuple(sorted(role.id for role in member.roles)), in_guild=True + ) + for member in guild.members + } + + users_to_create, users_to_update = get_users_for_sync(guild_users, api_users) + + for user in users_to_create: + await bot.api_client.post( + 'bot/users', + json={ + 'avatar_hash': user.avatar_hash, + 'discriminator': user.discriminator, + 'id': user.id, + 'in_guild': user.in_guild, + 'name': user.name, + 'roles': list(user.roles) + } + ) + + for user in users_to_update: + await bot.api_client.put( + 'bot/users/' + str(user.id), + json={ + 'avatar_hash': user.avatar_hash, + 'discriminator': user.discriminator, + 'id': user.id, + 'in_guild': user.in_guild, + 'name': user.name, + 'roles': list(user.roles) + } + ) + + return (len(users_to_create), len(users_to_update)) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index d6957e360..7b1003148 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -1,19 +1,11 @@ import logging -import random import time -from typing import Optional from discord import Colour, Embed -from discord.ext.commands import ( - BadArgument, Bot, - Context, group -) +from discord.ext.commands import Bot, Context, group -from bot.constants import ( - Channels, Cooldowns, ERROR_REPLIES, Keys, - MODERATION_ROLES, Roles, URLs -) -from bot.converters import TagContentConverter, TagNameConverter, ValidURL +from bot.constants import Channels, Cooldowns, Keys, MODERATION_ROLES, Roles +from bot.converters import TagContentConverter, TagNameConverter from bot.decorators import with_role from bot.pagination import LinePaginator @@ -35,72 +27,7 @@ class Tags: def __init__(self, bot: Bot): self.bot = bot self.tag_cooldowns = {} - self.headers = {"X-API-KEY": Keys.site_api} - - async def get_tag_data(self, tag_name=None) -> dict: - """ - Retrieve the tag_data from our API - - :param tag_name: the tag to retrieve - :return: - if tag_name was provided, returns a dict with tag data. - if not, returns a list of dicts with all tag data. - - """ - params = {} - - if tag_name: - params["tag_name"] = tag_name - - response = await self.bot.http_session.get(URLs.site_tags_api, headers=self.headers, params=params) - tag_data = await response.json() - - return tag_data - - async def post_tag_data(self, tag_name: str, tag_content: str, image_url: Optional[str]) -> dict: - """ - Send some tag_data to our API to have it saved in the database. - - :param tag_name: The name of the tag to create or edit. - :param tag_content: The content of the tag. - :param image_url: The image URL of the tag, can be `None`. - :return: json response from the API in the following format: - { - 'success': bool - } - """ - - params = { - 'tag_name': tag_name, - 'tag_content': tag_content, - 'image_url': image_url - } - - response = await self.bot.http_session.post(URLs.site_tags_api, headers=self.headers, json=params) - tag_data = await response.json() - - return tag_data - - async def delete_tag_data(self, tag_name: str) -> dict: - """ - Delete a tag using our API. - - :param tag_name: The name of the tag to delete. - :return: json response from the API in the following format: - { - 'success': bool - } - """ - - params = {} - - if tag_name: - params['tag_name'] = tag_name - - response = await self.bot.http_session.delete(URLs.site_tags_api, headers=self.headers, json=params) - tag_data = await response.json() - - return tag_data + self.headers = {"Authorization": f"Token {Keys.site_api}"} @group(name='tags', aliases=('tag', 't'), hidden=True, invoke_without_command=True) async def tags_group(self, ctx: Context, *, tag_name: TagNameConverter = None): @@ -148,69 +75,32 @@ class Tags: f"Cooldown ends in {time_left:.1f} seconds.") return - tags = [] - - embed: Embed = Embed() - embed.colour = Colour.red() - tag_data = await self.get_tag_data(tag_name) - - # If we found something, prepare that data - if tag_data: - embed.colour = Colour.blurple() - - if tag_name: - log.debug(f"{ctx.author} requested the tag '{tag_name}'") - embed.title = tag_name - - if ctx.channel.id not in TEST_CHANNELS: - self.tag_cooldowns[tag_name] = { - "time": time.time(), - "channel": ctx.channel.id - } - - else: - embed.title = "**Current tags**" - - if isinstance(tag_data, list): - log.debug(f"{ctx.author} requested a list of all tags") - tags = [f"**»** {tag['tag_name']}" for tag in tag_data] - tags = sorted(tags) - - else: - embed.description = tag_data['tag_content'] - if tag_data['image_url'] is not None: - embed.set_image(url=tag_data['image_url']) + if tag_name is not None: + tag = await self.bot.api_client.get(f'bot/tags/{tag_name}') + if ctx.channel.id not in TEST_CHANNELS: + self.tag_cooldowns[tag_name] = { + "time": time.time(), + "channel": ctx.channel.id + } + await ctx.send(embed=Embed.from_data(tag['embed'])) - # If its invoked from error handler just ignore it. - elif hasattr(ctx, "invoked_from_error_handler"): - return - # If not, prepare an error message. else: - embed.colour = Colour.red() - - if isinstance(tag_data, dict): - log.warning(f"{ctx.author} requested the tag '{tag_name}', but it could not be found.") - embed.description = f"**{tag_name}** is an unknown tag name. Please check the spelling and try again." + tags = await self.bot.api_client.get('bot/tags') + if not tags: + await ctx.send(embed=Embed( + description="**There are no tags in the database!**", + colour=Colour.red() + )) else: - log.warning(f"{ctx.author} requested a list of all tags, but the tags database was empty!") - embed.description = "**There are no tags in the database!**" - - if tag_name: - embed.set_footer(text="To show a list of all tags, use !tags.") - embed.title = "Tag not found." - - # Paginate if this is a list of all tags - if tags: - log.debug(f"Returning a paginated list of all tags.") - return await LinePaginator.paginate( - (lines for lines in tags), - ctx, embed, - footer_text="To show a tag, type !tags <tagname>.", - empty=False, - max_lines=15 - ) - - return await ctx.send(embed=embed) + embed: Embed = Embed(title="**Current tags**") + await LinePaginator.paginate( + sorted(f"**»** {tag['title']}" for tag in tags), + ctx, + embed, + footer_text="To show a tag, type !tags <tagname>.", + empty=False, + max_lines=15 + ) @tags_group.command(name='set', aliases=('add', 'edit', 's')) @with_role(*MODERATION_ROLES) @@ -218,44 +108,36 @@ class Tags: self, ctx: Context, tag_name: TagNameConverter, + *, tag_content: TagContentConverter, - image_url: ValidURL = None ): """ - Create a new tag or edit an existing one. + Create a new tag or update an existing one. :param ctx: discord message context :param tag_name: The name of the tag to create or edit. :param tag_content: The content of the tag. - :param image_url: An optional image for the tag. """ - tag_name = tag_name.lower().strip() - tag_content = tag_content.strip() - - embed = Embed() - embed.colour = Colour.red() - tag_data = await self.post_tag_data(tag_name, tag_content, image_url) - - if tag_data.get("success"): - log.debug(f"{ctx.author} successfully added the following tag to our database: \n" - f"tag_name: {tag_name}\n" - f"tag_content: '{tag_content}'\n" - f"image_url: '{image_url}'") - embed.colour = Colour.blurple() - embed.title = "Tag successfully added" - embed.description = f"**{tag_name}** added to tag database." - else: - log.error("There was an unexpected database error when trying to add the following tag: \n" - f"tag_name: {tag_name}\n" - f"tag_content: '{tag_content}'\n" - f"image_url: '{image_url}'\n" - f"response: {tag_data}") - embed.title = "Database error" - embed.description = ("There was a problem adding the data to the tags database. " - "Please try again. If the problem persists, see the error logs.") + body = { + 'title': tag_name.lower().strip(), + 'embed': { + 'title': tag_name, + 'description': tag_content + } + } - return await ctx.send(embed=embed) + await self.bot.api_client.post('bot/tags', json=body) + + log.debug(f"{ctx.author} successfully added the following tag to our database: \n" + f"tag_name: {tag_name}\n" + f"tag_content: '{tag_content}'\n") + + await ctx.send(embed=Embed( + title="Tag successfully added", + description=f"**{tag_name}** added to tag database.", + colour=Colour.blurple() + )) @tags_group.command(name='delete', aliases=('remove', 'rm', 'd')) @with_role(Roles.admin, Roles.owner) @@ -267,45 +149,14 @@ class Tags: :param tag_name: The name of the tag to delete. """ - tag_name = tag_name.lower().strip() - embed = Embed() - embed.colour = Colour.red() - tag_data = await self.delete_tag_data(tag_name) - - if tag_data.get("success") is True: - log.debug(f"{ctx.author} successfully deleted the tag called '{tag_name}'") - embed.colour = Colour.blurple() - embed.title = tag_name - embed.description = f"Tag successfully removed: {tag_name}." + await self.bot.api_client.delete(f'bot/tags/{tag_name}') - elif tag_data.get("success") is False: - log.debug(f"{ctx.author} tried to delete a tag called '{tag_name}', but the tag does not exist.") - embed.colour = Colour.red() - embed.title = tag_name - embed.description = "Tag doesn't appear to exist." - - else: - log.error("There was an unexpected database error when trying to delete the following tag: \n" - f"tag_name: {tag_name}\n" - f"response: {tag_data}") - embed.title = "Database error" - embed.description = ("There was an unexpected error with deleting the data from the tags database. " - "Please try again. If the problem persists, see the error logs.") - - return await ctx.send(embed=embed) - - @get_command.error - @set_command.error - @delete_command.error - async def command_error(self, ctx, error): - if isinstance(error, BadArgument): - embed = Embed() - embed.colour = Colour.red() - embed.description = str(error) - embed.title = random.choice(ERROR_REPLIES) - await ctx.send(embed=embed) - else: - log.error(f"Unhandled tag command error: {error} ({error.original})") + log.debug(f"{ctx.author} successfully deleted the tag called '{tag_name}'") + await ctx.send(embed=Embed( + title=tag_name, + description=f"Tag successfully removed: {tag_name}.", + colour=Colour.blurple() + )) def setup(bot): diff --git a/bot/cogs/watchchannels/__init__.py b/bot/cogs/watchchannels/__init__.py new file mode 100644 index 000000000..ac7713803 --- /dev/null +++ b/bot/cogs/watchchannels/__init__.py @@ -0,0 +1,15 @@ +import logging + +from .bigbrother import BigBrother +from .talentpool import TalentPool + + +log = logging.getLogger(__name__) + + +def setup(bot): + bot.add_cog(BigBrother(bot)) + log.info("Cog loaded: BigBrother") + + bot.add_cog(TalentPool(bot)) + log.info("Cog loaded: TalentPool") diff --git a/bot/cogs/watchchannels/bigbrother.py b/bot/cogs/watchchannels/bigbrother.py new file mode 100644 index 000000000..e7b3d70bc --- /dev/null +++ b/bot/cogs/watchchannels/bigbrother.py @@ -0,0 +1,100 @@ +import logging +from collections import ChainMap +from typing import Union + +from discord import User +from discord.ext.commands import Context, group + +from bot.constants import Channels, Roles, Webhooks +from bot.decorators import with_role +from bot.utils.moderation import post_infraction +from .watchchannel import WatchChannel, proxy_user + +log = logging.getLogger(__name__) + + +class BigBrother(WatchChannel): + """Monitors users by relaying their messages to a watch channel to assist with moderation.""" + + def __init__(self, bot) -> None: + super().__init__( + bot, + destination=Channels.big_brother_logs, + webhook_id=Webhooks.big_brother, + api_endpoint='bot/infractions', + api_default_params={'active': 'true', 'type': 'watch', 'ordering': '-inserted_at'}, + logger=log + ) + + @group(name='bigbrother', aliases=('bb',), invoke_without_command=True) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def bigbrother_group(self, ctx: Context) -> None: + """Monitors users by relaying their messages to the Big Brother watch channel.""" + await ctx.invoke(self.bot.get_command("help"), "bigbrother") + + @bigbrother_group.command(name='watched', aliases=('all', 'list')) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def watched_command(self, ctx: Context, update_cache: bool = True) -> None: + """ + Shows the users that are currently being monitored by Big Brother. + + The optional kwarg `update_cache` can be used to update the user + cache using the API before listing the users. + """ + await self.list_watched_users(ctx, update_cache) + + @bigbrother_group.command(name='watch', aliases=('w',)) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def watch_command(self, ctx: Context, user: Union[User, proxy_user], *, reason: str) -> None: + """ + Relay messages sent by the given `user` to the `#big-brother` channel. + + A `reason` for adding the user to Big Brother is required and will be displayed + in the header when relaying messages of this user to the watchchannel. + """ + if user.bot: + await ctx.send(f":x: I'm sorry {ctx.author}, I'm afraid I can't do that. I only watch humans.") + return + + if not await self.fetch_user_cache(): + await ctx.send(f":x: Updating the user cache failed, can't watch user {user}") + return + + if user.id in self.watched_users: + await ctx.send(":x: The specified user is already being watched.") + return + + response = await post_infraction( + ctx, user, type='watch', reason=reason, hidden=True + ) + + if response is not None: + self.watched_users[user.id] = response + await ctx.send(f":white_check_mark: Messages sent by {user} will now be relayed to Big Brother.") + + @bigbrother_group.command(name='unwatch', aliases=('uw',)) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def unwatch_command(self, ctx: Context, user: Union[User, proxy_user], *, reason: str) -> None: + """Stop relaying messages by the given `user`.""" + active_watches = await self.bot.api_client.get( + self.api_endpoint, + params=ChainMap( + self.api_default_params, + {"user__id": str(user.id)} + ) + ) + if active_watches: + [infraction] = active_watches + + await self.bot.api_client.patch( + f"{self.api_endpoint}/{infraction['id']}", + json={'active': False} + ) + + await post_infraction(ctx, user, type='watch', reason=f"Unwatched: {reason}", hidden=True, active=False) + + await ctx.send(f":white_check_mark: Messages sent by {user} will no longer be relayed.") + + self._remove_user(user.id) + else: + await ctx.send(":x: The specified user is currently not being watched.") diff --git a/bot/cogs/watchchannels/talentpool.py b/bot/cogs/watchchannels/talentpool.py new file mode 100644 index 000000000..6fbe2bc03 --- /dev/null +++ b/bot/cogs/watchchannels/talentpool.py @@ -0,0 +1,233 @@ +import logging +import textwrap +from collections import ChainMap +from typing import Union + +from aiohttp.client_exceptions import ClientResponseError +from discord import Color, Embed, Member, User +from discord.ext.commands import Context, group + +from bot.constants import Channels, Guild, Roles, Webhooks +from bot.decorators import with_role +from bot.pagination import LinePaginator +from .watchchannel import WatchChannel, proxy_user + +log = logging.getLogger(__name__) +STAFF_ROLES = Roles.owner, Roles.admin, Roles.moderator, Roles.helpers # <- In constants after the merge? + + +class TalentPool(WatchChannel): + """Relays messages of helper candidates to a watch channel to observe them.""" + + def __init__(self, bot) -> None: + super().__init__( + bot, + destination=Channels.talent_pool, + webhook_id=Webhooks.talent_pool, + api_endpoint='bot/nominations', + api_default_params={'active': 'true', 'ordering': '-inserted_at'}, + logger=log, + ) + + @group(name='talentpool', aliases=('tp', 'talent', 'nomination', 'n'), invoke_without_command=True) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def nomination_group(self, ctx: Context) -> None: + """Highlights the activity of helper nominees by relaying their messages to the talent pool channel.""" + + await ctx.invoke(self.bot.get_command("help"), "talentpool") + + @nomination_group.command(name='watched', aliases=('all', 'list')) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def watched_command(self, ctx: Context, update_cache: bool = True) -> None: + """ + Shows the users that are currently being monitored in the talent pool. + + The optional kwarg `update_cache` can be used to update the user + cache using the API before listing the users. + """ + await self.list_watched_users(ctx, update_cache) + + @nomination_group.command(name='watch', aliases=('w', 'add', 'a')) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def watch_command(self, ctx: Context, user: Union[Member, User, proxy_user], *, reason: str) -> None: + """ + Relay messages sent by the given `user` to the `#talent-pool` channel. + + A `reason` for adding the user to the talent pool is required and will be displayed + in the header when relaying messages of this user to the channel. + """ + if user.bot: + await ctx.send(f":x: I'm sorry {ctx.author}, I'm afraid I can't do that. I only watch humans.") + return + + if isinstance(user, Member) and any(role.id in STAFF_ROLES for role in user.roles): + await ctx.send(f":x: Nominating staff members, eh? Here's a cookie :cookie:") + return + + if not await self.fetch_user_cache(): + await ctx.send(f":x: Failed to update the user cache; can't add {user}") + return + + if user.id in self.watched_users: + await ctx.send(":x: The specified user is already being watched in the talent pool") + return + + # Manual request with `raise_for_status` as False because we want the actual response + session = self.bot.api_client.session + url = self.bot.api_client._url_for(self.api_endpoint) + kwargs = { + 'json': { + 'actor': ctx.author.id, + 'reason': reason, + 'user': user.id + }, + 'raise_for_status': False, + } + async with session.post(url, **kwargs) as resp: + response_data = await resp.json() + + if resp.status == 400 and response_data.get('user', False): + await ctx.send(":x: The specified user can't be found in the database tables") + return + else: + resp.raise_for_status() + + self.watched_users[user.id] = response_data + await ctx.send(f":white_check_mark: Messages sent by {user} will now be relayed to the talent pool channel") + + @nomination_group.command(name='history', aliases=('info', 'search')) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def history_command(self, ctx: Context, user: Union[User, proxy_user]) -> None: + """Shows the specified user's nomination history.""" + result = await self.bot.api_client.get( + self.api_endpoint, + params={ + 'user__id': str(user.id), + 'ordering': "-active,-inserted_at" + } + ) + if not result: + await ctx.send(":warning: This user has never been nominated") + return + + embed = Embed( + title=f"Nominations for {user.display_name} `({user.id})`", + color=Color.blue() + ) + lines = [self._nomination_to_string(nomination) for nomination in result] + await LinePaginator.paginate( + lines, + ctx=ctx, + embed=embed, + empty=True, + max_lines=3, + max_size=1000 + ) + + @nomination_group.command(name='unwatch', aliases=('end', )) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def unwatch_command(self, ctx: Context, user: Union[User, proxy_user], *, reason: str) -> None: + """ + Ends the active nomination of the specified user with the given reason. + + Providing a `reason` is required. + """ + active_nomination = await self.bot.api_client.get( + self.api_endpoint, + params=ChainMap( + self.api_default_params, + {"user__id": str(user.id)} + ) + ) + + if not active_nomination: + await ctx.send(":x: The specified user does not have an active nomination") + return + + [nomination] = active_nomination + await self.bot.api_client.patch( + f"{self.api_endpoint}/{nomination['id']}", + json={'end_reason': reason, 'active': False} + ) + await ctx.send(f":white_check_mark: Messages sent by {user} will no longer be relayed") + self._remove_user(user.id) + + @nomination_group.group(name='edit', aliases=('e',), invoke_without_command=True) + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def nomination_edit_group(self, ctx: Context) -> None: + """Commands to edit nominations.""" + + await ctx.invoke(self.bot.get_command("help"), "talentpool", "edit") + + @nomination_edit_group.command(name='reason') + @with_role(Roles.owner, Roles.admin, Roles.moderator) + async def edit_reason_command(self, ctx: Context, nomination_id: int, *, reason: str) -> None: + """ + Edits the reason/unnominate reason for the nomination with the given `id` depending on the status. + + If the nomination is active, the reason for nominating the user will be edited; + If the nomination is no longer active, the reason for ending the nomination will be edited instead. + """ + try: + nomination = await self.bot.api_client.get(f"{self.api_endpoint}/{nomination_id}") + except ClientResponseError as e: + if e.status == 404: + self.log.trace(f"Nomination API 404: Can't nomination with id {nomination_id}") + await ctx.send(f":x: Can't find a nomination with id `{nomination_id}`") + return + else: + raise + + field = "reason" if nomination["active"] else "end_reason" + + self.log.trace(f"Changing {field} for nomination with id {nomination_id} to {reason}") + + await self.bot.api_client.patch( + f"{self.api_endpoint}/{nomination_id}", + json={field: reason} + ) + + await ctx.send(f":white_check_mark: Updated the {field} of the nomination!") + + def _nomination_to_string(self, nomination_object: dict) -> str: + """Creates a string representation of a nomination.""" + guild = self.bot.get_guild(Guild.id) + + actor_id = nomination_object["actor"] + actor = guild.get_member(actor_id) + + active = nomination_object["active"] + log.debug(active) + log.debug(type(nomination_object["inserted_at"])) + + start_date = self._get_human_readable(nomination_object["inserted_at"]) + if active: + lines = textwrap.dedent( + f""" + =============== + Status: **Active** + Date: {start_date} + Actor: {actor.mention if actor else actor_id} + Reason: {nomination_object["reason"]} + Nomination ID: `{nomination_object["id"]}` + =============== + """ + ) + else: + end_date = self._get_human_readable(nomination_object["ended_at"]) + lines = textwrap.dedent( + f""" + =============== + Status: Inactive + Date: {start_date} + Actor: {actor.mention if actor else actor_id} + Reason: {nomination_object["reason"]} + + End date: {end_date} + Unwatch reason: {nomination_object["end_reason"]} + Nomination ID: `{nomination_object["id"]}` + =============== + """ + ) + + return lines.strip() diff --git a/bot/cogs/watchchannels/watchchannel.py b/bot/cogs/watchchannels/watchchannel.py new file mode 100644 index 000000000..fe6d6bb6e --- /dev/null +++ b/bot/cogs/watchchannels/watchchannel.py @@ -0,0 +1,353 @@ +import asyncio +import datetime +import logging +import re +import textwrap +from abc import ABC, abstractmethod +from collections import defaultdict, deque +from dataclasses import dataclass +from typing import Optional + +import aiohttp +import discord +from discord import Color, Embed, Message, Object, errors +from discord.ext.commands import BadArgument, Bot, Context + +from bot.cogs.modlog import ModLog +from bot.constants import BigBrother as BigBrotherConfig, Guild as GuildConfig, Icons +from bot.pagination import LinePaginator +from bot.utils import messages +from bot.utils.time import time_since + +log = logging.getLogger(__name__) + +URL_RE = re.compile(r"(https?://[^\s]+)") + + +def proxy_user(user_id: str) -> Object: + """A proxy user object that mocks a real User instance for when the later is not available.""" + try: + user_id = int(user_id) + except ValueError: + raise BadArgument + + user = Object(user_id) + user.mention = user.id + user.display_name = f"<@{user.id}>" + user.avatar_url_as = lambda static_format: None + user.bot = False + + return user + + +@dataclass +class MessageHistory: + last_author: Optional[int] = None + last_channel: Optional[int] = None + message_count: int = 0 + + +class WatchChannel(ABC): + """ABC with functionality for relaying users' messages to a certain channel.""" + + @abstractmethod + def __init__(self, bot: Bot, destination, webhook_id, api_endpoint, api_default_params, logger) -> None: + self.bot = bot + + self.destination = destination # E.g., Channels.big_brother_logs + self.webhook_id = webhook_id # E.g., Webhooks.big_brother + self.api_endpoint = api_endpoint # E.g., 'bot/infractions' + self.api_default_params = api_default_params # E.g., {'active': 'true', 'type': 'watch'} + self.log = logger # Logger of the child cog for a correct name in the logs + + self._consume_task = None + self.watched_users = defaultdict(dict) + self.message_queue = defaultdict(lambda: defaultdict(deque)) + self.consumption_queue = {} + self.retries = 5 + self.retry_delay = 10 + self.channel = None + self.webhook = None + self.message_history = MessageHistory() + + self._start = self.bot.loop.create_task(self.start_watchchannel()) + + @property + def modlog(self) -> ModLog: + """Provides access to the ModLog cog for alert purposes.""" + return self.bot.get_cog("ModLog") + + @property + def consuming_messages(self) -> bool: + """Checks if a consumption task is currently running.""" + if self._consume_task is None: + return False + + if self._consume_task.done(): + exc = self._consume_task.exception() + if exc: + self.log.exception( + f"The message queue consume task has failed with:", + exc_info=exc + ) + return False + + return True + + async def start_watchchannel(self) -> None: + """Starts the watch channel by getting the channel, webhook, and user cache ready.""" + await self.bot.wait_until_ready() + + # After updating d.py, this block can be replaced by `fetch_channel` with a try-except + for attempt in range(1, self.retries+1): + self.channel = self.bot.get_channel(self.destination) + if self.channel is None: + if attempt < self.retries: + await asyncio.sleep(self.retry_delay) + else: + break + else: + self.log.error(f"Failed to retrieve the text channel with id {self.destination}") + + # `get_webhook_info` has been renamed to `fetch_webhook` in newer versions of d.py + try: + self.webhook = await self.bot.get_webhook_info(self.webhook_id) + except (discord.HTTPException, discord.NotFound, discord.Forbidden): + self.log.exception(f"Failed to fetch webhook with id `{self.webhook_id}`") + + if self.channel is None or self.webhook is None: + self.log.error("Failed to start the watch channel; unloading the cog.") + + message = textwrap.dedent( + f""" + An error occurred while loading the text channel or webhook. + + TextChannel: {"**Failed to load**" if self.channel is None else "Loaded successfully"} + Webhook: {"**Failed to load**" if self.webhook is None else "Loaded successfully"} + + The Cog has been unloaded. + """ + ) + + await self.modlog.send_log_message( + title=f"Error: Failed to initialize the {self.__class__.__name__} watch channel", + text=message, + ping_everyone=True, + icon_url=Icons.token_removed, + colour=Color.red() + ) + + self.bot.remove_cog(self.__class__.__name__) + return + + if not await self.fetch_user_cache(): + await self.modlog.send_log_message( + title=f"Warning: Failed to retrieve user cache for the {self.__class__.__name__} watch channel", + text="Could not retrieve the list of watched users from the API and messages will not be relayed.", + ping_everyone=True, + icon=Icons.token_removed, + color=Color.red() + ) + + async def fetch_user_cache(self) -> bool: + """ + Fetches watched users from the API and updates the watched user cache accordingly. + + This function returns `True` if the update succeeded. + """ + try: + data = await self.bot.api_client.get(self.api_endpoint, params=self.api_default_params) + except aiohttp.ClientResponseError as e: + self.log.exception(f"Failed to fetch the watched users from the API", exc_info=e) + return False + + self.watched_users = defaultdict(dict) + + for entry in data: + user_id = entry.pop('user') + self.watched_users[user_id] = entry + + return True + + async def on_message(self, msg: Message) -> None: + """Queues up messages sent by watched users.""" + if msg.author.id in self.watched_users: + if not self.consuming_messages: + self._consume_task = self.bot.loop.create_task(self.consume_messages()) + + self.log.trace(f"Received message: {msg.content} ({len(msg.attachments)} attachments)") + self.message_queue[msg.author.id][msg.channel.id].append(msg) + + async def consume_messages(self, delay_consumption: bool = True) -> None: + """Consumes the message queues to log watched users' messages.""" + if delay_consumption: + self.log.trace(f"Sleeping {BigBrotherConfig.log_delay} seconds before consuming message queue") + await asyncio.sleep(BigBrotherConfig.log_delay) + + self.log.trace(f"Started consuming the message queue") + + # If the previous consumption Task failed, first consume the existing comsumption_queue + if not self.consumption_queue: + self.consumption_queue = self.message_queue.copy() + self.message_queue.clear() + + for user_channel_queues in self.consumption_queue.values(): + for channel_queue in user_channel_queues.values(): + while channel_queue: + msg = channel_queue.popleft() + + self.log.trace(f"Consuming message {msg.id} ({len(msg.attachments)} attachments)") + await self.relay_message(msg) + + self.consumption_queue.clear() + + if self.message_queue: + self.log.trace("Channel queue not empty: Continuing consuming queues") + self._consume_task = self.bot.loop.create_task(self.consume_messages(delay_consumption=False)) + else: + self.log.trace("Done consuming messages.") + + async def webhook_send( + self, + content: Optional[str] = None, + username: Optional[str] = None, + avatar_url: Optional[str] = None, + embed: Optional[Embed] = None, + ) -> None: + """Sends a message to the webhook with the specified kwargs.""" + try: + await self.webhook.send(content=content, username=username, avatar_url=avatar_url, embed=embed) + except discord.HTTPException as exc: + self.log.exception( + f"Failed to send a message to the webhook", + exc_info=exc + ) + + async def relay_message(self, msg: Message) -> None: + """Relays the message to the relevant watch channel.""" + limit = BigBrotherConfig.header_message_limit + + if ( + msg.author.id != self.message_history.last_author + or msg.channel.id != self.message_history.last_channel + or self.message_history.message_count >= limit + ): + self.message_history = MessageHistory(last_author=msg.author.id, last_channel=msg.channel.id) + + await self.send_header(msg) + + cleaned_content = msg.clean_content + + if cleaned_content: + # Put all non-media URLs in a code block to prevent embeds + media_urls = {embed.url for embed in msg.embeds if embed.type in ("image", "video")} + for url in URL_RE.findall(cleaned_content): + if url not in media_urls: + cleaned_content = cleaned_content.replace(url, f"`{url}`") + await self.webhook_send( + cleaned_content, + username=msg.author.display_name, + avatar_url=msg.author.avatar_url + ) + + if msg.attachments: + try: + await messages.send_attachments(msg, self.webhook) + except (errors.Forbidden, errors.NotFound): + e = Embed( + description=":x: **This message contained an attachment, but it could not be retrieved**", + color=Color.red() + ) + await self.webhook_send( + embed=e, + username=msg.author.display_name, + avatar_url=msg.author.avatar_url + ) + except discord.HTTPException as exc: + self.log.exception( + f"Failed to send an attachment to the webhook", + exc_info=exc + ) + + self.message_history.message_count += 1 + + async def send_header(self, msg) -> None: + """Sends a header embed with information about the relayed messages to the watch channel.""" + user_id = msg.author.id + + guild = self.bot.get_guild(GuildConfig.id) + actor = guild.get_member(self.watched_users[user_id]['actor']) + actor = actor.display_name if actor else self.watched_users[user_id]['actor'] + + inserted_at = self.watched_users[user_id]['inserted_at'] + time_delta = self._get_time_delta(inserted_at) + + reason = self.watched_users[user_id]['reason'] + + embed = Embed(description=f"{msg.author.mention} in [#{msg.channel.name}]({msg.jump_url})") + embed.set_footer(text=f"Added {time_delta} by {actor} | Reason: {reason}") + + await self.webhook_send(embed=embed, username=msg.author.display_name, avatar_url=msg.author.avatar_url) + + async def list_watched_users(self, ctx: Context, update_cache: bool = True) -> None: + """ + Gives an overview of the watched user list for this channel. + + The optional kwarg `update_cache` specifies whether the cache should + be refreshed by polling the API. + """ + if update_cache: + if not await self.fetch_user_cache(): + await ctx.send(f":x: Failed to update {self.__class__.__name__} user cache, serving from cache") + update_cache = False + + lines = [] + for user_id, user_data in self.watched_users.items(): + inserted_at = user_data['inserted_at'] + time_delta = self._get_time_delta(inserted_at) + lines.append(f"• <@{user_id}> (added {time_delta})") + + lines = lines or ("There's nothing here yet.",) + embed = Embed( + title=f"{self.__class__.__name__} watched users ({'updated' if update_cache else 'cached'})", + color=Color.blue() + ) + await LinePaginator.paginate(lines, ctx, embed, empty=False) + + @staticmethod + def _get_time_delta(time_string: str) -> str: + """Returns the time in human-readable time delta format.""" + date_time = datetime.datetime.strptime( + time_string, + "%Y-%m-%dT%H:%M:%S.%fZ" + ).replace(tzinfo=None) + time_delta = time_since(date_time, precision="minutes", max_units=1) + + return time_delta + + @staticmethod + def _get_human_readable(time_string: str, output_format: str = "%Y-%m-%d %H:%M:%S") -> str: + date_time = datetime.datetime.strptime( + time_string, + "%Y-%m-%dT%H:%M:%S.%fZ" + ).replace(tzinfo=None) + return date_time.strftime(output_format) + + def _remove_user(self, user_id: int) -> None: + """Removes a user from a watch channel.""" + self.watched_users.pop(user_id, None) + self.message_queue.pop(user_id, None) + self.consumption_queue.pop(user_id, None) + + def cog_unload(self) -> None: + """Takes care of unloading the cog and canceling the consumption task.""" + self.log.trace(f"Unloading the cog") + if not self._consume_task.done(): + self._consume_task.cancel() + try: + self._consume_task.result() + except asyncio.CancelledError as e: + self.log.exception( + f"The consume task was canceled. Messages may be lost.", + exc_info=e + ) diff --git a/bot/constants.py b/bot/constants.py index 17e60a418..ead26c91d 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -356,6 +356,14 @@ class Channels(metaclass=YAMLGetter): verification: int +class Webhooks(metaclass=YAMLGetter): + section = "guild" + subsection = "webhooks" + + talent_pool: int + big_brother: int + + class Roles(metaclass=YAMLGetter): section = "guild" subsection = "roles" @@ -390,18 +398,12 @@ class Keys(metaclass=YAMLGetter): site_api: str -class RabbitMQ(metaclass=YAMLGetter): - section = "rabbitmq" - - host: str - password: str - port: int - username: str - - class URLs(metaclass=YAMLGetter): section = "urls" + # Snekbox endpoints + snekbox_eval_api: str + # Discord API endpoints discord_api: str discord_invite_api: str diff --git a/bot/converters.py b/bot/converters.py index 91f30ac5e..30ea7ca0f 100644 --- a/bot/converters.py +++ b/bot/converters.py @@ -1,6 +1,8 @@ import logging +from datetime import datetime from ssl import CertificateError +import dateparser import discord from aiohttp import ClientConnectorError from discord.ext.commands import BadArgument, Context, Converter @@ -151,3 +153,22 @@ class TagContentConverter(Converter): raise BadArgument("Tag contents should not be empty, or filled with whitespace.") return tag_content + + +class ExpirationDate(Converter): + DATEPARSER_SETTINGS = { + 'PREFER_DATES_FROM': 'future', + 'TIMEZONE': 'UTC', + 'TO_TIMEZONE': 'UTC' + } + + async def convert(self, ctx, expiration_string: str): + expiry = dateparser.parse(expiration_string, settings=self.DATEPARSER_SETTINGS) + if expiry is None: + raise BadArgument(f"Failed to parse expiration date from `{expiration_string}`") + + now = datetime.utcnow() + if expiry < now: + expiry = now + (now - expiry) + + return expiry diff --git a/bot/utils/messages.py b/bot/utils/messages.py index fc38b0127..94a8b36ed 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -1,9 +1,9 @@ import asyncio import contextlib from io import BytesIO -from typing import Sequence +from typing import Sequence, Union -from discord import Embed, File, Message, TextChannel +from discord import Embed, File, Message, TextChannel, Webhook from discord.abc import Snowflake from discord.errors import HTTPException @@ -78,9 +78,9 @@ async def wait_for_deletion( await message.delete() -async def send_attachments(message: Message, destination: TextChannel): +async def send_attachments(message: Message, destination: Union[TextChannel, Webhook]): """ - Re-uploads each attachment in a message to the given channel. + Re-uploads each attachment in a message to the given channel or webhook. Each attachment is sent as a separate message to more easily comply with the 8 MiB request size limit. If attachments are too large, they are instead grouped into a single embed which links to them. @@ -97,7 +97,16 @@ async def send_attachments(message: Message, destination: TextChannel): if attachment.size <= MAX_SIZE - 512: with BytesIO() as file: await attachment.save(file) - await destination.send(file=File(file, filename=attachment.filename)) + attachment_file = File(file, filename=attachment.filename) + + if isinstance(destination, TextChannel): + await destination.send(file=attachment_file) + else: + await destination.send( + file=attachment_file, + username=message.author.display_name, + avatar_url=message.author.avatar_url + ) else: large.append(attachment) except HTTPException as e: @@ -109,4 +118,11 @@ async def send_attachments(message: Message, destination: TextChannel): if large: embed = Embed(description=f"\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large)) embed.set_footer(text="Attachments exceed upload size limit.") - await destination.send(embed=embed) + if isinstance(destination, TextChannel): + await destination.send(embed=embed) + else: + await destination.send( + embed=embed, + username=message.author.display_name, + avatar_url=message.author.avatar_url + ) diff --git a/bot/utils/moderation.py b/bot/utils/moderation.py index 724b455bc..c1eb98dd6 100644 --- a/bot/utils/moderation.py +++ b/bot/utils/moderation.py @@ -1,11 +1,12 @@ import logging +from datetime import datetime from typing import Union from aiohttp import ClientError from discord import Member, Object, User from discord.ext.commands import Context -from bot.constants import Keys, URLs +from bot.constants import Keys log = logging.getLogger(__name__) @@ -13,33 +14,33 @@ HEADERS = {"X-API-KEY": Keys.site_api} async def post_infraction( - ctx: Context, user: Union[Member, Object, User], type: str, reason: str, duration: str = None, hidden: bool = False + ctx: Context, + user: Union[Member, Object, User], + type: str, + reason: str, + expires_at: datetime = None, + hidden: bool = False, + active: bool = True, ): payload = { - "type": type, + "actor": ctx.message.author.id, + "hidden": hidden, "reason": reason, - "user_id": str(user.id), - "actor_id": str(ctx.message.author.id), - "hidden": hidden + "type": type, + "user": user.id, + "active": active } - if duration: - payload['duration'] = duration + if expires_at: + payload['expires_at'] = expires_at.isoformat() try: - response = await ctx.bot.http_session.post( - URLs.site_infractions, - headers=HEADERS, - json=payload + response = await ctx.bot.api_client.post( + 'bot/infractions', json=payload ) except ClientError: log.exception("There was an error adding an infraction.") await ctx.send(":x: There was an error adding the infraction.") return - response_object = await response.json() - if "error_code" in response_object: - await ctx.send(f":x: There was an error adding the infraction: {response_object['error_message']}") - return - - return response_object + return response diff --git a/bot/utils/service_discovery.py b/bot/utils/service_discovery.py deleted file mode 100644 index 8d79096bd..000000000 --- a/bot/utils/service_discovery.py +++ /dev/null @@ -1,22 +0,0 @@ -import datetime -import socket -import time -from contextlib import closing - -from bot.constants import RabbitMQ - -THIRTY_SECONDS = datetime.timedelta(seconds=30) - - -def wait_for_rmq(): - start = datetime.datetime.now() - - while True: - if datetime.datetime.now() - start > THIRTY_SECONDS: - return False - - with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: - if sock.connect_ex((RabbitMQ.host, RabbitMQ.port)) == 0: - return True - - time.sleep(0.5) diff --git a/bot/utils/time.py b/bot/utils/time.py index 8e5d4e1bd..a330c9cd8 100644 --- a/bot/utils/time.py +++ b/bot/utils/time.py @@ -106,7 +106,7 @@ async def wait_until(time: datetime.datetime): :param time: A datetime.datetime object to wait until. """ - delay = time - datetime.datetime.now(tz=datetime.timezone.utc) + delay = time - datetime.datetime.utcnow() delay_seconds = delay.total_seconds() if delay_seconds > 1.0: diff --git a/config-default.yml b/config-default.yml index 91b4d6cce..2f5dcf5dc 100644 --- a/config-default.yml +++ b/config-default.yml @@ -136,6 +136,10 @@ guild: rockstars: &ROCKSTARS_ROLE 458226413825294336 team_leader: 501324292341104650 + webhooks: + talent_pool: 569145364800602132 + big_brother: 569133704568373283 + filter: @@ -228,13 +232,6 @@ keys: site_api: !ENV "BOT_API_KEY" -rabbitmq: - host: "pdrmq" - password: !ENV ["RABBITMQ_DEFAULT_PASS", "guest"] - port: 5672 - username: !ENV ["RABBITMQ_DEFAULT_USER", "guest"] - - urls: # PyDis site vars site: &DOMAIN "pythondiscord.com" @@ -262,6 +259,9 @@ urls: site_user_complete_api: !JOIN [*SCHEMA, *API, "/bot/users/complete"] paste_service: !JOIN [*SCHEMA, *PASTE, "/{key}"] + # Snekbox + snekbox_eval_api: "http://localhost:8060/eval" + # Env vars deploy: !ENV "DEPLOY_URL" status: !ENV "STATUS_URL" diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile deleted file mode 100644 index e46db756a..000000000 --- a/docker/base.Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.6-alpine3.7 - -RUN apk add --update tini -RUN apk add --update build-base -RUN apk add --update libffi-dev -RUN apk add --update zlib -RUN apk add --update jpeg-dev -RUN apk add --update libxml2 libxml2-dev libxslt-dev -RUN apk add --update zlib-dev -RUN apk add --update freetype-dev -RUN apk add --update git - -ENV LIBRARY_PATH=/lib:/usr/lib -ENV PIPENV_VENV_IN_PROJECT=1 -ENV PIPENV_IGNORE_VIRTUALENVS=1 -ENV PIPENV_NOSPIN=1 -ENV PIPENV_HIDE_EMOJIS=1 diff --git a/docker/bot.Dockerfile b/docker/bot.Dockerfile deleted file mode 100644 index 5a07a612b..000000000 --- a/docker/bot.Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM pythondiscord/bot-base:latest - -ENV PIPENV_VENV_IN_PROJECT=1 -ENV PIPENV_IGNORE_VIRTUALENVS=1 -ENV PIPENV_NOSPIN=1 -ENV PIPENV_HIDE_EMOJIS=1 - -RUN pip install -U pipenv - -RUN mkdir -p /bot -COPY . /bot -WORKDIR /bot - -RUN pipenv install --deploy - -ENTRYPOINT ["/sbin/tini", "--"] -CMD ["pipenv", "run", "start"] diff --git a/scripts/deploy-azure.sh b/scripts/deploy-azure.sh index 6b3dea508..af69ab46b 100644 --- a/scripts/deploy-azure.sh +++ b/scripts/deploy-azure.sh @@ -2,30 +2,11 @@ cd .. -# Build and deploy on master branch, only if not a pull request -if [[ ($BUILD_SOURCEBRANCHNAME == 'master') && ($SYSTEM_PULLREQUEST_PULLREQUESTID == '') ]]; then - changed_lines=$(git diff HEAD~1 HEAD docker/base.Dockerfile | wc -l) - - if [ $changed_lines != '0' ]; then - echo "base.Dockerfile was changed" - - echo "Building bot base" - docker build -t pythondiscord/bot-base:latest -f docker/base.Dockerfile . - - echo "Pushing image to Docker Hub" - docker push pythondiscord/bot-base:latest - else - echo "base.Dockerfile was not changed, not building" - fi - +# Build and deploy on django branch, only if not a pull request +if [[ ($BUILD_SOURCEBRANCHNAME == 'django') && ($SYSTEM_PULLREQUEST_PULLREQUESTID == '') ]]; then echo "Building image" - docker build -t pythondiscord/bot:latest -f docker/bot.Dockerfile . + docker build -t pythondiscord/bot:django . echo "Pushing image" - docker push pythondiscord/bot:latest - - echo "Deploying container" - curl -H "token: $1" $2 -else - echo "Skipping deploy" -fi
\ No newline at end of file + docker push pythondiscord/bot:django +fi diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/__init__.py diff --git a/tests/cogs/__init__.py b/tests/cogs/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/cogs/__init__.py diff --git a/tests/cogs/sync/__init__.py b/tests/cogs/sync/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/cogs/sync/__init__.py diff --git a/tests/cogs/sync/test_roles.py b/tests/cogs/sync/test_roles.py new file mode 100644 index 000000000..18682f39f --- /dev/null +++ b/tests/cogs/sync/test_roles.py @@ -0,0 +1,64 @@ +from bot.cogs.sync.syncers import Role, get_roles_for_sync + + +def test_get_roles_for_sync_empty_return_for_equal_roles(): + api_roles = {Role(id=41, name='name', colour=33, permissions=0x8)} + guild_roles = {Role(id=41, name='name', colour=33, permissions=0x8)} + + assert get_roles_for_sync(guild_roles, api_roles) == (set(), set()) + + +def test_get_roles_for_sync_returns_roles_to_update_with_non_id_diff(): + api_roles = {Role(id=41, name='old name', colour=35, permissions=0x8)} + guild_roles = {Role(id=41, name='new name', colour=33, permissions=0x8)} + + assert get_roles_for_sync(guild_roles, api_roles) == ( + set(), + guild_roles + ) + + +def test_get_roles_only_returns_roles_that_require_update(): + api_roles = { + Role(id=41, name='old name', colour=33, permissions=0x8), + Role(id=53, name='other role', colour=55, permissions=0) + } + guild_roles = { + Role(id=41, name='new name', colour=35, permissions=0x8), + Role(id=53, name='other role', colour=55, permissions=0) + } + + assert get_roles_for_sync(guild_roles, api_roles) == ( + set(), + {Role(id=41, name='new name', colour=35, permissions=0x8)}, + ) + + +def test_get_roles_returns_new_roles_in_first_tuple_element(): + api_roles = { + Role(id=41, name='name', colour=35, permissions=0x8), + } + guild_roles = { + Role(id=41, name='name', colour=35, permissions=0x8), + Role(id=53, name='other role', colour=55, permissions=0) + } + + assert get_roles_for_sync(guild_roles, api_roles) == ( + {Role(id=53, name='other role', colour=55, permissions=0)}, + set() + ) + + +def test_get_roles_returns_roles_to_update_and_new_roles(): + api_roles = { + Role(id=41, name='old name', colour=35, permissions=0x8), + } + guild_roles = { + Role(id=41, name='new name', colour=40, permissions=0x16), + Role(id=53, name='other role', colour=55, permissions=0) + } + + assert get_roles_for_sync(guild_roles, api_roles) == ( + {Role(id=53, name='other role', colour=55, permissions=0)}, + {Role(id=41, name='new name', colour=40, permissions=0x16)} + ) diff --git a/tests/cogs/sync/test_users.py b/tests/cogs/sync/test_users.py new file mode 100644 index 000000000..a863ae35b --- /dev/null +++ b/tests/cogs/sync/test_users.py @@ -0,0 +1,69 @@ +from bot.cogs.sync.syncers import User, get_users_for_sync + + +def fake_user(**kwargs): + kwargs.setdefault('id', 43) + kwargs.setdefault('name', 'bob the test man') + kwargs.setdefault('discriminator', 1337) + kwargs.setdefault('avatar_hash', None) + kwargs.setdefault('roles', (666,)) + kwargs.setdefault('in_guild', True) + return User(**kwargs) + + +def test_get_users_for_sync_returns_nothing_for_empty_params(): + assert get_users_for_sync({}, {}) == (set(), set()) + + +def test_get_users_for_sync_returns_nothing_for_equal_users(): + api_users = {43: fake_user()} + guild_users = {43: fake_user()} + + assert get_users_for_sync(guild_users, api_users) == (set(), set()) + + +def test_get_users_for_sync_returns_users_to_update_on_non_id_field_diff(): + api_users = {43: fake_user()} + guild_users = {43: fake_user(name='new fancy name')} + + assert get_users_for_sync(guild_users, api_users) == ( + set(), + {fake_user(name='new fancy name')} + ) + + +def test_get_users_for_sync_returns_users_to_create_with_new_ids_on_guild(): + api_users = {43: fake_user()} + guild_users = {43: fake_user(), 63: fake_user(id=63)} + + assert get_users_for_sync(guild_users, api_users) == ( + {fake_user(id=63)}, + set() + ) + + +def test_get_users_for_sync_updates_in_guild_field_on_user_leave(): + api_users = {43: fake_user(), 63: fake_user(id=63)} + guild_users = {43: fake_user()} + + assert get_users_for_sync(guild_users, api_users) == ( + set(), + {fake_user(id=63, in_guild=False)} + ) + + +def test_get_users_for_sync_updates_and_creates_users_as_needed(): + api_users = {43: fake_user()} + guild_users = {63: fake_user(id=63)} + + assert get_users_for_sync(guild_users, api_users) == ( + {fake_user(id=63)}, + {fake_user(in_guild=False)} + ) + + +def test_get_users_for_sync_does_not_duplicate_update_users(): + api_users = {43: fake_user(in_guild=False)} + guild_users = {} + + assert get_users_for_sync(guild_users, api_users) == (set(), set()) @@ -2,5 +2,5 @@ max-line-length=120 application_import_names=bot exclude=.cache,.venv -ignore=B311,W503,E226,S311 +ignore=B311,W503,E226,S311,T000 import-order-style=pycharm |