diff options
| author | 2020-06-22 08:42:12 +0800 | |
|---|---|---|
| committer | 2020-06-22 08:42:12 +0800 | |
| commit | 3e7e66ca0158a864dde825d8aeaad1b32c87b9ab (patch) | |
| tree | 1773e4de764f95254c89ddf7b18c67857c6ca211 | |
| parent | Logging Tests: Simplify `DEBUG_MODE` `False` test (diff) | |
| parent | Merge pull request #1009 from python-discord/bug/mod/bot-2a/webhook-clyde (diff) | |
Merge branch 'master' into logging-tests
Diffstat (limited to '')
68 files changed, 2732 insertions, 790 deletions
| diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..8760b35ec --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,32 @@ +name: "Code scanning - action" + +on: +  push: +  pull_request: +  schedule: +    - cron: '0 12 * * *' + +jobs: +  CodeQL-Build: + +    runs-on: ubuntu-latest + +    steps: +    - name: Checkout repository +      uses: actions/checkout@v2 +      with: +        fetch-depth: 2 + +    - run: git checkout HEAD^2 +      if: ${{ github.event_name == 'pull_request' }} + +    - name: Initialize CodeQL +      uses: github/codeql-action/init@v1 +      with: +        languages: python + +    - name: Autobuild +      uses: github/codeql-action/autobuild@v1 + +    - name: Perform CodeQL Analysis +      uses: github/codeql-action/analyze@v1 @@ -4,25 +4,27 @@ verify_ssl = true  name = "pypi"  [packages] -discord-py = "~=1.3.2" +aio-pika = "~=6.1"  aiodns = "~=2.0"  aiohttp = "~=3.5" -sphinx = "~=2.2" -markdownify = "~=0.4" -lxml = "~=4.4" -pyyaml = "~=5.1" +aioredis = "~=1.3.1" +beautifulsoup4 = "~=4.9" +colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} +coloredlogs = "~=14.0" +deepdiff = "~=4.0" +discord.py = "~=1.3.2" +fakeredis = "~=1.4" +feedparser = "~=5.2"  fuzzywuzzy = "~=0.17" -aio-pika = "~=6.1" +lxml = "~=4.4" +markdownify = "~=0.4" +more_itertools = "~=8.2"  python-dateutil = "~=2.8" -deepdiff = "~=4.0" +pyyaml = "~=5.1"  requests = "~=2.22" -more_itertools = "~=8.2"  sentry-sdk = "~=0.14" -coloredlogs = "~=14.0" -colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"} +sphinx = "~=2.2"  statsd = "~=3.3" -feedparser = "~=5.2" -beautifulsoup4 = "~=4.9"  [dev-packages]  coverage = "~=5.0" @@ -48,4 +50,5 @@ precommit = "pre-commit install"  build = "docker build -t pythondiscord/bot:latest -f Dockerfile ."  push = "docker push pythondiscord/bot:latest"  test = "coverage run -m unittest" +html = "coverage html"  report = "coverage report" diff --git a/Pipfile.lock b/Pipfile.lock index 4e7050a13..0e591710c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@  {      "_meta": {          "hash": { -            "sha256": "64620e7e825c74fd3010821fb30843b19f5dafb2b5a1f6eafedc0a5febd99b69" +            "sha256": "0297accc3d614d3da8080b89d56ef7fe489c28a0ada8102df396a604af7ee330"          },          "pipfile-spec": 6,          "requires": { @@ -18,11 +18,11 @@      "default": {          "aio-pika": {              "hashes": [ -                "sha256:9e4614636296e0040055bd6b304e97a38cc9796669ef391fc9b36649831d43ee", -                "sha256:c9d242b3c7142d64b185feb6c5cce4154962610e89ec2e9b52bd69ef01f89b2f" +                "sha256:c4cbbeb85b3c7bf81bc127371846cd949e6231717ce1e6ac7ee1dd5ede21f866", +                "sha256:ec7fef24f588d90314873463ab4f2c3debce0bd8830e49e3786586be96bc2e8e"              ],              "index": "pypi", -            "version": "==6.6.0" +            "version": "==6.6.1"          },          "aiodns": {              "hashes": [ @@ -50,12 +50,20 @@              "index": "pypi",              "version": "==3.6.2"          }, +        "aioredis": { +            "hashes": [ +                "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a", +                "sha256:b61808d7e97b7cd5a92ed574937a079c9387fdadd22bfbfa7ad2fd319ecc26e3" +            ], +            "index": "pypi", +            "version": "==1.3.1" +        },          "aiormq": {              "hashes": [ -                "sha256:286e0b0772075580466e45f98f051b9728a9316b9c36f0c14c7bc1409be375b0", -                "sha256:7ed7d6df6b57af7f8bce7d1ebcbdfc32b676192e46703e81e9e217316e56b5bd" +                "sha256:41a9d4eb17db805f30ed172f3f609fe0c2b16657fb15b1b67df19d251dd93c0d", +                "sha256:7c19477a9450824cb79f9949fd238f4148e2c0dca67756a2868863c387209f04"              ], -            "version": "==3.2.1" +            "version": "==3.2.2"          },          "alabaster": {              "hashes": [ @@ -87,12 +95,12 @@          },          "beautifulsoup4": {              "hashes": [ -                "sha256:594ca51a10d2b3443cbac41214e12dbb2a1cd57e1a7344659849e2e20ba6a8d8", -                "sha256:a4bbe77fd30670455c5296242967a123ec28c37e9702a8a81bd2f20a4baf0368", -                "sha256:d4e96ac9b0c3a6d3f0caae2e4124e6055c5dcafde8e2f831ff194c104f0775a0" +                "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7", +                "sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8", +                "sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c"              ],              "index": "pypi", -            "version": "==4.9.0" +            "version": "==4.9.1"          },          "certifi": {              "hashes": [ @@ -166,11 +174,19 @@              "index": "pypi",              "version": "==4.3.2"          }, -        "discord-py": { +        "discord": {              "hashes": [ -                "sha256:406871b06d86c3dc49fba63238519f28628dac946fef8a0e22988ff58ec05580" +                "sha256:9d4debb4a37845543bd4b92cb195bc53a302797333e768e70344222857ff1559", +                "sha256:ff6653655e342e7721dfb3f10421345fd852c2a33f2cca912b1c39b3778a9429"              ],              "index": "pypi", +            "version": "==1.0.1" +        }, +        "discord.py": { +            "hashes": [ +                "sha256:406871b06d86c3dc49fba63238519f28628dac946fef8a0e22988ff58ec05580", +                "sha256:ad00e34c72d2faa8db2157b651d05f3c415d7d05078e7e41dc9e8dc240051beb" +            ],              "version": "==1.3.3"          },          "docutils": { @@ -180,6 +196,14 @@              ],              "version": "==0.16"          }, +        "fakeredis": { +            "hashes": [ +                "sha256:4d170886865a91dbc8b7f8cbd4e5d488f4c5f2f25dfae127f001617bbe9e8f97", +                "sha256:647b2593d349d9d4e566c8dadb2e4c71ba35be5bdc4f1f7ac2d565a12a965053" +            ], +            "index": "pypi", +            "version": "==1.4.1" +        },          "feedparser": {              "hashes": [                  "sha256:bd030652c2d08532c034c27fcd7c85868e7fa3cb2b17f230a44a6bbc92519bf9", @@ -197,6 +221,51 @@              "index": "pypi",              "version": "==0.18.0"          }, +        "hiredis": { +            "hashes": [ +                "sha256:01b577f84c20ecc9c07fc4c184231b08e3c3942de096fa99978e053de231c423", +                "sha256:01ff0900134166961c9e339df77c33b72f7edc5cb41739f0babcd9faa345926e", +                "sha256:03ed34a13316d0c34213c4fd46e0fa3a5299073f4d4f08e93fed8c2108b399b3", +                "sha256:040436e91df5143aff9e0debb49530d0b17a6bd52200ce568621c31ef581b10d", +                "sha256:091eb38fbf968d1c5b703e412bbbd25f43a7967d8400842cee33a5a07b33c27b", +                "sha256:102f9b9dc6ed57feb3a7c9bdf7e71cb7c278fe8df1edfcfe896bc3e0c2be9447", +                "sha256:2b4b392c7e3082860c8371fab3ae762139090f9115819e12d9f56060f9ede05d", +                "sha256:2c9cc0b986397b833073f466e6b9e9c70d1d4dc2c2c1b3e9cae3a23102ff296c", +                "sha256:2fa65a9df683bca72073cd77709ddeb289ea2b114d3775d225fbbcc5faf808c5", +                "sha256:38437a681f17c975fd22349e72c29bc643f8e7eb2d6dc5df419eac59afa4d7ce", +                "sha256:3b3428fa3cf1ee178807b52c9bee8950ab94cd4eaa9bfae8c1bbae3c49501d34", +                "sha256:3dd8c2fae7f5494978facb0e93297dd627b1a3f536f3b070cf0a7d9157a07dcb", +                "sha256:4414a96c212e732723b5c3d7c04d386ebbb2ec359e1de646322cbc3f875cbd0d", +                "sha256:48c627581ad4ef60adbac980981407939acf13a0e18f093502c7b542223c4f19", +                "sha256:4a60e71625a2d78d8ab84dfb2fa2cfd9458c964b6e6c04fea76d9ade153fb371", +                "sha256:585ace09f434e43d8a8dbeb366865b1a044d7c06319b3c7372a0a00e63b860f4", +                "sha256:74b364b3f06c9cf0a53f7df611045bc9437ed972a283fa1f0b12537236d23ddc", +                "sha256:75c65c3850e89e9daa68d1b9bedd5806f177d60aa5a7b0953b4829481cfc1f72", +                "sha256:7f052de8bf744730a9120dbdc67bfeb7605a01f69fb8e7ba5c475af33c24e145", +                "sha256:8113a7d5e87ecf57cd4ae263cc9e429adb9a3e59f5a7768da5d3312a8d0a051a", +                "sha256:84857ce239eb8ed191ac78e77ff65d52902f00f30f4ee83bf80eb71da73b70e6", +                "sha256:8644a48ddc4a40b3e3a6b9443f396c2ee353afb2d45656c4fc68d04a82e8e3f7", +                "sha256:936aa565e673536e8a211e43ec43197406f24cd1f290138bd143765079c8ba00", +                "sha256:9afeb88c67bbc663b9f27385c496da056d06ad87f55df6e393e1516cfecb0461", +                "sha256:9d62cc7880110e4f83b0a51d218f465d3095e2751fbddd34e553dbd106a929ff", +                "sha256:a1fadd062fc8d647ff39220c57ea2b48c99bb73f18223828ec97f88fc27e7898", +                "sha256:a7754a783b1e5d6f627c19d099b178059c62f782ab62b4d8ba165b9fbc2ee34c", +                "sha256:aa59dd63bb3f736de4fc2d080114429d5d369dfb3265f771778e8349d67a97a4", +                "sha256:ae2ee0992f8de249715435942137843a93db204dd7db1e7cc9bdc5a8436443e8", +                "sha256:b36842d7cf32929d568f37ec5b3173b72b2ec6572dec4d6be6ce774762215aee", +                "sha256:bcbf9379c553b5facc6c04c1e5569b44b38ff16bcbf354676287698d61ee0c92", +                "sha256:cbccbda6f1c62ab460449d9c85fdf24d0d32a6bf45176581151e53cc26a5d910", +                "sha256:d0caf98dfb8af395d6732bd16561c0a2458851bea522e39f12f04802dbf6f502", +                "sha256:d6456afeddba036def1a36d8a2758eca53202308d83db20ab5d0b66590919627", +                "sha256:dbaef9a21a4f10bc281684ee4124f169e62bb533c2a92b55f8c06f64f9af7b8f", +                "sha256:dce84916c09aaece006272b37234ae84a8ed13abb3a4d341a23933b8701abfb5", +                "sha256:eb8c9c8b9869539d58d60ff4a28373a22514d40495911451343971cb4835b7a9", +                "sha256:efc98b14ee3a8595e40b1425e8d42f5fd26f11a7b215a81ef9259068931754f4", +                "sha256:fa2dc05b87d97acc1c6ae63f3e0f39eae5246565232484b08db6bf2dc1580678", +                "sha256:fe7d6ce9f6a5fbe24f09d95ea93e9c7271abc4e1565da511e1449b107b4d7848" +            ], +            "version": "==1.0.1" +        },          "humanfriendly": {              "hashes": [                  "sha256:bf52ec91244819c780341a3438d5d7b09f431d3f113a475147ac9b7b167a3d12", @@ -227,36 +296,36 @@          },          "lxml": {              "hashes": [ -                "sha256:06d4e0bbb1d62e38ae6118406d7cdb4693a3fa34ee3762238bcb96c9e36a93cd", -                "sha256:0701f7965903a1c3f6f09328c1278ac0eee8f56f244e66af79cb224b7ef3801c", -                "sha256:1f2c4ec372bf1c4a2c7e4bb20845e8bcf8050365189d86806bad1e3ae473d081", -                "sha256:4235bc124fdcf611d02047d7034164897ade13046bda967768836629bc62784f", -                "sha256:5828c7f3e615f3975d48f40d4fe66e8a7b25f16b5e5705ffe1d22e43fb1f6261", -                "sha256:585c0869f75577ac7a8ff38d08f7aac9033da2c41c11352ebf86a04652758b7a", -                "sha256:5d467ce9c5d35b3bcc7172c06320dddb275fea6ac2037f72f0a4d7472035cea9", -                "sha256:63dbc21efd7e822c11d5ddbedbbb08cd11a41e0032e382a0fd59b0b08e405a3a", -                "sha256:7bc1b221e7867f2e7ff1933165c0cec7153dce93d0cdba6554b42a8beb687bdb", -                "sha256:8620ce80f50d023d414183bf90cc2576c2837b88e00bea3f33ad2630133bbb60", -                "sha256:8a0ebda56ebca1a83eb2d1ac266649b80af8dd4b4a3502b2c1e09ac2f88fe128", -                "sha256:90ed0e36455a81b25b7034038e40880189169c308a3df360861ad74da7b68c1a", -                "sha256:95e67224815ef86924fbc2b71a9dbd1f7262384bca4bc4793645794ac4200717", -                "sha256:afdb34b715daf814d1abea0317b6d672476b498472f1e5aacbadc34ebbc26e89", -                "sha256:b4b2c63cc7963aedd08a5f5a454c9f67251b1ac9e22fd9d72836206c42dc2a72", -                "sha256:d068f55bda3c2c3fcaec24bd083d9e2eede32c583faf084d6e4b9daaea77dde8", -                "sha256:d5b3c4b7edd2e770375a01139be11307f04341ec709cf724e0f26ebb1eef12c3", -                "sha256:deadf4df349d1dcd7b2853a2c8796593cc346600726eff680ed8ed11812382a7", -                "sha256:df533af6f88080419c5a604d0d63b2c33b1c0c4409aba7d0cb6de305147ea8c8", -                "sha256:e4aa948eb15018a657702fee0b9db47e908491c64d36b4a90f59a64741516e77", -                "sha256:e5d842c73e4ef6ed8c1bd77806bf84a7cb535f9c0cf9b2c74d02ebda310070e1", -                "sha256:ebec08091a22c2be870890913bdadd86fcd8e9f0f22bcb398abd3af914690c15", -                "sha256:edc15fcfd77395e24543be48871c251f38132bb834d9fdfdad756adb6ea37679", -                "sha256:f2b74784ed7e0bc2d02bd53e48ad6ba523c9b36c194260b7a5045071abbb1012", -                "sha256:fa071559f14bd1e92077b1b5f6c22cf09756c6de7139370249eb372854ce51e6", -                "sha256:fd52e796fee7171c4361d441796b64df1acfceb51f29e545e812f16d023c4bbc", -                "sha256:fe976a0f1ef09b3638778024ab9fb8cde3118f203364212c198f71341c0715ca" -            ], -            "index": "pypi", -            "version": "==4.5.0" +                "sha256:06748c7192eab0f48e3d35a7adae609a329c6257495d5e53878003660dc0fec6", +                "sha256:0790ddca3f825dd914978c94c2545dbea5f56f008b050e835403714babe62a5f", +                "sha256:1aa7a6197c1cdd65d974f3e4953764eee3d9c7b67e3966616b41fab7f8f516b7", +                "sha256:22c6d34fdb0e65d5f782a4d1a1edb52e0a8365858dafb1c08cb1d16546cf0786", +                "sha256:2754d4406438c83144f9ffd3628bbe2dcc6d62b20dbc5c1ec4bc4385e5d44b42", +                "sha256:27ee0faf8077c7c1a589573b1450743011117f1aa1a91d5ae776bbc5ca6070f2", +                "sha256:2b02c106709466a93ed424454ce4c970791c486d5fcdf52b0d822a7e29789626", +                "sha256:2d1ddce96cf15f1254a68dba6935e6e0f1fe39247de631c115e84dd404a6f031", +                "sha256:4f282737d187ae723b2633856085c31ae5d4d432968b7f3f478a48a54835f5c4", +                "sha256:51bb4edeb36d24ec97eb3e6a6007be128b720114f9a875d6b370317d62ac80b9", +                "sha256:7eee37c1b9815e6505847aa5e68f192e8a1b730c5c7ead39ff317fde9ce29448", +                "sha256:7fd88cb91a470b383aafad554c3fe1ccf6dfb2456ff0e84b95335d582a799804", +                "sha256:9144ce36ca0824b29ebc2e02ca186e54040ebb224292072250467190fb613b96", +                "sha256:925baf6ff1ef2c45169f548cc85204433e061360bfa7d01e1be7ae38bef73194", +                "sha256:a636346c6c0e1092ffc202d97ec1843a75937d8c98aaf6771348ad6422e44bb0", +                "sha256:a87dbee7ad9dce3aaefada2081843caf08a44a8f52e03e0a4cc5819f8398f2f4", +                "sha256:a9e3b8011388e7e373565daa5e92f6c9cb844790dc18e43073212bb3e76f7007", +                "sha256:afb53edf1046599991fb4a7d03e601ab5f5422a5435c47ee6ba91ec3b61416a6", +                "sha256:b26719890c79a1dae7d53acac5f089d66fd8cc68a81f4e4bd355e45470dc25e1", +                "sha256:b7462cdab6fffcda853338e1741ce99706cdf880d921b5a769202ea7b94e8528", +                "sha256:b77975465234ff49fdad871c08aa747aae06f5e5be62866595057c43f8d2f62c", +                "sha256:c47a8a5d00060122ca5908909478abce7bbf62d812e3fc35c6c802df8fb01fe7", +                "sha256:c79e5debbe092e3c93ca4aee44c9a7631bdd407b2871cb541b979fd350bbbc29", +                "sha256:d8d40e0121ca1606aa9e78c28a3a7d88a05c06b3ca61630242cded87d8ce55fa", +                "sha256:ee2be8b8f72a2772e72ab926a3bccebf47bb727bda41ae070dc91d1fb759b726", +                "sha256:f95d28193c3863132b1f55c1056036bf580b5a488d908f7d22a04ace8935a3a9", +                "sha256:fadd2a63a2bfd7fb604508e553d1cf68eca250b2fbdbd81213b5f6f2fbf23529" +            ], +            "index": "pypi", +            "version": "==4.5.1"          },          "markdownify": {              "hashes": [ @@ -305,46 +374,46 @@          },          "more-itertools": {              "hashes": [ -                "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c", -                "sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507" +                "sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be", +                "sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982"              ],              "index": "pypi", -            "version": "==8.2.0" +            "version": "==8.3.0"          },          "multidict": {              "hashes": [ -                "sha256:317f96bc0950d249e96d8d29ab556d01dd38888fbe68324f46fd834b430169f1", -                "sha256:42f56542166040b4474c0c608ed051732033cd821126493cf25b6c276df7dd35", -                "sha256:4b7df040fb5fe826d689204f9b544af469593fb3ff3a069a6ad3409f742f5928", -                "sha256:544fae9261232a97102e27a926019100a9db75bec7b37feedd74b3aa82f29969", -                "sha256:620b37c3fea181dab09267cd5a84b0f23fa043beb8bc50d8474dd9694de1fa6e", -                "sha256:6e6fef114741c4d7ca46da8449038ec8b1e880bbe68674c01ceeb1ac8a648e78", -                "sha256:7774e9f6c9af3f12f296131453f7b81dabb7ebdb948483362f5afcaac8a826f1", -                "sha256:85cb26c38c96f76b7ff38b86c9d560dea10cf3459bb5f4caf72fc1bb932c7136", -                "sha256:a326f4240123a2ac66bb163eeba99578e9d63a8654a59f4688a79198f9aa10f8", -                "sha256:ae402f43604e3b2bc41e8ea8b8526c7fa7139ed76b0d64fc48e28125925275b2", -                "sha256:aee283c49601fa4c13adc64c09c978838a7e812f85377ae130a24d7198c0331e", -                "sha256:b51249fdd2923739cd3efc95a3d6c363b67bbf779208e9f37fd5e68540d1a4d4", -                "sha256:bb519becc46275c594410c6c28a8a0adc66fe24fef154a9addea54c1adb006f5", -                "sha256:c2c37185fb0af79d5c117b8d2764f4321eeb12ba8c141a95d0aa8c2c1d0a11dd", -                "sha256:dc561313279f9d05a3d0ffa89cd15ae477528ea37aa9795c4654588a3287a9ab", -                "sha256:e439c9a10a95cb32abd708bb8be83b2134fa93790a4fb0535ca36db3dda94d20", -                "sha256:fc3b4adc2ee8474cb3cd2a155305d5f8eda0a9c91320f83e55748e1fcb68f8e3" -            ], -            "version": "==4.7.5" +                "sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a", +                "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000", +                "sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2", +                "sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507", +                "sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5", +                "sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7", +                "sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d", +                "sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463", +                "sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19", +                "sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3", +                "sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b", +                "sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c", +                "sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87", +                "sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7", +                "sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430", +                "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255", +                "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d" +            ], +            "version": "==4.7.6"          },          "ordered-set": {              "hashes": [ -                "sha256:a7bfa858748c73b096e43db14eb23e2bc714a503f990c89fac8fab9b0ee79724" +                "sha256:a31008c57f9c9776b12eb8841b1f61d1e4d70dfbbe8875ccfa2403c54af3d51b"              ], -            "version": "==3.1.1" +            "version": "==4.0.1"          },          "packaging": {              "hashes": [ -                "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3", -                "sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752" +                "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", +                "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"              ], -            "version": "==20.3" +            "version": "==20.4"          },          "pamqp": {              "hashes": [ @@ -418,10 +487,10 @@          },          "pytz": {              "hashes": [ -                "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", -                "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" +                "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", +                "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048"              ], -            "version": "==2019.3" +            "version": "==2020.1"          },          "pyyaml": {              "hashes": [ @@ -440,6 +509,13 @@              "index": "pypi",              "version": "==5.3.1"          }, +        "redis": { +            "hashes": [ +                "sha256:2ef11f489003f151777c064c5dbc6653dfb9f3eade159bcadc524619fddc2242", +                "sha256:6d65e84bc58091140081ee9d9c187aab0480097750fac44239307a3bdf0b1251" +            ], +            "version": "==3.5.2" +        },          "requests": {              "hashes": [                  "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", @@ -450,18 +526,18 @@          },          "sentry-sdk": {              "hashes": [ -                "sha256:23808d571d2461a4ce3784ec12bbee5bdb8c026c143fe79d36cef8a6d653e71f", -                "sha256:bb90a4e19c7233a580715fc986cc44be2c48fc10b31e71580a2037e1c94b6950" +                "sha256:0e5e947d0f7a969314aa23669a94a9712be5a688ff069ff7b9fc36c66adc160c", +                "sha256:799a8bf76b012e3030a881be00e97bc0b922ce35dde699c6537122b751d80e2c"              ],              "index": "pypi", -            "version": "==0.14.3" +            "version": "==0.14.4"          },          "six": {              "hashes": [ -                "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", -                "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" +                "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", +                "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"              ], -            "version": "==1.14.0" +            "version": "==1.15.0"          },          "snowballstemmer": {              "hashes": [ @@ -470,12 +546,19 @@              ],              "version": "==2.0.0"          }, +        "sortedcontainers": { +            "hashes": [ +                "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", +                "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" +            ], +            "version": "==2.1.0" +        },          "soupsieve": {              "hashes": [ -                "sha256:e914534802d7ffd233242b785229d5ba0766a7f487385e3f714446a07bf540ae", -                "sha256:fcd71e08c0aee99aca1b73f45478549ee7e7fc006d51b37bec9e9def7dc22b69" +                "sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55", +                "sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"              ], -            "version": "==2.0" +            "version": "==2.0.1"          },          "sphinx": {              "hashes": [ @@ -595,10 +678,10 @@      "develop": {          "appdirs": {              "hashes": [ -                "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", -                "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" +                "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", +                "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"              ], -            "version": "==1.4.3" +            "version": "==1.4.4"          },          "attrs": {              "hashes": [ @@ -657,13 +740,6 @@              ],              "version": "==0.3.0"          }, -        "entrypoints": { -            "hashes": [ -                "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", -                "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" -            ], -            "version": "==0.3" -        },          "filelock": {              "hashes": [                  "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", @@ -673,11 +749,11 @@          },          "flake8": {              "hashes": [ -                "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb", -                "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca" +                "sha256:c69ac1668e434d37a2d2880b3ca9aafd54b3a10a3ac1ab101d22f29e29cf8634", +                "sha256:ccaa799ef9893cebe69fdfefed76865aeaefbb94cb8545617b2298786a4de9a5"              ],              "index": "pypi", -            "version": "==3.7.9" +            "version": "==3.8.2"          },          "flake8-annotations": {              "hashes": [ @@ -743,10 +819,10 @@          },          "identify": {              "hashes": [ -                "sha256:2bb8760d97d8df4408f4e805883dad26a2d076f04be92a10a3e43f09c6060742", -                "sha256:faffea0fd8ec86bb146ac538ac350ed0c73908326426d387eded0bcc9d077522" +                "sha256:0f3c3aac62b51b86fea6ff52fe8ff9e06f57f10411502443809064d23e16f1c2", +                "sha256:f9ad3d41f01e98eb066b6e05c5b184fd1e925fadec48eb165b4e01c72a1ef3a7"              ], -            "version": "==1.4.14" +            "version": "==1.4.16"          },          "mccabe": {              "hashes": [ @@ -771,18 +847,18 @@          },          "pre-commit": {              "hashes": [ -                "sha256:487c675916e6f99d355ec5595ad77b325689d423ef4839db1ed2f02f639c9522", -                "sha256:c0aa11bce04a7b46c5544723aedf4e81a4d5f64ad1205a30a9ea12d5e81969e1" +                "sha256:5559e09afcac7808933951ffaf4ff9aac524f31efbc3f24d021540b6c579813c", +                "sha256:703e2e34cbe0eedb0d319eff9f7b83e2022bb5a3ab5289a6a8841441076514d0"              ],              "index": "pypi", -            "version": "==2.2.0" +            "version": "==2.4.0"          },          "pycodestyle": {              "hashes": [ -                "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", -                "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" +                "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367", +                "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"              ], -            "version": "==2.5.0" +            "version": "==2.6.0"          },          "pydocstyle": {              "hashes": [ @@ -793,10 +869,10 @@          },          "pyflakes": {              "hashes": [ -                "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", -                "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" +                "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92", +                "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"              ], -            "version": "==2.1.1" +            "version": "==2.2.0"          },          "pyyaml": {              "hashes": [ @@ -817,10 +893,10 @@          },          "six": {              "hashes": [ -                "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", -                "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" +                "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", +                "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"              ], -            "version": "==1.14.0" +            "version": "==1.15.0"          },          "snowballstemmer": {              "hashes": [ @@ -831,10 +907,10 @@          },          "toml": {              "hashes": [ -                "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", -                "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" +                "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f", +                "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88"              ], -            "version": "==0.10.0" +            "version": "==0.10.1"          },          "unittest-xml-reporting": {              "hashes": [ @@ -846,10 +922,10 @@          },          "virtualenv": {              "hashes": [ -                "sha256:5021396e8f03d0d002a770da90e31e61159684db2859d0ba4850fbea752aa675", -                "sha256:ac53ade75ca189bc97b6c1d9ec0f1a50efe33cbf178ae09452dcd9fd309013c1" +                "sha256:a116629d4e7f4d03433b8afa27f43deba09d48bc48f5ecefa4f015a178efb6cf", +                "sha256:a730548b27366c5e6cbdf6f97406d861cccece2e22275e8e1a757aeff5e00c70"              ], -            "version": "==20.0.18" +            "version": "==20.0.21"          }      }  } diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d56675029..4500cb6e8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,6 +22,7 @@ jobs:        REDDIT_CLIENT_ID: spam        REDDIT_SECRET: ham        WOLFRAM_API_KEY: baz +      REDIS_PASSWORD: ''      steps:        - task: UsePythonVersion@0 diff --git a/bot/__main__.py b/bot/__main__.py index aa1d1aee8..4e0d4a111 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -3,7 +3,9 @@ import logging  import discord  import sentry_sdk  from discord.ext.commands import when_mentioned_or +from sentry_sdk.integrations.aiohttp import AioHttpIntegration  from sentry_sdk.integrations.logging import LoggingIntegration +from sentry_sdk.integrations.redis import RedisIntegration  from bot import constants, patches  from bot.bot import Bot @@ -15,7 +17,11 @@ sentry_logging = LoggingIntegration(  sentry_sdk.init(      dsn=constants.Bot.sentry_dsn, -    integrations=[sentry_logging] +    integrations=[ +        sentry_logging, +        AioHttpIntegration(), +        RedisIntegration(), +    ]  )  bot = Bot( diff --git a/bot/bot.py b/bot/bot.py index a85a22aa9..313652d11 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -5,7 +5,9 @@ import warnings  from typing import Optional  import aiohttp +import aioredis  import discord +import fakeredis.aioredis  from discord.ext import commands  from sentry_sdk import push_scope @@ -28,6 +30,9 @@ class Bot(commands.Bot):          super().__init__(*args, **kwargs)          self.http_session: Optional[aiohttp.ClientSession] = None +        self.redis_session: Optional[aioredis.Redis] = None +        self.redis_ready = asyncio.Event() +        self.redis_closed = False          self.api_client = api.APIClient(loop=self.loop)          self._connector = None @@ -44,6 +49,30 @@ class Bot(commands.Bot):          self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") +    async def _create_redis_session(self) -> None: +        """ +        Create the Redis connection pool, and then open the redis event gate. + +        If constants.Redis.use_fakeredis is True, we'll set up a fake redis pool instead +        of attempting to communicate with a real Redis server. This is useful because it +        means contributors don't necessarily need to get Redis running locally just +        to run the bot. + +        The fakeredis cache won't have persistence across restarts, but that +        usually won't matter for local bot testing. +        """ +        if constants.Redis.use_fakeredis: +            log.info("Using fakeredis instead of communicating with a real Redis server.") +            self.redis_session = await fakeredis.aioredis.create_redis_pool() +        else: +            self.redis_session = await aioredis.create_redis_pool( +                address=(constants.Redis.host, constants.Redis.port), +                password=constants.Redis.password, +            ) + +        self.redis_closed = False +        self.redis_ready.set() +      def add_cog(self, cog: commands.Cog) -> None:          """Adds a "cog" to the bot and logs the operation."""          super().add_cog(cog) @@ -78,6 +107,12 @@ class Bot(commands.Bot):          if self.stats._transport:              self.stats._transport.close() +        if self.redis_session: +            self.redis_closed = True +            self.redis_session.close() +            self.redis_ready.clear() +            await self.redis_session.wait_closed() +      async def login(self, *args, **kwargs) -> None:          """Re-create the connector and set up sessions before logging into Discord."""          self._recreate() @@ -85,7 +120,7 @@ class Bot(commands.Bot):          await super().login(*args, **kwargs)      def _recreate(self) -> None: -        """Re-create the connector, aiohttp session, and the APIClient.""" +        """Re-create the connector, aiohttp session, the APIClient and the Redis session."""          # Use asyncio for DNS resolution instead of threads so threads aren't spammed.          # Doesn't seem to have any state with regards to being closed, so no need to worry?          self._resolver = aiohttp.AsyncResolver() @@ -96,6 +131,14 @@ class Bot(commands.Bot):                  "The previous connector was not closed; it will remain open and be overwritten"              ) +        if self.redis_session and not self.redis_session.closed: +            log.warning( +                "The previous redis pool was not closed; it will remain open and be overwritten" +            ) + +        # Create the redis session +        self.loop.create_task(self._create_redis_session()) +          # Use AF_INET as its socket family to prevent HTTPS related problems both locally          # and in production.          self._connector = aiohttp.TCPConnector( diff --git a/bot/cogs/antimalware.py b/bot/cogs/antimalware.py index 66b5073e8..ea257442e 100644 --- a/bot/cogs/antimalware.py +++ b/bot/cogs/antimalware.py @@ -1,4 +1,5 @@  import logging +import typing as t  from os.path import splitext  from discord import Embed, Message, NotFound @@ -9,6 +10,27 @@ from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLE  log = logging.getLogger(__name__) +PY_EMBED_DESCRIPTION = ( +    "It looks like you tried to attach a Python file - " +    f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" +) + +TXT_EMBED_DESCRIPTION = ( +    "**Uh-oh!** It looks like your message got zapped by our spam filter. " +    "We currently don't allow `.txt` attachments, so here are some tips to help you travel safely: \n\n" +    "• If you attempted to send a message longer than 2000 characters, try shortening your message " +    "to fit within the character limit or use a pasting service (see below) \n\n" +    "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " +    "{cmd_channel_mention} for more information) or use a pasting service like: " +    f"\n\n{URLs.site_schema}{URLs.site_paste}" +) + +DISALLOWED_EMBED_DESCRIPTION = ( +    "It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " +    f"We currently allow the following file types: **{', '.join(AntiMalwareConfig.whitelist)}**.\n\n" +    "Feel free to ask in {meta_channel_mention} if you think this is a mistake." +) +  class AntiMalware(Cog):      """Delete messages which contain attachments with non-whitelisted file extensions.""" @@ -29,34 +51,20 @@ class AntiMalware(Cog):              return          embed = Embed() -        file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} -        extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) +        extensions_blocked = self.get_disallowed_extensions(message)          blocked_extensions_str = ', '.join(extensions_blocked)          if ".py" in extensions_blocked:              # Short-circuit on *.py files to provide a pastebin link -            embed.description = ( -                "It looks like you tried to attach a Python file - " -                f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" -            ) +            embed.description = PY_EMBED_DESCRIPTION          elif ".txt" in extensions_blocked:              # Work around Discord AutoConversion of messages longer than 2000 chars to .txt              cmd_channel = self.bot.get_channel(Channels.bot_commands) -            embed.description = ( -                "**Uh-oh!** It looks like your message got zapped by our spam filter. " -                "We currently don't allow `.txt` attachments, so here are some tips to help you travel safely: \n\n" -                "• If you attempted to send a message longer than 2000 characters, try shortening your message " -                "to fit within the character limit or use a pasting service (see below) \n\n" -                "• If you tried to show someone your code, you can use codeblocks \n(run `!code-blocks` in " -                f"{cmd_channel.mention} for more information) or use a pasting service like: " -                f"\n\n{URLs.site_schema}{URLs.site_paste}" -            ) +            embed.description = TXT_EMBED_DESCRIPTION.format(cmd_channel_mention=cmd_channel.mention)          elif extensions_blocked: -            whitelisted_types = ', '.join(AntiMalwareConfig.whitelist)              meta_channel = self.bot.get_channel(Channels.meta) -            embed.description = ( -                f"It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " -                f"We currently allow the following file types: **{whitelisted_types}**.\n\n" -                f"Feel free to ask in {meta_channel.mention} if you think this is a mistake." +            embed.description = DISALLOWED_EMBED_DESCRIPTION.format( +                blocked_extensions_str=blocked_extensions_str, +                meta_channel_mention=meta_channel.mention,              )          if embed.description: @@ -73,6 +81,13 @@ class AntiMalware(Cog):              except NotFound:                  log.info(f"Tried to delete message `{message.id}`, but message could not be found.") +    @classmethod +    def get_disallowed_extensions(cls, message: Message) -> t.Iterable[str]: +        """Get an iterable containing all the disallowed extensions of attachments.""" +        file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} +        extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) +        return extensions_blocked +  def setup(bot: Bot) -> None:      """Load the AntiMalware cog.""" diff --git a/bot/cogs/antispam.py b/bot/cogs/antispam.py index d63acbc4a..0bcca578d 100644 --- a/bot/cogs/antispam.py +++ b/bot/cogs/antispam.py @@ -94,7 +94,7 @@ class DeletionContext:          await modlog.send_log_message(              icon_url=Icons.filtering,              colour=Colour(Colours.soft_red), -            title=f"Spam detected!", +            title="Spam detected!",              text=mod_alert_message,              thumbnail=last_message.author.avatar_url_as(static_format="png"),              channel_id=Channels.mod_alerts, @@ -130,7 +130,7 @@ class AntiSpam(Cog):              body += "\n\n**The cog has been unloaded.**"              await self.mod_log.send_log_message( -                title=f"Error: AntiSpam configuration validation failed!", +                title="Error: AntiSpam configuration validation failed!",                  text=body,                  ping_everyone=True,                  icon_url=Icons.token_removed, diff --git a/bot/cogs/bot.py b/bot/cogs/bot.py index f6aea51c5..a79b37d25 100644 --- a/bot/cogs/bot.py +++ b/bot/cogs/bot.py @@ -326,6 +326,8 @@ class BotCog(Cog, name="Bot"):                              log.trace("The code consists only of expressions, not sending instructions")                      if howto != "": +                        # Increase amount of codeblock correction in stats +                        self.bot.stats.incr("codeblock_corrections")                          howto_embed = Embed(description=howto)                          bot_message = await msg.channel.send(f"Hey {msg.author.mention}!", embed=howto_embed)                          self.codeblock_message_ids[msg.id] = bot_message.id diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py index b5d9132cb..368d91c85 100644 --- a/bot/cogs/clean.py +++ b/bot/cogs/clean.py @@ -1,16 +1,16 @@  import logging  import random  import re -from typing import Optional +from typing import Iterable, Optional  from discord import Colour, Embed, Message, TextChannel, User +from discord.ext import commands  from discord.ext.commands import Cog, Context, group  from bot.bot import Bot  from bot.cogs.moderation import ModLog  from bot.constants import ( -    Channels, CleanMessages, Colours, Event, -    Icons, MODERATION_ROLES, NEGATIVE_REPLIES +    Channels, CleanMessages, Colours, Event, Icons, MODERATION_ROLES, NEGATIVE_REPLIES  )  from bot.decorators import with_role @@ -41,10 +41,10 @@ class Clean(Cog):          self,          amount: int,          ctx: Context, +        channels: Iterable[TextChannel],          bots_only: bool = False,          user: User = None,          regex: Optional[str] = None, -        channel: Optional[TextChannel] = None      ) -> None:          """A helper function that does the actual message cleaning."""          def predicate_bots_only(message: Message) -> bool: @@ -110,48 +110,39 @@ class Clean(Cog):              predicate = None                     # Delete all messages          # Default to using the invoking context's channel -        if not channel: -            channel = ctx.channel +        if not channels: +            channels = [ctx.channel] + +        # Delete the invocation first +        self.mod_log.ignore(Event.message_delete, ctx.message.id) +        await ctx.message.delete() -        # Look through the history and retrieve message data          messages = []          message_ids = []          self.cleaning = True -        invocation_deleted = False - -        # To account for the invocation message, we index `amount + 1` messages. -        async for message in channel.history(limit=amount + 1): -            # If at any point the cancel command is invoked, we should stop. -            if not self.cleaning: -                return +        # Find the IDs of the messages to delete. IDs are needed in order to ignore mod log events. +        for channel in channels: +            async for message in channel.history(limit=amount): -            # Always start by deleting the invocation -            if not invocation_deleted: -                self.mod_log.ignore(Event.message_delete, message.id) -                await message.delete() -                invocation_deleted = True -                continue +                # If at any point the cancel command is invoked, we should stop. +                if not self.cleaning: +                    return -            # If the message passes predicate, let's save it. -            if predicate is None or predicate(message): -                message_ids.append(message.id) -                messages.append(message) +                # If the message passes predicate, let's save it. +                if predicate is None or predicate(message): +                    message_ids.append(message.id)          self.cleaning = False -        # We should ignore the ID's we stored, so we don't get mod-log spam. +        # Now let's delete the actual messages with purge.          self.mod_log.ignore(Event.message_delete, *message_ids) - -        # Use bulk delete to actually do the cleaning. It's far faster. -        await channel.purge( -            limit=amount, -            check=predicate -        ) +        for channel in channels: +            messages += await channel.purge(limit=amount, check=predicate)          # Reverse the list to restore chronological order          if messages: -            messages = list(reversed(messages)) +            messages = reversed(messages)              log_url = await self.mod_log.upload_log(messages, ctx.author.id)          else:              # Can't build an embed, nothing to clean! @@ -163,8 +154,10 @@ class Clean(Cog):              return          # Build the embed and send it +        target_channels = ", ".join(channel.mention for channel in channels) +          message = ( -            f"**{len(message_ids)}** messages deleted in <#{channel.id}> by **{ctx.author.name}**\n\n" +            f"**{len(message_ids)}** messages deleted in {target_channels} by **{ctx.author.name}**\n\n"              f"A log of the deleted messages can be found [here]({log_url})."          ) @@ -189,10 +182,10 @@ class Clean(Cog):          ctx: Context,          user: User,          amount: Optional[int] = 10, -        channel: TextChannel = None +        channels: commands.Greedy[TextChannel] = None      ) -> None:          """Delete messages posted by the provided user, stop cleaning after traversing `amount` messages.""" -        await self._clean_messages(amount, ctx, user=user, channel=channel) +        await self._clean_messages(amount, ctx, user=user, channels=channels)      @clean_group.command(name="all", aliases=["everything"])      @with_role(*MODERATION_ROLES) @@ -200,10 +193,10 @@ class Clean(Cog):          self,          ctx: Context,          amount: Optional[int] = 10, -        channel: TextChannel = None +        channels: commands.Greedy[TextChannel] = None      ) -> None:          """Delete all messages, regardless of poster, stop cleaning after traversing `amount` messages.""" -        await self._clean_messages(amount, ctx, channel=channel) +        await self._clean_messages(amount, ctx, channels=channels)      @clean_group.command(name="bots", aliases=["bot"])      @with_role(*MODERATION_ROLES) @@ -211,10 +204,10 @@ class Clean(Cog):          self,          ctx: Context,          amount: Optional[int] = 10, -        channel: TextChannel = None +        channels: commands.Greedy[TextChannel] = None      ) -> None:          """Delete all messages posted by a bot, stop cleaning after traversing `amount` messages.""" -        await self._clean_messages(amount, ctx, bots_only=True, channel=channel) +        await self._clean_messages(amount, ctx, bots_only=True, channels=channels)      @clean_group.command(name="regex", aliases=["word", "expression"])      @with_role(*MODERATION_ROLES) @@ -223,10 +216,10 @@ class Clean(Cog):          ctx: Context,          regex: str,          amount: Optional[int] = 10, -        channel: TextChannel = None +        channels: commands.Greedy[TextChannel] = None      ) -> None:          """Delete all messages that match a certain regex, stop cleaning after traversing `amount` messages.""" -        await self._clean_messages(amount, ctx, regex=regex, channel=channel) +        await self._clean_messages(amount, ctx, regex=regex, channels=channels)      @clean_group.command(name="stop", aliases=["cancel", "abort"])      @with_role(*MODERATION_ROLES) diff --git a/bot/cogs/defcon.py b/bot/cogs/defcon.py index 25b0a6ad5..4c0ad5914 100644 --- a/bot/cogs/defcon.py +++ b/bot/cogs/defcon.py @@ -81,7 +81,7 @@ class Defcon(Cog):              else:                  self.enabled = False                  self.days = timedelta(days=0) -                log.info(f"DEFCON disabled") +                log.info("DEFCON disabled")              await self.update_channel_topic() diff --git a/bot/cogs/duck_pond.py b/bot/cogs/duck_pond.py index 1f84a0609..5b6a7fd62 100644 --- a/bot/cogs/duck_pond.py +++ b/bot/cogs/duck_pond.py @@ -7,7 +7,7 @@ from discord.ext.commands import Cog  from bot import constants  from bot.bot import Bot -from bot.utils.messages import send_attachments +from bot.utils.messages import send_attachments, sub_clyde  log = logging.getLogger(__name__) @@ -58,7 +58,7 @@ class DuckPond(Cog):          try:              await self.webhook.send(                  content=content, -                username=username, +                username=sub_clyde(username),                  avatar_url=avatar_url,                  embed=embed              ) @@ -117,7 +117,7 @@ class DuckPond(Cog):                      avatar_url=message.author.avatar_url                  )              except discord.HTTPException: -                log.exception(f"Failed to send an attachment to the webhook") +                log.exception("Failed to send an attachment to the webhook")          await message.add_reaction("✅") diff --git a/bot/cogs/error_handler.py b/bot/cogs/error_handler.py index 23d1eed82..5de961116 100644 --- a/bot/cogs/error_handler.py +++ b/bot/cogs/error_handler.py @@ -9,7 +9,7 @@ from bot.api import ResponseCodeError  from bot.bot import Bot  from bot.constants import Channels  from bot.converters import TagNameConverter -from bot.decorators import InWhitelistCheckFailure +from bot.utils.checks import InWhitelistCheckFailure  log = logging.getLogger(__name__) @@ -166,7 +166,7 @@ class ErrorHandler(Cog):              await prepared_help_command              self.bot.stats.incr("errors.missing_required_argument")          elif isinstance(e, errors.TooManyArguments): -            await ctx.send(f"Too many arguments provided.") +            await ctx.send("Too many arguments provided.")              await prepared_help_command              self.bot.stats.incr("errors.too_many_arguments")          elif isinstance(e, errors.BadArgument): @@ -206,7 +206,7 @@ class ErrorHandler(Cog):          if isinstance(e, bot_missing_errors):              ctx.bot.stats.incr("errors.bot_permission_error")              await ctx.send( -                f"Sorry, it looks like I don't have the permissions or roles I need to do that." +                "Sorry, it looks like I don't have the permissions or roles I need to do that."              )          elif isinstance(e, (InWhitelistCheckFailure, errors.NoPrivateMessage)):              ctx.bot.stats.incr("errors.wrong_channel_or_dm_error") diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 6a703f5a1..76ea68660 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -1,10 +1,13 @@ +import asyncio  import logging  import re -from typing import Optional, Union +from datetime import datetime, timedelta +from typing import List, Mapping, Optional, Union +import dateutil  import discord.errors  from dateutil.relativedelta import relativedelta -from discord import Colour, DMChannel, Member, Message, TextChannel +from discord import Colour, HTTPException, Member, Message, NotFound, TextChannel  from discord.ext.commands import Cog  from discord.utils import escape_markdown @@ -14,6 +17,9 @@ from bot.constants import (      Channels, Colours,      Filter, Icons, URLs  ) +from bot.utils.redis_cache import RedisCache +from bot.utils.scheduling import Scheduler +from bot.utils.time import wait_until  log = logging.getLogger(__name__) @@ -40,6 +46,8 @@ TOKEN_WATCHLIST_PATTERNS = [  ]  WATCHLIST_PATTERNS = WORD_WATCHLIST_PATTERNS + TOKEN_WATCHLIST_PATTERNS +DAYS_BETWEEN_ALERTS = 3 +  def expand_spoilers(text: str) -> str:      """Return a string containing all interpretations of a spoilered message.""" @@ -49,11 +57,20 @@ def expand_spoilers(text: str) -> str:      ) -class Filtering(Cog): +OFFENSIVE_MSG_DELETE_TIME = timedelta(days=Filter.offensive_msg_delete_days) + + +class Filtering(Cog, Scheduler):      """Filtering out invites, blacklisting domains, and warning us of certain regular expressions.""" +    # Redis cache mapping a user ID to the last timestamp a bad nickname alert was sent +    name_alerts = RedisCache() +      def __init__(self, bot: Bot):          self.bot = bot +        super().__init__() + +        self.name_lock = asyncio.Lock()          staff_mistake_str = "If you believe this was a mistake, please let staff know!"          self.filters = { @@ -66,7 +83,8 @@ class Filtering(Cog):                  "notification_msg": (                      "Your post has been removed for abusing Unicode character rendering (aka Zalgo text). "                      f"{staff_mistake_str}" -                ) +                ), +                "schedule_deletion": False              },              "filter_invites": {                  "enabled": Filter.filter_invites, @@ -77,7 +95,8 @@ class Filtering(Cog):                  "notification_msg": (                      f"Per Rule 6, your invite link has been removed. {staff_mistake_str}\n\n"                      r"Our server rules can be found here: <https://pythondiscord.com/pages/rules>" -                ) +                ), +                "schedule_deletion": False              },              "filter_domains": {                  "enabled": Filter.filter_domains, @@ -87,22 +106,27 @@ class Filtering(Cog):                  "user_notification": Filter.notify_user_domains,                  "notification_msg": (                      f"Your URL has been removed because it matched a blacklisted domain. {staff_mistake_str}" -                ) +                ), +                "schedule_deletion": False              },              "watch_regex": {                  "enabled": Filter.watch_regex,                  "function": self._has_watch_regex_match,                  "type": "watchlist",                  "content_only": True, +                "schedule_deletion": True              },              "watch_rich_embeds": {                  "enabled": Filter.watch_rich_embeds,                  "function": self._has_rich_embed,                  "type": "watchlist",                  "content_only": False, -            }, +                "schedule_deletion": False +            }          } +        self.bot.loop.create_task(self.reschedule_offensive_msg_deletion()) +      @property      def mod_log(self) -> ModLog:          """Get currently loaded ModLog cog instance.""" @@ -112,6 +136,7 @@ class Filtering(Cog):      async def on_message(self, msg: Message) -> None:          """Invoke message filter for new messages."""          await self._filter_message(msg) +        await self.check_bad_words_in_name(msg.author)      @Cog.listener()      async def on_message_edit(self, before: Message, after: Message) -> None: @@ -126,6 +151,55 @@ class Filtering(Cog):              delta = relativedelta(after.edited_at, before.edited_at).microseconds          await self._filter_message(after, delta) +    @staticmethod +    def get_name_matches(name: str) -> List[re.Match]: +        """Check bad words from passed string (name). Return list of matches.""" +        matches = [] +        for pattern in WATCHLIST_PATTERNS: +            if match := pattern.search(name): +                matches.append(match) +        return matches + +    async def check_send_alert(self, member: Member) -> bool: +        """When there is less than 3 days after last alert, return `False`, otherwise `True`.""" +        if last_alert := await self.name_alerts.get(member.id): +            last_alert = datetime.utcfromtimestamp(last_alert) +            if datetime.utcnow() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert: +                log.trace(f"Last alert was too recent for {member}'s nickname.") +                return False + +        return True + +    async def check_bad_words_in_name(self, member: Member) -> None: +        """Send a mod alert every 3 days if a username still matches a watchlist pattern.""" +        # Use lock to avoid race conditions +        async with self.name_lock: +            # Check whether the users display name contains any words in our blacklist +            matches = self.get_name_matches(member.display_name) + +            if not matches or not await self.check_send_alert(member): +                return + +            log.info(f"Sending bad nickname alert for '{member.display_name}' ({member.id}).") + +            log_string = ( +                f"**User:** {member.mention} (`{member.id}`)\n" +                f"**Display Name:** {member.display_name}\n" +                f"**Bad Matches:** {', '.join(match.group() for match in matches)}" +            ) + +            await self.mod_log.send_log_message( +                icon_url=Icons.token_removed, +                colour=Colours.soft_red, +                title="Username filtering alert", +                text=log_string, +                channel_id=Channels.mod_alerts, +                thumbnail=member.avatar_url +            ) + +            # Update time when alert sent +            await self.name_alerts.set(member.id, datetime.utcnow().timestamp()) +      async def _filter_message(self, msg: Message, delta: Optional[int] = None) -> None:          """Filter the input message to see if it violates any of our rules, and then respond accordingly."""          # Should we filter this message? @@ -161,8 +235,10 @@ class Filtering(Cog):                          match = await _filter["function"](msg)                      if match: -                        # If this is a filter (not a watchlist), we should delete the message. -                        if _filter["type"] == "filter": +                        is_private = msg.channel.type is discord.ChannelType.private + +                        # If this is a filter (not a watchlist) and not in a DM, delete the message. +                        if _filter["type"] == "filter" and not is_private:                              try:                                  # Embeds (can?) trigger both the `on_message` and `on_message_edit`                                  # event handlers, triggering filtering twice for the same message. @@ -181,7 +257,21 @@ class Filtering(Cog):                              if _filter["user_notification"]:                                  await self.notify_member(msg.author, _filter["notification_msg"], msg.channel) -                        if isinstance(msg.channel, DMChannel): +                        # If the message is classed as offensive, we store it in the site db and +                        # it will be deleted it after one week. +                        if _filter["schedule_deletion"] and not is_private: +                            delete_date = (msg.created_at + OFFENSIVE_MSG_DELETE_TIME).isoformat() +                            data = { +                                'id': msg.id, +                                'channel_id': msg.channel.id, +                                'delete_date': delete_date +                            } + +                            await self.bot.api_client.post('bot/offensive-messages', json=data) +                            self.schedule_task(msg.id, data) +                            log.trace(f"Offensive message {msg.id} will be deleted on {delete_date}") + +                        if is_private:                              channel_str = "via DM"                          else:                              channel_str = f"in {msg.channel.mention}" @@ -212,7 +302,9 @@ class Filtering(Cog):                          additional_embeds = None                          additional_embeds_msg = None -                        if filter_name == "filter_invites": +                        # The function returns True for invalid invites. +                        # They have no data so additional embeds can't be created for them. +                        if filter_name == "filter_invites" and match is not True:                              additional_embeds = []                              for invite, data in match.items():                                  embed = discord.Embed(description=( @@ -296,7 +388,7 @@ class Filtering(Cog):          Attempts to catch some of common ways to try to cheat the system.          """ -        # Remove backslashes to prevent escape character aroundfuckery like +        # Remove backslashes to prevent escape character around fuckery like          # discord\.gg/gdudes-pony-farm          text = text.replace("\\", "") @@ -365,6 +457,46 @@ class Filtering(Cog):          except discord.errors.Forbidden:              await channel.send(f"{filtered_member.mention} {reason}") +    async def _scheduled_task(self, msg: dict) -> None: +        """Delete an offensive message once its deletion date is reached.""" +        delete_at = dateutil.parser.isoparse(msg['delete_date']).replace(tzinfo=None) + +        await wait_until(delete_at) +        await self.delete_offensive_msg(msg) + +    async def reschedule_offensive_msg_deletion(self) -> None: +        """Get all the pending message deletion from the API and reschedule them.""" +        await self.bot.wait_until_ready() +        response = await self.bot.api_client.get('bot/offensive-messages',) + +        now = datetime.utcnow() + +        for msg in response: +            delete_at = dateutil.parser.isoparse(msg['delete_date']).replace(tzinfo=None) + +            if delete_at < now: +                await self.delete_offensive_msg(msg) +            else: +                self.schedule_task(msg['id'], msg) + +    async def delete_offensive_msg(self, msg: Mapping[str, str]) -> None: +        """Delete an offensive message, and then delete it from the db.""" +        try: +            channel = self.bot.get_channel(msg['channel_id']) +            if channel: +                msg_obj = await channel.fetch_message(msg['id']) +                await msg_obj.delete() +        except NotFound: +            log.info( +                f"Tried to delete message {msg['id']}, but the message can't be found " +                f"(it has been probably already deleted)." +            ) +        except HTTPException as e: +            log.warning(f"Failed to delete message {msg['id']}: status {e.status}") + +        await self.bot.api_client.delete(f'bot/offensive-messages/{msg["id"]}') +        log.info(f"Deleted the offensive message with id {msg['id']}.") +  def setup(bot: Bot) -> None:      """Load the Filtering cog.""" diff --git a/bot/cogs/help_channels.py b/bot/cogs/help_channels.py index 1bd1f9d68..187adfe51 100644 --- a/bot/cogs/help_channels.py +++ b/bot/cogs/help_channels.py @@ -5,8 +5,7 @@ import logging  import random  import typing as t  from collections import deque -from contextlib import suppress -from datetime import datetime +from datetime import datetime, timedelta, timezone  from pathlib import Path  import discord @@ -15,6 +14,7 @@ from discord.ext import commands  from bot import constants  from bot.bot import Bot +from bot.utils import RedisCache  from bot.utils.checks import with_role_check  from bot.utils.scheduling import Scheduler @@ -22,20 +22,10 @@ log = logging.getLogger(__name__)  ASKING_GUIDE_URL = "https://pythondiscord.com/pages/asking-good-questions/"  MAX_CHANNELS_PER_CATEGORY = 50 -EXCLUDED_CHANNELS = (constants.Channels.how_to_get_help,) +EXCLUDED_CHANNELS = (constants.Channels.how_to_get_help, constants.Channels.cooldown) -AVAILABLE_TOPIC = """ -This channel is available. Feel free to ask a question in order to claim this channel! -""" - -IN_USE_TOPIC = """ -This channel is currently in use. If you'd like to discuss a different problem, please claim a new \ -channel from the Help: Available category. -""" - -DORMANT_TOPIC = """ -This channel is temporarily archived. If you'd like to ask a question, please use one of the \ -channels in the Help: Available category. +HELP_CHANNEL_TOPIC = """ +This is a Python help channel. You can claim your own help channel in the Python Help: Available category.  """  AVAILABLE_MSG = f""" @@ -64,11 +54,6 @@ question to maximize your chance of getting a good answer. If you're not sure ho  through our guide for [asking a good question]({ASKING_GUIDE_URL}).  """ -AVAILABLE_EMOJI = "✅" -IN_USE_ANSWERED_EMOJI = "⌛" -IN_USE_UNANSWERED_EMOJI = "⏳" -NAME_SEPARATOR = "|" -  CoroutineFunc = t.Callable[..., t.Coroutine] @@ -114,13 +99,24 @@ class HelpChannels(Scheduler, commands.Cog):      Help channels are named after the chemical elements in `bot/resources/elements.json`.      """ +    # This cache tracks which channels are claimed by which members. +    # RedisCache[discord.TextChannel.id, t.Union[discord.User.id, discord.Member.id]] +    help_channel_claimants = RedisCache() + +    # This cache maps a help channel to whether it has had any +    # activity other than the original claimant. True being no other +    # activity and False being other activity. +    # RedisCache[discord.TextChannel.id, bool] +    unanswered = RedisCache() + +    # This dictionary maps a help channel to the time it was claimed +    # RedisCache[discord.TextChannel.id, UtcPosixTimestamp] +    claim_times = RedisCache() +      def __init__(self, bot: Bot):          super().__init__()          self.bot = bot -        self.help_channel_claimants: ( -            t.Dict[discord.TextChannel, t.Union[discord.Member, discord.User]] -        ) = {}          # Categories          self.available_category: discord.CategoryChannel = None @@ -140,16 +136,6 @@ class HelpChannels(Scheduler, commands.Cog):          self.on_message_lock = asyncio.Lock()          self.init_task = self.bot.loop.create_task(self.init_cog()) -        # Stats - -        # This dictionary maps a help channel to the time it was claimed -        self.claim_times: t.Dict[int, datetime] = {} - -        # This dictionary maps a help channel to whether it has had any -        # activity other than the original claimant. True being no other -        # activity and False being other activity. -        self.unanswered: t.Dict[int, bool] = {} -      def cog_unload(self) -> None:          """Cancel the init task and scheduled tasks when the cog unloads."""          log.trace("Cog unload: cancelling the init_cog task") @@ -196,7 +182,7 @@ class HelpChannels(Scheduler, commands.Cog):              return None          log.debug(f"Creating a new dormant channel named {name}.") -        return await self.dormant_category.create_text_channel(name) +        return await self.dormant_category.create_text_channel(name, topic=HELP_CHANNEL_TOPIC)      def create_name_queue(self) -> deque:          """Return a queue of element names to use for creating new channels.""" @@ -212,7 +198,7 @@ class HelpChannels(Scheduler, commands.Cog):      async def dormant_check(self, ctx: commands.Context) -> bool:          """Return True if the user is the help channel claimant or passes the role check.""" -        if self.help_channel_claimants.get(ctx.channel) == ctx.author: +        if await self.help_channel_claimants.get(ctx.channel.id) == ctx.author.id:              log.trace(f"{ctx.author} is the help channel claimant, passing the check for dormant.")              self.bot.stats.incr("help.dormant_invoke.claimant")              return True @@ -237,10 +223,11 @@ class HelpChannels(Scheduler, commands.Cog):          log.trace("close command invoked; checking if the channel is in-use.")          if ctx.channel.category == self.in_use_category:              if await self.dormant_check(ctx): -                with suppress(KeyError): -                    del self.help_channel_claimants[ctx.channel] +                # Remove the claimant and the cooldown role +                await self.help_channel_claimants.delete(ctx.channel.id)                  await self.remove_cooldown_role(ctx.author) +                  # Ignore missing task when cooldown has passed but the channel still isn't dormant.                  self.cancel_task(ctx.author.id, ignore_missing=True) @@ -299,6 +286,15 @@ class HelpChannels(Scheduler, commands.Cog):              if channel.category_id == category.id and not self.is_excluded_channel(channel):                  yield channel +    async def get_in_use_time(self, channel_id: int) -> t.Optional[timedelta]: +        """Return the duration `channel_id` has been in use. Return None if it's not in use.""" +        log.trace(f"Calculating in use time for channel {channel_id}.") + +        claimed_timestamp = await self.claim_times.get(channel_id) +        if claimed_timestamp: +            claimed = datetime.utcfromtimestamp(claimed_timestamp) +            return datetime.utcnow() - claimed +      @staticmethod      def get_names() -> t.List[str]:          """ @@ -391,7 +387,7 @@ class HelpChannels(Scheduler, commands.Cog):              self.in_use_category = await self.try_get_channel(constants.Categories.help_in_use)              self.dormant_category = await self.try_get_channel(constants.Categories.help_dormant)          except discord.HTTPException: -            log.exception(f"Failed to get a category; cog will be removed") +            log.exception("Failed to get a category; cog will be removed")              self.bot.remove_cog(self.qualified_name)      async def init_cog(self) -> None: @@ -401,7 +397,7 @@ class HelpChannels(Scheduler, commands.Cog):          log.trace("Initialising the cog.")          await self.init_categories() -        await self.reset_send_permissions() +        await self.check_cooldowns()          self.channel_queue = self.create_channel_queue()          self.name_queue = self.create_name_queue() @@ -438,13 +434,13 @@ class HelpChannels(Scheduler, commands.Cog):          """Return True if `member` has the 'Help Cooldown' role."""          return any(constants.Roles.help_cooldown == role.id for role in member.roles) -    def is_dormant_message(self, message: t.Optional[discord.Message]) -> bool: -        """Return True if the contents of the `message` match `DORMANT_MSG`.""" +    def match_bot_embed(self, message: t.Optional[discord.Message], description: str) -> bool: +        """Return `True` if the bot's `message`'s embed description matches `description`."""          if not message or not message.embeds:              return False          embed = message.embeds[0] -        return message.author == self.bot.user and embed.description.strip() == DORMANT_MSG.strip() +        return message.author == self.bot.user and embed.description.strip() == description.strip()      @staticmethod      def is_in_category(channel: discord.TextChannel, category_id: int) -> bool: @@ -461,7 +457,11 @@ class HelpChannels(Scheduler, commands.Cog):          """          log.trace(f"Handling in-use channel #{channel} ({channel.id}).") -        idle_seconds = constants.HelpChannels.idle_minutes * 60 +        if not await self.is_empty(channel): +            idle_seconds = constants.HelpChannels.idle_minutes * 60 +        else: +            idle_seconds = constants.HelpChannels.deleted_idle_minutes * 60 +          time_elapsed = await self.get_idle_time(channel)          if time_elapsed is None or time_elapsed >= idle_seconds: @@ -538,8 +538,6 @@ class HelpChannels(Scheduler, commands.Cog):          await self.move_to_bottom_position(              channel=channel,              category_id=constants.Categories.help_available, -            name=f"{AVAILABLE_EMOJI}{NAME_SEPARATOR}{self.get_clean_channel_name(channel)}", -            topic=AVAILABLE_TOPIC,          )          self.report_stats() @@ -555,25 +553,21 @@ class HelpChannels(Scheduler, commands.Cog):          await self.move_to_bottom_position(              channel=channel,              category_id=constants.Categories.help_dormant, -            name=self.get_clean_channel_name(channel), -            topic=DORMANT_TOPIC,          )          self.bot.stats.incr(f"help.dormant_calls.{caller}") -        if channel.id in self.claim_times: -            claimed = self.claim_times[channel.id] -            in_use_time = datetime.now() - claimed +        in_use_time = await self.get_in_use_time(channel.id) +        if in_use_time:              self.bot.stats.timing("help.in_use_time", in_use_time) -        if channel.id in self.unanswered: -            if self.unanswered[channel.id]: -                self.bot.stats.incr("help.sessions.unanswered") -            else: -                self.bot.stats.incr("help.sessions.answered") +        unanswered = await self.unanswered.get(channel.id) +        if unanswered: +            self.bot.stats.incr("help.sessions.unanswered") +        elif unanswered is not None: +            self.bot.stats.incr("help.sessions.answered")          log.trace(f"Position of #{channel} ({channel.id}) is actually {channel.position}.") -          log.trace(f"Sending dormant message for #{channel} ({channel.id}).")          embed = discord.Embed(description=DORMANT_MSG)          await channel.send(embed=embed) @@ -589,8 +583,6 @@ class HelpChannels(Scheduler, commands.Cog):          await self.move_to_bottom_position(              channel=channel,              category_id=constants.Categories.help_in_use, -            name=f"{IN_USE_UNANSWERED_EMOJI}{NAME_SEPARATOR}{self.get_clean_channel_name(channel)}", -            topic=IN_USE_TOPIC,          )          timeout = constants.HelpChannels.idle_minutes * 60 @@ -654,19 +646,17 @@ class HelpChannels(Scheduler, commands.Cog):          if self.is_in_category(channel, constants.Categories.help_in_use):              log.trace(f"Checking if #{channel} ({channel.id}) has been answered.") -            # Check if there is an entry in unanswered (does not persist across restarts) -            if channel.id in self.unanswered: -                claimant_id = self.help_channel_claimants[channel].id +            # Check if there is an entry in unanswered +            if await self.unanswered.contains(channel.id): +                claimant_id = await self.help_channel_claimants.get(channel.id) +                if not claimant_id: +                    # The mapping for this channel doesn't exist, we can't do anything. +                    return                  # Check the message did not come from the claimant                  if claimant_id != message.author.id:                      # Mark the channel as answered -                    self.unanswered[channel.id] = False - -                    # Change the emoji in the channel name to signify activity -                    log.trace(f"#{channel} ({channel.id}) has been answered; changing its emoji") -                    name = self.get_clean_channel_name(channel) -                    await channel.edit(name=f"{IN_USE_ANSWERED_EMOJI}{NAME_SEPARATOR}{name}") +                    await self.unanswered.set(channel.id, False)      @commands.Cog.listener()      async def on_message(self, message: discord.Message) -> None: @@ -699,12 +689,15 @@ class HelpChannels(Scheduler, commands.Cog):              await self.move_to_in_use(channel)              await self.revoke_send_permissions(message.author)              # Add user with channel for dormant check. -            self.help_channel_claimants[channel] = message.author +            await self.help_channel_claimants.set(channel.id, message.author.id)              self.bot.stats.incr("help.claimed") -            self.claim_times[channel.id] = datetime.now() -            self.unanswered[channel.id] = True +            # Must use a timezone-aware datetime to ensure a correct POSIX timestamp. +            timestamp = datetime.now(timezone.utc).timestamp() +            await self.claim_times.set(channel.id, timestamp) + +            await self.unanswered.set(channel.id, True)              log.trace(f"Releasing on_message lock for {message.id}.") @@ -713,15 +706,54 @@ class HelpChannels(Scheduler, commands.Cog):          # be put in the queue.          await self.move_to_available() -    async def reset_send_permissions(self) -> None: -        """Reset send permissions in the Available category for claimants.""" -        log.trace("Resetting send permissions in the Available category.") +    @commands.Cog.listener() +    async def on_message_delete(self, msg: discord.Message) -> None: +        """ +        Reschedule an in-use channel to become dormant sooner if the channel is empty. + +        The new time for the dormant task is configured with `HelpChannels.deleted_idle_minutes`. +        """ +        if not self.is_in_category(msg.channel, constants.Categories.help_in_use): +            return + +        if not await self.is_empty(msg.channel): +            return + +        log.info(f"Claimant of #{msg.channel} ({msg.author}) deleted message, channel is empty now. Rescheduling task.") + +        # Cancel existing dormant task before scheduling new. +        self.cancel_task(msg.channel.id) + +        task = TaskData(constants.HelpChannels.deleted_idle_minutes * 60, self.move_idle_channel(msg.channel)) +        self.schedule_task(msg.channel.id, task) + +    async def is_empty(self, channel: discord.TextChannel) -> bool: +        """Return True if the most recent message in `channel` is the bot's `AVAILABLE_MSG`.""" +        msg = await self.get_last_message(channel) +        return self.match_bot_embed(msg, AVAILABLE_MSG) + +    async def check_cooldowns(self) -> None: +        """Remove expired cooldowns and re-schedule active ones.""" +        log.trace("Checking all cooldowns to remove or re-schedule them.")          guild = self.bot.get_guild(constants.Guild.id) +        cooldown = constants.HelpChannels.claim_minutes * 60 + +        for channel_id, member_id in await self.help_channel_claimants.items(): +            member = guild.get_member(member_id) +            if not member: +                continue  # Member probably left the guild. + +            in_use_time = await self.get_in_use_time(channel_id) -        # TODO: replace with a persistent cache cause checking every member is quite slow -        for member in guild.members: -            if self.is_claimant(member): +            if not in_use_time or in_use_time.seconds > cooldown: +                # Remove the role if no claim time could be retrieved or if the cooldown expired. +                # Since the channel is in the claimants cache, it is definitely strange for a time +                # to not exist. However, it isn't a reason to keep the user stuck with a cooldown.                  await self.remove_cooldown_role(member) +            else: +                # The member is still on a cooldown; re-schedule it for the remaining time. +                remaining = cooldown - in_use_time.seconds +                await self.schedule_cooldown_expiration(member, remaining)      async def add_cooldown_role(self, member: discord.Member) -> None:          """Add the help cooldown role to `member`.""" @@ -774,11 +806,14 @@ class HelpChannels(Scheduler, commands.Cog):          # Would mean the user somehow bypassed the lack of permissions (e.g. user is guild owner).          self.cancel_task(member.id, ignore_missing=True) -        timeout = constants.HelpChannels.claim_minutes * 60 -        callback = self.remove_cooldown_role(member) +        await self.schedule_cooldown_expiration(member, constants.HelpChannels.claim_minutes * 60) -        log.trace(f"Scheduling {member}'s ({member.id}) send message permissions to be reinstated.") -        self.schedule_task(member.id, TaskData(timeout, callback)) +    async def schedule_cooldown_expiration(self, member: discord.Member, seconds: int) -> None: +        """Schedule the cooldown role for `member` to be removed after a duration of `seconds`.""" +        log.trace(f"Scheduling removal of {member}'s ({member.id}) cooldown.") + +        callback = self.remove_cooldown_role(member) +        self.schedule_task(member.id, TaskData(seconds, callback))      async def send_available_message(self, channel: discord.TextChannel) -> None:          """Send the available message by editing a dormant message or sending a new message.""" @@ -788,7 +823,7 @@ class HelpChannels(Scheduler, commands.Cog):          embed = discord.Embed(description=AVAILABLE_MSG)          msg = await self.get_last_message(channel) -        if self.is_dormant_message(msg): +        if self.match_bot_embed(msg, DORMANT_MSG):              log.trace(f"Found dormant message {msg.id} in {channel_info}; editing it.")              await msg.edit(embed=embed)          else: diff --git a/bot/cogs/information.py b/bot/cogs/information.py index ef2f308ca..f0bd1afdb 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -6,15 +6,16 @@ from collections import Counter, defaultdict  from string import Template  from typing import Any, Mapping, Optional, Union -from discord import Colour, Embed, Member, Message, Role, Status, utils +from discord import ChannelType, Colour, Embed, Guild, Member, Message, Role, Status, utils +from discord.abc import GuildChannel  from discord.ext.commands import BucketType, Cog, Context, Paginator, command, group  from discord.utils import escape_markdown  from bot import constants  from bot.bot import Bot -from bot.decorators import InWhitelistCheckFailure, in_whitelist, with_role +from bot.decorators import in_whitelist, with_role  from bot.pagination import LinePaginator -from bot.utils.checks import cooldown_with_role_bypass, with_role_check +from bot.utils.checks import InWhitelistCheckFailure, cooldown_with_role_bypass, with_role_check  from bot.utils.time import time_since  log = logging.getLogger(__name__) @@ -26,6 +27,49 @@ class Information(Cog):      def __init__(self, bot: Bot):          self.bot = bot +    @staticmethod +    def role_can_read(channel: GuildChannel, role: Role) -> bool: +        """Return True if `role` can read messages in `channel`.""" +        overwrites = channel.overwrites_for(role) +        return overwrites.read_messages is True + +    def get_staff_channel_count(self, guild: Guild) -> int: +        """ +        Get the number of channels that are staff-only. + +        We need to know two things about a channel: +        - Does the @everyone role have explicit read deny permissions? +        - Do staff roles have explicit read allow permissions? + +        If the answer to both of these questions is yes, it's a staff channel. +        """ +        channel_ids = set() +        for channel in guild.channels: +            if channel.type is ChannelType.category: +                continue + +            everyone_can_read = self.role_can_read(channel, guild.default_role) + +            for role in constants.STAFF_ROLES: +                role_can_read = self.role_can_read(channel, guild.get_role(role)) +                if role_can_read and not everyone_can_read: +                    channel_ids.add(channel.id) +                    break + +        return len(channel_ids) + +    @staticmethod +    def get_channel_type_counts(guild: Guild) -> str: +        """Return the total amounts of the various types of channels in `guild`.""" +        channel_counter = Counter(c.type for c in guild.channels) +        channel_type_list = [] +        for channel, count in channel_counter.items(): +            channel_type = str(channel).title() +            channel_type_list.append(f"{channel_type} channels: {count}") + +        channel_type_list = sorted(channel_type_list) +        return "\n".join(channel_type_list) +      @with_role(*constants.MODERATION_ROLES)      @command(name="roles")      async def roles_info(self, ctx: Context) -> None: @@ -102,15 +146,16 @@ class Information(Cog):          roles = len(ctx.guild.roles)          member_count = ctx.guild.member_count - -        # How many of each type of channel? -        channels = Counter(c.type for c in ctx.guild.channels) -        channel_counts = "".join(sorted(f"{str(ch).title()} channels: {channels[ch]}\n" for ch in channels)).strip() +        channel_counts = self.get_channel_type_counts(ctx.guild)          # How many of each user status?          statuses = Counter(member.status for member in ctx.guild.members)          embed = Embed(colour=Colour.blurple()) +        # How many staff members and staff channels do we have? +        staff_member_count = len(ctx.guild.get_role(constants.Roles.helpers).members) +        staff_channel_count = self.get_staff_channel_count(ctx.guild) +          # Because channel_counts lacks leading whitespace, it breaks the dedent if it's inserted directly by the          # f-string. While this is correctly formated by Discord, it makes unit testing difficult. To keep the formatting          # without joining a tuple of strings we can use a Template string to insert the already-formatted channel_counts @@ -122,12 +167,16 @@ class Information(Cog):                  Voice region: {region}                  Features: {features} -                **Counts** +                **Channel counts** +                $channel_counts +                Staff channels: {staff_channel_count} + +                **Member counts**                  Members: {member_count:,} +                Staff members: {staff_member_count}                  Roles: {roles} -                $channel_counts -                **Members** +                **Member statuses**                  {constants.Emojis.status_online} {statuses[Status.online]:,}                  {constants.Emojis.status_idle} {statuses[Status.idle]:,}                  {constants.Emojis.status_dnd} {statuses[Status.dnd]:,} diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index e62a36c43..3b28526b2 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -1,4 +1,5 @@  import logging +import textwrap  import typing as t  import discord @@ -52,7 +53,7 @@ class Infractions(InfractionScheduler, commands.Cog):      # region: Permanent infractions      @command() -    async def warn(self, ctx: Context, user: Member, *, reason: str = None) -> None: +    async def warn(self, ctx: Context, user: Member, *, reason: t.Optional[str] = None) -> None:          """Warn a user for the given reason."""          infraction = await utils.post_infraction(ctx, user, "warning", reason, active=False)          if infraction is None: @@ -61,12 +62,12 @@ class Infractions(InfractionScheduler, commands.Cog):          await self.apply_infraction(ctx, infraction, user)      @command() -    async def kick(self, ctx: Context, user: Member, *, reason: str = None) -> None: +    async def kick(self, ctx: Context, user: Member, *, reason: t.Optional[str] = None) -> None:          """Kick a user for the given reason."""          await self.apply_kick(ctx, user, reason, active=False)      @command() -    async def ban(self, ctx: Context, user: FetchedMember, *, reason: str = None) -> None: +    async def ban(self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None) -> None:          """Permanently ban a user for the given reason and stop watching them with Big Brother."""          await self.apply_ban(ctx, user, reason) @@ -74,7 +75,7 @@ class Infractions(InfractionScheduler, commands.Cog):      # region: Temporary infractions      @command(aliases=["mute"]) -    async def tempmute(self, ctx: Context, user: Member, duration: Expiry, *, reason: str = None) -> None: +    async def tempmute(self, ctx: Context, user: Member, duration: Expiry, *, reason: t.Optional[str] = None) -> None:          """          Temporarily mute a user for the given reason and duration. @@ -93,7 +94,14 @@ class Infractions(InfractionScheduler, commands.Cog):          await self.apply_mute(ctx, user, reason, expires_at=duration)      @command() -    async def tempban(self, ctx: Context, user: FetchedMember, duration: Expiry, *, reason: str = None) -> None: +    async def tempban( +        self, +        ctx: Context, +        user: FetchedMember, +        duration: Expiry, +        *, +        reason: t.Optional[str] = None +    ) -> None:          """          Temporarily ban a user for the given reason and duration. @@ -115,7 +123,7 @@ class Infractions(InfractionScheduler, commands.Cog):      # region: Permanent shadow infractions      @command(hidden=True) -    async def note(self, ctx: Context, user: FetchedMember, *, reason: str = None) -> None: +    async def note(self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None) -> None:          """Create a private note for a user with the given reason without notifying the user."""          infraction = await utils.post_infraction(ctx, user, "note", reason, hidden=True, active=False)          if infraction is None: @@ -124,12 +132,12 @@ class Infractions(InfractionScheduler, commands.Cog):          await self.apply_infraction(ctx, infraction, user)      @command(hidden=True, aliases=['shadowkick', 'skick']) -    async def shadow_kick(self, ctx: Context, user: Member, *, reason: str = None) -> None: +    async def shadow_kick(self, ctx: Context, user: Member, *, reason: t.Optional[str] = None) -> None:          """Kick a user for the given reason without notifying the user."""          await self.apply_kick(ctx, user, reason, hidden=True, active=False)      @command(hidden=True, aliases=['shadowban', 'sban']) -    async def shadow_ban(self, ctx: Context, user: FetchedMember, *, reason: str = None) -> None: +    async def shadow_ban(self, ctx: Context, user: FetchedMember, *, reason: t.Optional[str] = None) -> None:          """Permanently ban a user for the given reason without notifying the user."""          await self.apply_ban(ctx, user, reason, hidden=True) @@ -137,7 +145,13 @@ class Infractions(InfractionScheduler, commands.Cog):      # region: Temporary shadow infractions      @command(hidden=True, aliases=["shadowtempmute, stempmute", "shadowmute", "smute"]) -    async def shadow_tempmute(self, ctx: Context, user: Member, duration: Expiry, *, reason: str = None) -> None: +    async def shadow_tempmute( +        self, ctx: Context, +        user: Member, +        duration: Expiry, +        *, +        reason: t.Optional[str] = None +    ) -> None:          """          Temporarily mute a user for the given reason and duration without notifying the user. @@ -162,7 +176,7 @@ class Infractions(InfractionScheduler, commands.Cog):          user: FetchedMember,          duration: Expiry,          *, -        reason: str = None +        reason: t.Optional[str] = None      ) -> None:          """          Temporarily ban a user for the given reason and duration without notifying the user. @@ -197,7 +211,7 @@ class Infractions(InfractionScheduler, commands.Cog):      # endregion      # region: Base apply functions -    async def apply_mute(self, ctx: Context, user: Member, reason: str, **kwargs) -> None: +    async def apply_mute(self, ctx: Context, user: Member, reason: t.Optional[str], **kwargs) -> None:          """Apply a mute infraction with kwargs passed to `post_infraction`."""          if await utils.get_active_infraction(ctx, user, "mute"):              return @@ -217,7 +231,7 @@ class Infractions(InfractionScheduler, commands.Cog):          await self.apply_infraction(ctx, infraction, user, action())      @respect_role_hierarchy() -    async def apply_kick(self, ctx: Context, user: Member, reason: str, **kwargs) -> None: +    async def apply_kick(self, ctx: Context, user: Member, reason: t.Optional[str], **kwargs) -> None:          """Apply a kick infraction with kwargs passed to `post_infraction`."""          infraction = await utils.post_infraction(ctx, user, "kick", reason, active=False, **kwargs)          if infraction is None: @@ -225,11 +239,14 @@ class Infractions(InfractionScheduler, commands.Cog):          self.mod_log.ignore(Event.member_remove, user.id) +        if reason: +            reason = textwrap.shorten(reason, width=512, placeholder="...") +          action = user.kick(reason=reason)          await self.apply_infraction(ctx, infraction, user, action)      @respect_role_hierarchy() -    async def apply_ban(self, ctx: Context, user: UserSnowflake, reason: str, **kwargs) -> None: +    async def apply_ban(self, ctx: Context, user: UserSnowflake, reason: t.Optional[str], **kwargs) -> None:          """          Apply a ban infraction with kwargs passed to `post_infraction`. @@ -258,6 +275,9 @@ class Infractions(InfractionScheduler, commands.Cog):          self.mod_log.ignore(Event.member_remove, user.id) +        if reason: +            reason = textwrap.shorten(reason, width=512, placeholder="...") +          action = ctx.guild.ban(user, reason=reason, delete_message_days=0)          await self.apply_infraction(ctx, infraction, user, action) @@ -278,7 +298,7 @@ class Infractions(InfractionScheduler, commands.Cog):      # endregion      # region: Base pardon functions -    async def pardon_mute(self, user_id: int, guild: discord.Guild, reason: str) -> t.Dict[str, str]: +    async def pardon_mute(self, user_id: int, guild: discord.Guild, reason: t.Optional[str]) -> t.Dict[str, str]:          """Remove a user's muted role, DM them a notification, and return a log dict."""          user = guild.get_member(user_id)          log_text = {} @@ -304,7 +324,7 @@ class Infractions(InfractionScheduler, commands.Cog):          return log_text -    async def pardon_ban(self, user_id: int, guild: discord.Guild, reason: str) -> t.Dict[str, str]: +    async def pardon_ban(self, user_id: int, guild: discord.Guild, reason: t.Optional[str]) -> t.Dict[str, str]:          """Remove a user's ban on the Discord guild and return a log dict."""          user = discord.Object(user_id)          log_text = {} diff --git a/bot/cogs/moderation/management.py b/bot/cogs/moderation/management.py index edfdfd9e2..c39c7f3bc 100644 --- a/bot/cogs/moderation/management.py +++ b/bot/cogs/moderation/management.py @@ -12,7 +12,7 @@ from bot.bot import Bot  from bot.converters import Expiry, InfractionSearchQuery, allowed_strings, proxy_user  from bot.pagination import LinePaginator  from bot.utils import time -from bot.utils.checks import in_channel_check, with_role_check +from bot.utils.checks import in_whitelist_check, with_role_check  from . import utils  from .infractions import Infractions  from .modlog import ModLog @@ -49,8 +49,8 @@ class ModManagement(commands.Cog):      async def infraction_edit(          self,          ctx: Context, -        infraction_id: t.Union[int, allowed_strings("l", "last", "recent")], -        duration: t.Union[Expiry, allowed_strings("p", "permanent"), None], +        infraction_id: t.Union[int, allowed_strings("l", "last", "recent")],  # noqa: F821 +        duration: t.Union[Expiry, allowed_strings("p", "permanent"), None],   # noqa: F821          *,          reason: str = None      ) -> None: @@ -83,14 +83,14 @@ class ModManagement(commands.Cog):                  "actor__id": ctx.author.id,                  "ordering": "-inserted_at"              } -            infractions = await self.bot.api_client.get(f"bot/infractions", params=params) +            infractions = await self.bot.api_client.get("bot/infractions", params=params)              if infractions:                  old_infraction = infractions[0]                  infraction_id = old_infraction["id"]              else:                  await ctx.send( -                    f":x: Couldn't find most recent infraction; you have never given an infraction." +                    ":x: Couldn't find most recent infraction; you have never given an infraction."                  )                  return          else: @@ -224,7 +224,7 @@ class ModManagement(commands.Cog):      ) -> None:          """Send a paginated embed of infractions for the specified user."""          if not infractions: -            await ctx.send(f":warning: No infractions could be found for that query.") +            await ctx.send(":warning: No infractions could be found for that query.")              return          lines = tuple( @@ -283,10 +283,16 @@ class ModManagement(commands.Cog):      # This cannot be static (must have a __func__ attribute).      def cog_check(self, ctx: Context) -> bool: -        """Only allow moderators from moderator channels to invoke the commands in this cog.""" +        """Only allow moderators inside moderator channels to invoke the commands in this cog."""          checks = [              with_role_check(ctx, *constants.MODERATION_ROLES), -            in_channel_check(ctx, *constants.MODERATION_CHANNELS) +            in_whitelist_check( +                ctx, +                channels=constants.MODERATION_CHANNELS, +                categories=[constants.Categories.modmail], +                redirect=None, +                fail_silently=True, +            )          ]          return all(checks) diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index beef7a8ef..41472c64c 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -98,7 +98,10 @@ class ModLog(Cog, name="ModLog"):          footer: t.Optional[str] = None,      ) -> Context:          """Generate log embed and send to logging channel.""" -        embed = discord.Embed(description=text) +        # Truncate string directly here to avoid removing newlines +        embed = discord.Embed( +            description=text[:2045] + "..." if len(text) > 2048 else text +        )          if title and icon_url:              embed.set_author(name=title, icon_url=icon_url) @@ -552,6 +555,10 @@ class ModLog(Cog, name="ModLog"):          channel = message.channel          author = message.author +        # Ignore DMs. +        if not message.guild: +            return +          if message.guild.id != GuildConstant.id or channel.id in GuildConstant.modlog_blacklist:              return diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py index dc42bee2e..d75a72ddb 100644 --- a/bot/cogs/moderation/scheduler.py +++ b/bot/cogs/moderation/scheduler.py @@ -91,7 +91,7 @@ class InfractionScheduler(Scheduler):          log.trace(f"Applying {infr_type} infraction #{id_} to {user}.")          # Default values for the confirmation message and mod log. -        confirm_msg = f":ok_hand: applied" +        confirm_msg = ":ok_hand: applied"          # Specifying an expiry for a note or warning makes no sense.          if infr_type in ("note", "warning"): @@ -101,11 +101,16 @@ class InfractionScheduler(Scheduler):          dm_result = ""          dm_log_text = "" -        expiry_log_text = f"Expires: {expiry}" if expiry else "" +        expiry_log_text = f"\nExpires: {expiry}" if expiry else ""          log_title = "applied"          log_content = None +        failed = False          # DM the user about the infraction if it's not a shadow/hidden infraction. +        # This needs to happen before we apply the infraction, as the bot cannot +        # send DMs to user that it doesn't share a guild with. If we were to +        # apply kick/ban infractions first, this would mean that we'd make it +        # impossible for us to deliver a DM. See python-discord/bot#982.          if not infraction["hidden"]:              dm_result = f"{constants.Emojis.failmail} "              dm_log_text = "\nDM: **Failed**" @@ -122,18 +127,17 @@ class InfractionScheduler(Scheduler):                      dm_result = ":incoming_envelope: "                      dm_log_text = "\nDM: Sent" +        end_msg = ""          if infraction["actor"] == self.bot.user.id:              log.trace(                  f"Infraction #{id_} actor is bot; including the reason in the confirmation message."              ) - -            end_msg = f" (reason: {infraction['reason']})" +            if reason: +                end_msg = f" (reason: {textwrap.shorten(reason, width=1500, placeholder='...')})"          elif ctx.channel.id not in STAFF_CHANNELS:              log.trace(                  f"Infraction #{id_} context is not in a staff channel; omitting infraction count."              ) - -            end_msg = ""          else:              log.trace(f"Fetching total infraction count for {user}.") @@ -154,7 +158,7 @@ class InfractionScheduler(Scheduler):                      self.schedule_task(infraction["id"], infraction)              except discord.HTTPException as e:                  # Accordingly display that applying the infraction failed. -                confirm_msg = f":x: failed to apply" +                confirm_msg = ":x: failed to apply"                  expiry_msg = ""                  log_content = ctx.author.mention                  log_title = "failed to apply" @@ -164,12 +168,23 @@ class InfractionScheduler(Scheduler):                      log.warning(f"{log_msg}: bot lacks permissions.")                  else:                      log.exception(log_msg) +                failed = True + +        if failed: +            log.trace(f"Deleted infraction {infraction['id']} from database because applying infraction failed.") +            try: +                await self.bot.api_client.delete(f"bot/infractions/{id_}") +            except ResponseCodeError as e: +                confirm_msg += " and failed to delete" +                log_title += " and failed to delete" +                log.error(f"Deletion of {infr_type} infraction #{id_} failed with error code {e.status}.") +            infr_message = "" +        else: +            infr_message = f" **{infr_type}** to {user.mention}{expiry_msg}{end_msg}"          # Send a confirmation message to the invoking context.          log.trace(f"Sending infraction #{id_} confirmation message.") -        await ctx.send( -            f"{dm_result}{confirm_msg} **{infr_type}** to {user.mention}{expiry_msg}{end_msg}." -        ) +        await ctx.send(f"{dm_result}{confirm_msg}{infr_message}.")          # Send a log message to the mod log.          log.trace(f"Sending apply mod log for infraction #{id_}.") @@ -180,9 +195,8 @@ class InfractionScheduler(Scheduler):              thumbnail=user.avatar_url_as(static_format="png"),              text=textwrap.dedent(f"""                  Member: {user.mention} (`{user.id}`) -                Actor: {ctx.message.author}{dm_log_text} +                Actor: {ctx.message.author}{dm_log_text}{expiry_log_text}                  Reason: {reason} -                {expiry_log_text}              """),              content=log_content,              footer=f"ID {infraction['id']}" @@ -281,7 +295,7 @@ class InfractionScheduler(Scheduler):              log.warning(f"Failed to pardon {infr_type} infraction #{id_} for {user}.")          else: -            confirm_msg = f":ok_hand: pardoned" +            confirm_msg = ":ok_hand: pardoned"              log_title = "pardoned"              log.info(f"Pardoned {infr_type} infraction #{id_} for {user}.") @@ -294,6 +308,9 @@ class InfractionScheduler(Scheduler):                  f"{log_text.get('Failure', '')}"              ) +        # Move reason to end of entry to avoid cutting out some keys +        log_text["Reason"] = log_text.pop("Reason") +          # Send a log message to the mod log.          await self.mod_log.send_log_message(              icon_url=utils.INFRACTION_ICONS[infr_type][1], @@ -353,7 +370,7 @@ class InfractionScheduler(Scheduler):                  )          except discord.Forbidden:              log.warning(f"Failed to deactivate infraction #{id_} ({type_}): bot lacks permissions.") -            log_text["Failure"] = f"The bot lacks permissions to do this (role hierarchy?)" +            log_text["Failure"] = "The bot lacks permissions to do this (role hierarchy?)"              log_content = mod_role.mention          except discord.HTTPException as e:              log.exception(f"Failed to deactivate infraction #{id_} ({type_})") @@ -402,11 +419,14 @@ class InfractionScheduler(Scheduler):          # Send a log message to the mod log.          if send_log: -            log_title = f"expiration failed" if "Failure" in log_text else "expired" +            log_title = "expiration failed" if "Failure" in log_text else "expired"              user = self.bot.get_user(user_id)              avatar = user.avatar_url_as(static_format="png") if user else None +            # Move reason to end so when reason is too long, this is not gonna cut out required items. +            log_text["Reason"] = log_text.pop("Reason") +              log.trace(f"Sending deactivation mod log for infraction #{id_}.")              await self.mod_log.send_log_message(                  icon_url=utils.INFRACTION_ICONS[type_][1], @@ -416,7 +436,6 @@ class InfractionScheduler(Scheduler):                  text="\n".join(f"{k}: {v}" for k, v in log_text.items()),                  footer=f"ID: {id_}",                  content=log_content, -              )          return log_text diff --git a/bot/cogs/moderation/silence.py b/bot/cogs/moderation/silence.py index 1ef3967a9..c8ab6443b 100644 --- a/bot/cogs/moderation/silence.py +++ b/bot/cogs/moderation/silence.py @@ -1,7 +1,7 @@  import asyncio  import logging  from contextlib import suppress -from typing import Optional +from typing import NamedTuple, Optional  from discord import TextChannel  from discord.ext import commands, tasks @@ -11,10 +11,18 @@ from bot.bot import Bot  from bot.constants import Channels, Emojis, Guild, MODERATION_ROLES, Roles  from bot.converters import HushDurationConverter  from bot.utils.checks import with_role_check +from bot.utils.scheduling import Scheduler  log = logging.getLogger(__name__) +class TaskData(NamedTuple): +    """Data for a scheduled task.""" + +    delay: int +    ctx: Context + +  class SilenceNotifier(tasks.Loop):      """Loop notifier for posting notices to `alert_channel` containing added channels.""" @@ -53,15 +61,25 @@ class SilenceNotifier(tasks.Loop):              await self._alert_channel.send(f"<@&{Roles.moderators}> currently silenced channels: {channels_text}") -class Silence(commands.Cog): +class Silence(Scheduler, commands.Cog):      """Commands for stopping channel messages for `verified` role in a channel."""      def __init__(self, bot: Bot): +        super().__init__()          self.bot = bot          self.muted_channels = set()          self._get_instance_vars_task = self.bot.loop.create_task(self._get_instance_vars())          self._get_instance_vars_event = asyncio.Event() +    async def _scheduled_task(self, task: TaskData) -> None: +        """Calls `self.unsilence` on expired silenced channel to unsilence it.""" +        await asyncio.sleep(task.delay) +        log.info("Unsilencing channel after set delay.") + +        # Because `self.unsilence` explicitly cancels this scheduled task, it is shielded +        # to avoid prematurely cancelling itself +        await asyncio.shield(task.ctx.invoke(self.unsilence)) +      async def _get_instance_vars(self) -> None:          """Get instance variables after they're available to get from the guild."""          await self.bot.wait_until_guild_available() @@ -90,9 +108,13 @@ class Silence(commands.Cog):              return          await ctx.send(f"{Emojis.check_mark} silenced current channel for {duration} minute(s).") -        await asyncio.sleep(duration*60) -        log.info(f"Unsilencing channel after set delay.") -        await ctx.invoke(self.unsilence) + +        task_data = TaskData( +            delay=duration*60, +            ctx=ctx +        ) + +        self.schedule_task(ctx.channel.id, task_data)      @commands.command(aliases=("unhush",))      async def unsilence(self, ctx: Context) -> None: @@ -103,7 +125,9 @@ class Silence(commands.Cog):          """          await self._get_instance_vars_event.wait()          log.debug(f"Unsilencing channel #{ctx.channel} from {ctx.author}'s command.") -        if await self._unsilence(ctx.channel): +        if not await self._unsilence(ctx.channel): +            await ctx.send(f"{Emojis.cross_mark} current channel was not silenced.") +        else:              await ctx.send(f"{Emojis.check_mark} unsilenced current channel.")      async def _silence(self, channel: TextChannel, persistent: bool, duration: Optional[int]) -> bool: @@ -140,6 +164,7 @@ class Silence(commands.Cog):          if current_overwrite.send_messages is False:              await channel.set_permissions(self._verified_role, **dict(current_overwrite, send_messages=None))              log.info(f"Unsilenced channel #{channel} ({channel.id}).") +            self.cancel_task(channel.id)              self.notifier.remove_channel(channel)              self.muted_channels.discard(channel)              return True diff --git a/bot/cogs/moderation/superstarify.py b/bot/cogs/moderation/superstarify.py index 29855c325..45a010f00 100644 --- a/bot/cogs/moderation/superstarify.py +++ b/bot/cogs/moderation/superstarify.py @@ -183,10 +183,10 @@ class Superstarify(InfractionScheduler, Cog):              text=textwrap.dedent(f"""                  Member: {member.mention} (`{member.id}`)                  Actor: {ctx.message.author} -                Reason: {reason}                  Expires: {expiry_str}                  Old nickname: `{old_nick}`                  New nickname: `{forced_nick}` +                Reason: {reason}              """),              footer=f"ID {id_}"          ) diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py index e4e0f1ec2..fb55287b6 100644 --- a/bot/cogs/moderation/utils.py +++ b/bot/cogs/moderation/utils.py @@ -41,7 +41,6 @@ async def post_user(ctx: Context, user: UserSnowflake) -> t.Optional[dict]:          log.debug("The user being added to the DB is not a Member or User object.")      payload = { -        'avatar_hash': getattr(user, 'avatar', 0),          'discriminator': int(getattr(user, 'discriminator', 0)),          'id': user.id,          'in_guild': False, @@ -143,12 +142,14 @@ async def notify_infraction(      """DM a user about their new infraction and return True if the DM is successful."""      log.trace(f"Sending {user} a DM about their {infr_type} infraction.") +    text = textwrap.dedent(f""" +        **Type:** {infr_type.capitalize()} +        **Expires:** {expires_at or "N/A"} +        **Reason:** {reason or "No reason provided."} +    """) +      embed = discord.Embed( -        description=textwrap.dedent(f""" -            **Type:** {infr_type.capitalize()} -            **Expires:** {expires_at or "N/A"} -            **Reason:** {reason or "No reason provided."} -            """), +        description=textwrap.shorten(text, width=2048, placeholder="..."),          colour=Colours.soft_red      ) diff --git a/bot/cogs/python_news.py b/bot/cogs/python_news.py index 57ce61638..adefd5c7c 100644 --- a/bot/cogs/python_news.py +++ b/bot/cogs/python_news.py @@ -10,6 +10,7 @@ from discord.ext.tasks import loop  from bot import constants  from bot.bot import Bot +from bot.utils.messages import sub_clyde  PEPS_RSS_URL = "https://www.python.org/dev/peps/peps.rss/" @@ -109,6 +110,9 @@ class PythonNews(Cog):              )              payload["data"]["pep"].append(pep_nr) +            # Increase overall PEP new stat +            self.bot.stats.incr("python_news.posted.pep") +              if msg.channel.is_news():                  log.trace("Publishing PEP annnouncement because it was in a news channel")                  await msg.publish() @@ -150,6 +154,7 @@ class PythonNews(Cog):                  if (                          thread_information["thread_id"] in existing_news["data"][maillist] +                        or 'Re: ' in thread_information["subject"]                          or new_date.date() < date.today()                  ):                      continue @@ -168,6 +173,9 @@ class PythonNews(Cog):                  )                  payload["data"][maillist].append(thread_information["thread_id"]) +                # Increase this specific maillist counter in stats +                self.bot.stats.incr(f"python_news.posted.{maillist.replace('-', '_')}") +                  if msg.channel.is_news():                      log.trace("Publishing mailing list message because it was in a news channel")                      await msg.publish() @@ -201,7 +209,7 @@ class PythonNews(Cog):          return await self.webhook.send(              embed=embed, -            username=webhook_profile_name, +            username=sub_clyde(webhook_profile_name),              avatar_url=AVATAR_URL,              wait=True          ) diff --git a/bot/cogs/reddit.py b/bot/cogs/reddit.py index 5f2aec7a5..d853ab2ea 100644 --- a/bot/cogs/reddit.py +++ b/bot/cogs/reddit.py @@ -16,6 +16,7 @@ from bot.constants import Channels, ERROR_REPLIES, Emojis, Reddit as RedditConfi  from bot.converters import Subreddit  from bot.decorators import with_role  from bot.pagination import LinePaginator +from bot.utils.messages import sub_clyde  log = logging.getLogger(__name__) @@ -218,15 +219,19 @@ class Reddit(Cog):          for subreddit in RedditConfig.subreddits:              top_posts = await self.get_top_posts(subreddit=subreddit, time="day") -            await self.webhook.send(username=f"{subreddit} Top Daily Posts", embed=top_posts) +            username = sub_clyde(f"{subreddit} Top Daily Posts") +            message = await self.webhook.send(username=username, embed=top_posts, wait=True) + +            if message.channel.is_news(): +                await message.publish()      async def top_weekly_posts(self) -> None:          """Post a summary of the top posts."""          for subreddit in RedditConfig.subreddits:              # Send and pin the new weekly posts.              top_posts = await self.get_top_posts(subreddit=subreddit, time="week") - -            message = await self.webhook.send(wait=True, username=f"{subreddit} Top Weekly Posts", embed=top_posts) +            username = sub_clyde(f"{subreddit} Top Weekly Posts") +            message = await self.webhook.send(wait=True, username=username, embed=top_posts)              if subreddit.lower() == "r/python":                  if not self.channel: @@ -242,6 +247,9 @@ class Reddit(Cog):                  await message.pin() +                if message.channel.is_news(): +                    await message.publish() +      @group(name="reddit", invoke_without_command=True)      async def reddit_group(self, ctx: Context) -> None:          """View the top posts from various subreddits.""" diff --git a/bot/cogs/site.py b/bot/cogs/site.py index 7fc2a9c34..ac29daa1d 100644 --- a/bot/cogs/site.py +++ b/bot/cogs/site.py @@ -33,7 +33,7 @@ class Site(Cog):          embed.colour = Colour.blurple()          embed.description = (              f"[Our official website]({url}) is an open-source community project " -            "created with Python and Flask. It contains information about the server " +            "created with Python and Django. It contains information about the server "              "itself, lets you sign up for upcoming events, has its own wiki, contains "              "a list of valuable learning resources, and much more."          ) @@ -133,6 +133,9 @@ class Site(Cog):              await ctx.send(f":x: Invalid rule indices: {indices}")              return +        for rule in rules: +            self.bot.stats.incr(f"rule_uses.{rule}") +          final_rules = tuple(f"**{pick}.** {full_rules[pick - 1]}" for pick in rules)          await LinePaginator.paginate(final_rules, ctx, rules_embed, max_lines=3) diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index c2782b9c8..a2a7574d4 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -47,6 +47,7 @@ EVAL_ROLES = (Roles.helpers, Roles.moderators, Roles.admins, Roles.owners, Roles  SIGKILL = 9  REEVAL_EMOJI = '\U0001f501'  # :repeat: +REEVAL_TIMEOUT = 30  class Snekbox(Cog): @@ -205,6 +206,12 @@ class Snekbox(Cog):              if paste_link:                  msg = f"{msg}\nFull output: {paste_link}" +            # Collect stats of eval fails + successes +            if icon == ":x:": +                self.bot.stats.incr("snekbox.python.fail") +            else: +                self.bot.stats.incr("snekbox.python.success") +              response = await ctx.send(msg)              self.bot.loop.create_task(                  wait_for_deletion(response, user_ids=(ctx.author.id,), client=ctx.bot) @@ -227,7 +234,7 @@ class Snekbox(Cog):                  _, new_message = await self.bot.wait_for(                      'message_edit',                      check=_predicate_eval_message_edit, -                    timeout=10 +                    timeout=REEVAL_TIMEOUT                  )                  await ctx.message.add_reaction(REEVAL_EMOJI)                  await self.bot.wait_for( @@ -292,6 +299,18 @@ class Snekbox(Cog):              await ctx.send_help(ctx.command)              return +        if Roles.helpers in (role.id for role in ctx.author.roles): +            self.bot.stats.incr("snekbox_usages.roles.helpers") +        else: +            self.bot.stats.incr("snekbox_usages.roles.developers") + +        if ctx.channel.category_id == Categories.help_in_use: +            self.bot.stats.incr("snekbox_usages.channels.help") +        elif ctx.channel.id == Channels.bot_commands: +            self.bot.stats.incr("snekbox_usages.channels.bot_commands") +        else: +            self.bot.stats.incr("snekbox_usages.channels.topical") +          log.info(f"Received code from {ctx.author} for evaluation:\n{code}")          while True: diff --git a/bot/cogs/stats.py b/bot/cogs/stats.py index d253db913..d42f55466 100644 --- a/bot/cogs/stats.py +++ b/bot/cogs/stats.py @@ -2,9 +2,11 @@ import string  from datetime import datetime  from discord import Member, Message, Status -from discord.ext.commands import Bot, Cog, Context +from discord.ext.commands import Cog, Context +from discord.ext.tasks import loop -from bot.constants import Channels, Guild, Stats as StatConf +from bot.bot import Bot +from bot.constants import Categories, Channels, Guild, Stats as StatConf  CHANNEL_NAME_OVERRIDES = { @@ -23,6 +25,7 @@ class Stats(Cog):      def __init__(self, bot: Bot):          self.bot = bot          self.last_presence_update = None +        self.update_guild_boost.start()      @Cog.listener()      async def on_message(self, message: Message) -> None: @@ -33,6 +36,13 @@ class Stats(Cog):          if message.guild.id != Guild.id:              return +        cat = getattr(message.channel, "category", None) +        if cat is not None and cat.id == Categories.modmail: +            if message.channel.id != Channels.incidents: +                # Do not report modmail channels to stats, there are too many +                # of them for interesting statistics to be drawn out of this. +                return +          reformatted_name = message.channel.name.replace('-', '_')          if CHANNEL_NAME_OVERRIDES.get(message.channel.id): @@ -59,7 +69,7 @@ class Stats(Cog):          if member.guild.id != Guild.id:              return -        self.bot.stats.gauge(f"guild.total_members", len(member.guild.members)) +        self.bot.stats.gauge("guild.total_members", len(member.guild.members))      @Cog.listener()      async def on_member_leave(self, member: Member) -> None: @@ -67,7 +77,7 @@ class Stats(Cog):          if member.guild.id != Guild.id:              return -        self.bot.stats.gauge(f"guild.total_members", len(member.guild.members)) +        self.bot.stats.gauge("guild.total_members", len(member.guild.members))      @Cog.listener()      async def on_member_update(self, _before: Member, after: Member) -> None: @@ -101,6 +111,18 @@ class Stats(Cog):          self.bot.stats.gauge("guild.status.do_not_disturb", dnd)          self.bot.stats.gauge("guild.status.offline", offline) +    @loop(hours=1) +    async def update_guild_boost(self) -> None: +        """Post the server boost level and tier every hour.""" +        await self.bot.wait_until_guild_available() +        g = self.bot.get_guild(Guild.id) +        self.bot.stats.gauge("boost.amount", g.premium_subscription_count) +        self.bot.stats.gauge("boost.tier", g.premium_tier) + +    def cog_unload(self) -> None: +        """Stop the boost statistic task on unload of the Cog.""" +        self.update_guild_boost.stop() +  def setup(bot: Bot) -> None:      """Load the stats cog.""" diff --git a/bot/cogs/sync/cog.py b/bot/cogs/sync/cog.py index 5708be3f4..7cc3726b2 100644 --- a/bot/cogs/sync/cog.py +++ b/bot/cogs/sync/cog.py @@ -94,7 +94,6 @@ class Sync(Cog):          the database, the user is added.          """          packed = { -            'avatar_hash': member.avatar,              'discriminator': int(member.discriminator),              'id': member.id,              'in_guild': True, @@ -135,12 +134,11 @@ class Sync(Cog):      @Cog.listener()      async def on_user_update(self, before: User, after: User) -> None:          """Update the user information in the database if a relevant change is detected.""" -        attrs = ("name", "discriminator", "avatar") +        attrs = ("name", "discriminator")          if any(getattr(before, attr) != getattr(after, attr) for attr in attrs):              updated_information = {                  "name": after.name,                  "discriminator": int(after.discriminator), -                "avatar_hash": after.avatar,              }              await self.patch_user(after.id, updated_information=updated_information) diff --git a/bot/cogs/sync/syncers.py b/bot/cogs/sync/syncers.py index e55bf27fd..536455668 100644 --- a/bot/cogs/sync/syncers.py +++ b/bot/cogs/sync/syncers.py @@ -17,7 +17,7 @@ log = logging.getLogger(__name__)  # These objects are declared as namedtuples because tuples are hashable,  # something that we make use of when diffing site roles against guild roles.  _Role = namedtuple('Role', ('id', 'name', 'colour', 'permissions', 'position')) -_User = namedtuple('User', ('id', 'name', 'discriminator', 'avatar_hash', 'roles', 'in_guild')) +_User = namedtuple('User', ('id', 'name', 'discriminator', 'roles', 'in_guild'))  _Diff = namedtuple('Diff', ('created', 'updated', 'deleted')) @@ -298,7 +298,6 @@ class UserSyncer(Syncer):                  id=member.id,                  name=member.name,                  discriminator=int(member.discriminator), -                avatar_hash=member.avatar,                  roles=tuple(sorted(role.id for role in member.roles)),                  in_guild=True              ) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index a813ffff5..6f03a3475 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -4,7 +4,7 @@ import time  from pathlib import Path  from typing import Callable, Dict, Iterable, List, Optional -from discord import Colour, Embed +from discord import Colour, Embed, Member  from discord.ext.commands import Cog, Context, group  from bot import constants @@ -35,21 +35,36 @@ class Tags(Cog):      @staticmethod      def get_tags() -> dict:          """Get all tags.""" -        # Save all tags in memory.          cache = {} -        tag_files = Path("bot", "resources", "tags").iterdir() -        for file in tag_files: -            tag_title = file.stem -            tag = { -                "title": tag_title, -                "embed": { -                    "description": file.read_text(encoding="utf-8") + +        base_path = Path("bot", "resources", "tags") +        for file in base_path.glob("**/*"): +            if file.is_file(): +                tag_title = file.stem +                tag = { +                    "title": tag_title, +                    "embed": { +                        "description": file.read_text(encoding="utf8"), +                    }, +                    "restricted_to": "developers",                  } -            } -            cache[tag_title] = tag + +                # Convert to a list to allow negative indexing. +                parents = list(file.relative_to(base_path).parents) +                if len(parents) > 1: +                    # -1 would be '.' hence -2 is used as the index. +                    tag["restricted_to"] = parents[-2].name + +                cache[tag_title] = tag +          return cache      @staticmethod +    def check_accessibility(user: Member, tag: dict) -> bool: +        """Check if user can access a tag.""" +        return tag["restricted_to"].lower() in [role.name.lower() for role in user.roles] + +    @staticmethod      def _fuzzy_search(search: str, target: str) -> float:          """A simple scoring algorithm based on how many letters are found / total, with order in mind."""          current, index = 0, 0 @@ -93,7 +108,7 @@ class Tags(Cog):              return self._get_suggestions(tag_name)          return found -    def _get_tags_via_content(self, check: Callable[[Iterable], bool], keywords: str) -> list: +    def _get_tags_via_content(self, check: Callable[[Iterable], bool], keywords: str, user: Member) -> list:          """          Search for tags via contents. @@ -114,7 +129,8 @@ class Tags(Cog):          matching_tags = []          for tag in self._cache.values(): -            if check(query in tag['embed']['description'].casefold() for query in keywords_processed): +            matches = (query in tag['embed']['description'].casefold() for query in keywords_processed) +            if self.check_accessibility(user, tag) and check(matches):                  matching_tags.append(tag)          return matching_tags @@ -152,7 +168,7 @@ class Tags(Cog):          Only search for tags that has ALL the keywords.          """ -        matching_tags = self._get_tags_via_content(all, keywords) +        matching_tags = self._get_tags_via_content(all, keywords, ctx.author)          await self._send_matching_tags(ctx, keywords, matching_tags)      @search_tag_content.command(name='any') @@ -162,7 +178,7 @@ class Tags(Cog):          Search for tags that has ANY of the keywords.          """ -        matching_tags = self._get_tags_via_content(any, keywords or 'any') +        matching_tags = self._get_tags_via_content(any, keywords or 'any', ctx.author)          await self._send_matching_tags(ctx, keywords, matching_tags)      @tags_group.command(name='get', aliases=('show', 'g')) @@ -198,7 +214,13 @@ class Tags(Cog):              return          if tag_name is not None: -            founds = self._get_tag(tag_name) +            temp_founds = self._get_tag(tag_name) + +            founds = [] + +            for found_tag in temp_founds: +                if self.check_accessibility(ctx.author, found_tag): +                    founds.append(found_tag)              if len(founds) == 1:                  tag = founds[0] @@ -237,7 +259,10 @@ class Tags(Cog):              else:                  embed: Embed = Embed(title="**Current tags**")                  await LinePaginator.paginate( -                    sorted(f"**»**   {tag['title']}" for tag in tags), +                    sorted( +                        f"**»**   {tag['title']}" for tag in tags +                        if self.check_accessibility(ctx.author, tag) +                    ),                      ctx,                      embed,                      footer_text=FOOTER_TEXT, diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index 6721f0e02..ef979f222 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -2,20 +2,22 @@ import base64  import binascii  import logging  import re -import struct  import typing as t -from datetime import datetime -from discord import Colour, Message +from discord import Colour, Message, NotFound  from discord.ext.commands import Cog -from discord.utils import snowflake_time +from bot import utils  from bot.bot import Bot  from bot.cogs.moderation import ModLog  from bot.constants import Channels, Colours, Event, Icons  log = logging.getLogger(__name__) +LOG_MESSAGE = ( +    "Censored a seemingly valid token sent by {author} (`{author_id}`) in {channel}, " +    "token was `{user_id}.{timestamp}.{hmac}`" +)  DELETION_MESSAGE_TEMPLATE = (      "Hey {mention}! I noticed you posted a seemingly valid Discord API "      "token in your message and have removed your message. " @@ -25,15 +27,22 @@ DELETION_MESSAGE_TEMPLATE = (      "Feel free to re-post it with the token removed. "      "If you believe this was a mistake, please let us know!"  ) -DISCORD_EPOCH_TIMESTAMP = datetime(2017, 1, 1) +DISCORD_EPOCH = 1_420_070_400  TOKEN_EPOCH = 1_293_840_000 -TOKEN_RE = re.compile( -    r"[^\s\.()\"']+"  # Matches token part 1: The user ID string, encoded as base64 -    r"\."             # Matches a literal dot between the token parts -    r"[^\s\.()\"']+"  # Matches token part 2: The creation timestamp, as an integer -    r"\."             # Matches a literal dot between the token parts -    r"[^\s\.()\"']+"  # Matches token part 3: The HMAC, unused by us, but check that it isn't empty -) + +# Three parts delimited by dots: user ID, creation timestamp, HMAC. +# The HMAC isn't parsed further, but it's in the regex to ensure it at least exists in the string. +# Each part only matches base64 URL-safe characters. +# Padding has never been observed, but the padding character '=' is matched just in case. +TOKEN_RE = re.compile(r"([\w\-=]+)\.([\w\-=]+)\.([\w\-=]+)", re.ASCII) + + +class Token(t.NamedTuple): +    """A Discord Bot token.""" + +    user_id: str +    timestamp: str +    hmac: str  class TokenRemover(Cog): @@ -54,6 +63,10 @@ class TokenRemover(Cog):          See: https://discordapp.com/developers/docs/reference#snowflakes          """ +        # Ignore DMs; can't delete messages in there anyway. +        if not msg.guild or msg.author.bot: +            return +          found_token = self.find_token_in_message(msg)          if found_token:              await self.take_action(msg, found_token) @@ -65,64 +78,61 @@ class TokenRemover(Cog):          See: https://discordapp.com/developers/docs/reference#snowflakes          """ -        found_token = self.find_token_in_message(after) -        if found_token: -            await self.take_action(after, found_token) +        await self.on_message(after) -    async def take_action(self, msg: Message, found_token: str) -> None: -        """Remove the `msg` containing a token an send a mod_log message.""" -        user_id, creation_timestamp, hmac = found_token.split('.') +    async def take_action(self, msg: Message, found_token: Token) -> None: +        """Remove the `msg` containing the `found_token` and send a mod log message."""          self.mod_log.ignore(Event.message_delete, msg.id) -        await msg.delete() + +        try: +            await msg.delete() +        except NotFound: +            log.debug(f"Failed to remove token in message {msg.id}: message already deleted.") +            return +          await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention)) -        message = ( -            "Censored a seemingly valid token sent by " -            f"{msg.author} (`{msg.author.id}`) in {msg.channel.mention}, token was " -            f"`{user_id}.{creation_timestamp}.{'x' * len(hmac)}`" -        ) -        log.debug(message) +        log_message = self.format_log_message(msg, found_token) +        log.debug(log_message)          # Send pretty mod log embed to mod-alerts          await self.mod_log.send_log_message(              icon_url=Icons.token_removed,              colour=Colour(Colours.soft_red),              title="Token removed!", -            text=message, +            text=log_message,              thumbnail=msg.author.avatar_url_as(static_format="png"),              channel_id=Channels.mod_alerts,          )          self.bot.stats.incr("tokens.removed_tokens") +    @staticmethod +    def format_log_message(msg: Message, token: Token) -> str: +        """Return the log message to send for `token` being censored in `msg`.""" +        return LOG_MESSAGE.format( +            author=msg.author, +            author_id=msg.author.id, +            channel=msg.channel.mention, +            user_id=token.user_id, +            timestamp=token.timestamp, +            hmac='x' * len(token.hmac), +        ) +      @classmethod -    def find_token_in_message(cls, msg: Message) -> t.Optional[str]: +    def find_token_in_message(cls, msg: Message) -> t.Optional[Token]:          """Return a seemingly valid token found in `msg` or `None` if no token is found.""" -        if msg.author.bot: -            return - -        # Use findall rather than search to guard against method calls prematurely returning the +        # Use finditer rather than search to guard against method calls prematurely returning the          # token check (e.g. `message.channel.send` also matches our token pattern) -        maybe_matches = TOKEN_RE.findall(msg.content) -        for substr in maybe_matches: -            if cls.is_maybe_token(substr): +        for match in TOKEN_RE.finditer(msg.content): +            token = Token(*match.groups()) +            if cls.is_valid_user_id(token.user_id) and cls.is_valid_timestamp(token.timestamp):                  # Short-circuit on first match -                return substr +                return token          # No matching substring          return -    @classmethod -    def is_maybe_token(cls, test_str: str) -> bool: -        """Check the provided string to see if it is a seemingly valid token.""" -        try: -            user_id, creation_timestamp, hmac = test_str.split('.') -        except ValueError: -            return False - -        if cls.is_valid_user_id(user_id) and cls.is_valid_timestamp(creation_timestamp): -            return True -      @staticmethod      def is_valid_user_id(b64_content: str) -> bool:          """ @@ -130,29 +140,41 @@ class TokenRemover(Cog):          See: https://discordapp.com/developers/docs/reference#snowflakes          """ -        b64_content += '=' * (-len(b64_content) % 4) +        b64_content = utils.pad_base64(b64_content)          try: -            content: bytes = base64.b64decode(b64_content) -            return content.decode('utf-8').isnumeric() -        except (binascii.Error, UnicodeDecodeError): +            decoded_bytes = base64.urlsafe_b64decode(b64_content) +            string = decoded_bytes.decode('utf-8') + +            # isdigit on its own would match a lot of other Unicode characters, hence the isascii. +            return string.isascii() and string.isdigit() +        except (binascii.Error, ValueError):              return False      @staticmethod      def is_valid_timestamp(b64_content: str) -> bool:          """ -        Check potential token to see if it contains a valid timestamp. +        Return True if `b64_content` decodes to a valid timestamp. -        See: https://discordapp.com/developers/docs/reference#snowflakes +        If the timestamp is greater than the Discord epoch, it's probably valid. +        See: https://i.imgur.com/7WdehGn.png          """ -        b64_content += '=' * (-len(b64_content) % 4) +        b64_content = utils.pad_base64(b64_content)          try: -            content = base64.urlsafe_b64decode(b64_content) -            snowflake = struct.unpack('i', content)[0] -        except (binascii.Error, struct.error): +            decoded_bytes = base64.urlsafe_b64decode(b64_content) +            timestamp = int.from_bytes(decoded_bytes, byteorder="big") +        except (binascii.Error, ValueError) as e: +            log.debug(f"Failed to decode token timestamp '{b64_content}': {e}") +            return False + +        # Seems like newer tokens don't need the epoch added, but add anyway since an upper bound +        # is not checked. +        if timestamp + TOKEN_EPOCH >= DISCORD_EPOCH: +            return True +        else: +            log.debug(f"Invalid token timestamp '{b64_content}': smaller than Discord epoch")              return False -        return snowflake_time(snowflake + TOKEN_EPOCH) < DISCORD_EPOCH_TIMESTAMP  def setup(bot: Bot) -> None: diff --git a/bot/cogs/utils.py b/bot/cogs/utils.py index 6b59d37c8..697bf60ce 100644 --- a/bot/cogs/utils.py +++ b/bot/cogs/utils.py @@ -6,7 +6,7 @@ from email.parser import HeaderParser  from io import StringIO  from typing import Tuple, Union -from discord import Colour, Embed +from discord import Colour, Embed, utils  from discord.ext.commands import BadArgument, Cog, Context, command  from bot.bot import Bot @@ -145,7 +145,7 @@ class Utils(Cog):                  u_code = f"\\U{digit:>08}"              url = f"https://www.compart.com/en/unicode/U+{digit:>04}"              name = f"[{unicodedata.name(char, '')}]({url})" -            info = f"`{u_code.ljust(10)}`: {name} - {char}" +            info = f"`{u_code.ljust(10)}`: {name} - {utils.escape_markdown(char)}"              return info, u_code          charlist, rawlist = zip(*(get_info(c) for c in characters)) @@ -253,8 +253,8 @@ class Utils(Cog):      async def send_pep_zero(self, ctx: Context) -> None:          """Send information about PEP 0."""          pep_embed = Embed( -            title=f"**PEP 0 - Index of Python Enhancement Proposals (PEPs)**", -            description=f"[Link](https://www.python.org/dev/peps/)" +            title="**PEP 0 - Index of Python Enhancement Proposals (PEPs)**", +            description="[Link](https://www.python.org/dev/peps/)"          )          pep_embed.set_thumbnail(url=ICON_URL)          pep_embed.add_field(name="Status", value="Active") diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 77e8b5706..ae156cf70 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -1,16 +1,14 @@  import logging  from contextlib import suppress -from datetime import datetime  from discord import Colour, Forbidden, Message, NotFound, Object -from discord.ext import tasks  from discord.ext.commands import Cog, Context, command  from bot import constants  from bot.bot import Bot  from bot.cogs.moderation import ModLog -from bot.decorators import InWhitelistCheckFailure, in_whitelist, without_role -from bot.utils.checks import without_role_check +from bot.decorators import in_whitelist, without_role +from bot.utils.checks import InWhitelistCheckFailure, without_role_check  log = logging.getLogger(__name__) @@ -34,14 +32,6 @@ If you'd like to unsubscribe from the announcement notifications, simply send `!  <#{constants.Channels.bot_commands}>.  """ -if constants.DEBUG_MODE: -    PERIODIC_PING = "Periodic checkpoint message successfully sent." -else: -    PERIODIC_PING = ( -        f"@everyone To verify that you have read our rules, please type `{constants.Bot.prefix}accept`." -        " If you encounter any problems during the verification process, " -        f"send a direct message to a staff member." -    )  BOT_MESSAGE_DELETE_DELAY = 10 @@ -50,7 +40,6 @@ class Verification(Cog):      def __init__(self, bot: Bot):          self.bot = bot -        self.periodic_ping.start()      @property      def mod_log(self) -> ModLog: @@ -65,9 +54,7 @@ class Verification(Cog):          if message.author.bot:              # They're a bot, delete their message after the delay. -            # But not the periodic ping; we like that one. -            if message.content != PERIODIC_PING: -                await message.delete(delay=BOT_MESSAGE_DELETE_DELAY) +            await message.delete(delay=BOT_MESSAGE_DELETE_DELAY)              return          # if a user mentions a role or guild member @@ -198,34 +185,6 @@ class Verification(Cog):          else:              return True -    @tasks.loop(hours=12) -    async def periodic_ping(self) -> None: -        """Every week, mention @everyone to remind them to verify.""" -        messages = self.bot.get_channel(constants.Channels.verification).history(limit=10) -        need_to_post = True  # True if a new message needs to be sent. - -        async for message in messages: -            if message.author == self.bot.user and message.content == PERIODIC_PING: -                delta = datetime.utcnow() - message.created_at  # Time since last message. -                if delta.days >= 7:  # Message is older than a week. -                    await message.delete() -                else: -                    need_to_post = False - -                break - -        if need_to_post: -            await self.bot.get_channel(constants.Channels.verification).send(PERIODIC_PING) - -    @periodic_ping.before_loop -    async def before_ping(self) -> None: -        """Only start the loop when the bot is ready.""" -        await self.bot.wait_until_guild_available() - -    def cog_unload(self) -> None: -        """Cancel the periodic ping task when the cog is unloaded.""" -        self.periodic_ping.cancel() -  def setup(bot: Bot) -> None:      """Load the Verification cog.""" diff --git a/bot/cogs/watchchannels/bigbrother.py b/bot/cogs/watchchannels/bigbrother.py index e4fb173e0..702d371f4 100644 --- a/bot/cogs/watchchannels/bigbrother.py +++ b/bot/cogs/watchchannels/bigbrother.py @@ -1,4 +1,5 @@  import logging +import textwrap  from collections import ChainMap  from discord.ext.commands import Cog, Context, group @@ -97,8 +98,8 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"):              if len(history) > 1:                  total = f"({len(history) // 2} previous infractions in total)" -                end_reason = history[0]["reason"] -                start_reason = f"Watched: {history[1]['reason']}" +                end_reason = textwrap.shorten(history[0]["reason"], width=500, placeholder="...") +                start_reason = f"Watched: {textwrap.shorten(history[1]['reason'], width=500, placeholder='...')}"                  msg += f"\n\nUser's previous watch reasons {total}:```{start_reason}\n\n{end_reason}```"          else:              msg = ":x: Failed to post the infraction: response was empty." diff --git a/bot/cogs/watchchannels/talentpool.py b/bot/cogs/watchchannels/talentpool.py index 9a85c68c2..14547105f 100644 --- a/bot/cogs/watchchannels/talentpool.py +++ b/bot/cogs/watchchannels/talentpool.py @@ -61,7 +61,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"):              return          if isinstance(user, Member) and any(role.id in STAFF_ROLES for role in user.roles): -            await ctx.send(f":x: Nominating staff members, eh? Here's a cookie :cookie:") +            await ctx.send(":x: Nominating staff members, eh? Here's a cookie :cookie:")              return          if not await self.fetch_user_cache(): @@ -106,8 +106,8 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"):          if history:              total = f"({len(history)} previous nominations in total)" -            start_reason = f"Watched: {history[0]['reason']}" -            end_reason = f"Unwatched: {history[0]['end_reason']}" +            start_reason = f"Watched: {textwrap.shorten(history[0]['reason'], width=500, placeholder='...')}" +            end_reason = f"Unwatched: {textwrap.shorten(history[0]['end_reason'], width=500, placeholder='...')}"              msg += f"\n\nUser's previous watch reasons {total}:```{start_reason}\n\n{end_reason}```"          await ctx.send(msg) @@ -224,7 +224,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"):                  Status: **Active**                  Date: {start_date}                  Actor: {actor.mention if actor else actor_id} -                Reason: {nomination_object["reason"]} +                Reason: {textwrap.shorten(nomination_object["reason"], width=200, placeholder="...")}                  Nomination ID: `{nomination_object["id"]}`                  ===============                  """ @@ -237,10 +237,10 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"):                  Status: Inactive                  Date: {start_date}                  Actor: {actor.mention if actor else actor_id} -                Reason: {nomination_object["reason"]} +                Reason: {textwrap.shorten(nomination_object["reason"], width=200, placeholder="...")}                  End date: {end_date} -                Unwatch reason: {nomination_object["end_reason"]} +                Unwatch reason: {textwrap.shorten(nomination_object["end_reason"], width=200, placeholder="...")}                  Nomination ID: `{nomination_object["id"]}`                  ===============                  """ diff --git a/bot/cogs/watchchannels/watchchannel.py b/bot/cogs/watchchannels/watchchannel.py index 479820444..7c58a0fb5 100644 --- a/bot/cogs/watchchannels/watchchannel.py +++ b/bot/cogs/watchchannels/watchchannel.py @@ -82,7 +82,7 @@ class WatchChannel(metaclass=CogABCMeta):              exc = self._consume_task.exception()              if exc:                  self.log.exception( -                    f"The message queue consume task has failed with:", +                    "The message queue consume task has failed with:",                      exc_info=exc                  )              return False @@ -146,7 +146,7 @@ class WatchChannel(metaclass=CogABCMeta):          try:              data = await self.bot.api_client.get(self.api_endpoint, params=self.api_default_params)          except ResponseCodeError as err: -            self.log.exception(f"Failed to fetch the watched users from the API", exc_info=err) +            self.log.exception("Failed to fetch the watched users from the API", exc_info=err)              return False          self.watched_users = defaultdict(dict) @@ -173,7 +173,7 @@ class WatchChannel(metaclass=CogABCMeta):              self.log.trace(f"Sleeping {BigBrotherConfig.log_delay} seconds before consuming message queue")              await asyncio.sleep(BigBrotherConfig.log_delay) -        self.log.trace(f"Started consuming the message queue") +        self.log.trace("Started consuming the message queue")          # If the previous consumption Task failed, first consume the existing comsumption_queue          if not self.consumption_queue: @@ -204,11 +204,12 @@ class WatchChannel(metaclass=CogABCMeta):          embed: Optional[Embed] = None,      ) -> None:          """Sends a message to the webhook with the specified kwargs.""" +        username = messages.sub_clyde(username)          try:              await self.webhook.send(content=content, username=username, avatar_url=avatar_url, embed=embed)          except discord.HTTPException as exc:              self.log.exception( -                f"Failed to send a message to the webhook", +                "Failed to send a message to the webhook",                  exc_info=exc              ) @@ -254,7 +255,7 @@ class WatchChannel(metaclass=CogABCMeta):                  )              except discord.HTTPException as exc:                  self.log.exception( -                    f"Failed to send an attachment to the webhook", +                    "Failed to send an attachment to the webhook",                      exc_info=exc                  ) @@ -280,8 +281,9 @@ class WatchChannel(metaclass=CogABCMeta):          else:              message_jump = f"in [#{msg.channel.name}]({msg.jump_url})" +        footer = f"Added {time_delta} by {actor} | Reason: {reason}"          embed = Embed(description=f"{msg.author.mention} {message_jump}") -        embed.set_footer(text=f"Added {time_delta} by {actor} | Reason: {reason}") +        embed.set_footer(text=textwrap.shorten(footer, width=128, placeholder="..."))          await self.webhook_send(embed=embed, username=msg.author.display_name, avatar_url=msg.author.avatar_url) @@ -326,13 +328,13 @@ class WatchChannel(metaclass=CogABCMeta):      def cog_unload(self) -> None:          """Takes care of unloading the cog and canceling the consumption task.""" -        self.log.trace(f"Unloading the cog") +        self.log.trace("Unloading the cog")          if self._consume_task and not self._consume_task.done():              self._consume_task.cancel()              try:                  self._consume_task.result()              except asyncio.CancelledError as e:                  self.log.exception( -                    f"The consume task was canceled. Messages may be lost.", +                    "The consume task was canceled. Messages may be lost.",                      exc_info=e                  ) diff --git a/bot/cogs/webhook_remover.py b/bot/cogs/webhook_remover.py index 1b5c3f821..543869215 100644 --- a/bot/cogs/webhook_remover.py +++ b/bot/cogs/webhook_remover.py @@ -1,7 +1,7 @@  import logging  import re -from discord import Colour, Message +from discord import Colour, Message, NotFound  from discord.ext.commands import Cog  from bot.bot import Bot @@ -35,7 +35,13 @@ class WebhookRemover(Cog):          """Delete `msg` and send a warning that it contained the Discord webhook `redacted_url`."""          # Don't log this, due internal delete, not by user. Will make different entry.          self.mod_log.ignore(Event.message_delete, msg.id) -        await msg.delete() + +        try: +            await msg.delete() +        except NotFound: +            log.debug(f"Failed to remove webhook in message {msg.id}: message already deleted.") +            return +          await msg.channel.send(ALERT_MESSAGE_TEMPLATE.format(user=msg.author.mention))          message = ( @@ -59,6 +65,10 @@ class WebhookRemover(Cog):      @Cog.listener()      async def on_message(self, msg: Message) -> None:          """Check if a Discord webhook URL is in `message`.""" +        # Ignore DMs; can't delete messages in there anyway. +        if not msg.guild or msg.author.bot: +            return +          matches = WEBHOOK_URL_RE.search(msg.content)          if matches:              await self.delete_and_respond(msg, matches[1] + "xxx") diff --git a/bot/constants.py b/bot/constants.py index fd280e9de..a1b392c82 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -15,7 +15,7 @@ import os  from collections.abc import Mapping  from enum import Enum  from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional  import yaml @@ -198,7 +198,18 @@ class Bot(metaclass=YAMLGetter):      prefix: str      token: str -    sentry_dsn: str +    sentry_dsn: Optional[str] + + +class Redis(metaclass=YAMLGetter): +    section = "bot" +    subsection = "redis" + +    host: str +    port: int +    password: Optional[str] +    use_fakeredis: bool  # If this is True, Bot will use fakeredis.aioredis +  class Filter(metaclass=YAMLGetter):      section = "filter" @@ -215,6 +226,7 @@ class Filter(metaclass=YAMLGetter):      notify_user_domains: bool      ping_everyone: bool +    offensive_msg_delete_days: int      guild_invite_whitelist: List[int]      domain_blacklist: List[str]      word_watchlist: List[str] @@ -365,6 +377,7 @@ class Categories(metaclass=YAMLGetter):      help_available: int      help_in_use: int      help_dormant: int +    modmail: int  class Channels(metaclass=YAMLGetter): @@ -377,6 +390,7 @@ class Channels(metaclass=YAMLGetter):      attachment_log: int      big_brother_logs: int      bot_commands: int +    cooldown: int      defcon: int      dev_contrib: int      dev_core: int @@ -384,6 +398,7 @@ class Channels(metaclass=YAMLGetter):      esoteric: int      helpers: int      how_to_get_help: int +    incidents: int      message_log: int      meta: int      mod_alerts: int @@ -448,7 +463,7 @@ class Guild(metaclass=YAMLGetter):  class Keys(metaclass=YAMLGetter):      section = "keys" -    site_api: str +    site_api: Optional[str]  class URLs(metaclass=YAMLGetter): @@ -491,8 +506,8 @@ class Reddit(metaclass=YAMLGetter):      section = "reddit"      subreddits: list -    client_id: str -    secret: str +    client_id: Optional[str] +    secret: Optional[str]  class Wolfram(metaclass=YAMLGetter): @@ -500,7 +515,7 @@ class Wolfram(metaclass=YAMLGetter):      user_limit_day: int      guild_limit_day: int -    key: str +    key: Optional[str]  class AntiSpam(metaclass=YAMLGetter): @@ -541,6 +556,7 @@ class HelpChannels(metaclass=YAMLGetter):      claim_minutes: int      cmd_whitelist: List[int]      idle_minutes: int +    deleted_idle_minutes: int      max_available: int      max_total_channels: int      name_prefix: str @@ -609,13 +625,10 @@ PROJECT_ROOT = os.path.abspath(os.path.join(BOT_DIR, os.pardir))  MODERATION_ROLES = Guild.moderation_roles  STAFF_ROLES = Guild.staff_roles -# Roles combinations +# Channel combinations  STAFF_CHANNELS = Guild.staff_channels - -# Default Channel combinations  MODERATION_CHANNELS = Guild.moderation_channels -  # Bot replies  NEGATIVE_REPLIES = [      "Noooooo!!", diff --git a/bot/converters.py b/bot/converters.py index 72c46fdf0..4deb59f87 100644 --- a/bot/converters.py +++ b/bot/converters.py @@ -217,7 +217,10 @@ class Duration(Converter):          delta = relativedelta(**duration_dict)          now = datetime.utcnow() -        return now + delta +        try: +            return now + delta +        except ValueError: +            raise BadArgument(f"`{duration}` results in a datetime outside the supported range.")  class ISODateTime(Converter): diff --git a/bot/decorators.py b/bot/decorators.py index 306f0830c..500197c89 100644 --- a/bot/decorators.py +++ b/bot/decorators.py @@ -9,37 +9,21 @@ from weakref import WeakValueDictionary  from discord import Colour, Embed, Member  from discord.errors import NotFound  from discord.ext import commands -from discord.ext.commands import CheckFailure, Cog, Context +from discord.ext.commands import Cog, Context  from bot.constants import Channels, ERROR_REPLIES, RedirectOutput -from bot.utils.checks import with_role_check, without_role_check +from bot.utils.checks import in_whitelist_check, with_role_check, without_role_check  log = logging.getLogger(__name__) -class InWhitelistCheckFailure(CheckFailure): -    """Raised when the `in_whitelist` check fails.""" - -    def __init__(self, redirect_channel: Optional[int]) -> None: -        self.redirect_channel = redirect_channel - -        if redirect_channel: -            redirect_message = f" here. Please use the <#{redirect_channel}> channel instead" -        else: -            redirect_message = "" - -        error_message = f"You are not allowed to use that command{redirect_message}." - -        super().__init__(error_message) - -  def in_whitelist(      *,      channels: Container[int] = (),      categories: Container[int] = (),      roles: Container[int] = (),      redirect: Optional[int] = Channels.bot_commands, - +    fail_silently: bool = False,  ) -> Callable:      """      Check if a command was issued in a whitelisted context. @@ -54,36 +38,9 @@ def in_whitelist(      redirected to the `redirect` channel that was passed (default: #bot-commands) or simply      told that they're not allowed to use this particular command (if `None` was passed).      """ -    if redirect and redirect not in channels: -        # It does not make sense for the channel whitelist to not contain the redirection -        # channel (if applicable). That's why we add the redirection channel to the `channels` -        # container if it's not already in it. As we allow any container type to be passed, -        # we first create a tuple in order to safely add the redirection channel. -        # -        # Note: It's possible for the redirect channel to be in a whitelisted category, but -        # there's no easy way to check that and as a channel can easily be moved in and out of -        # categories, it's probably not wise to rely on its category in any case. -        channels = tuple(channels) + (redirect,) -      def predicate(ctx: Context) -> bool: -        """Check if a command was issued in a whitelisted context.""" -        if channels and ctx.channel.id in channels: -            log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted channel.") -            return True - -        # Only check the category id if we have a category whitelist and the channel has a `category_id` -        if categories and hasattr(ctx.channel, "category_id") and ctx.channel.category_id in categories: -            log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted category.") -            return True - -        # Only check the roles whitelist if we have one and ensure the author's roles attribute returns -        # an iterable to prevent breakage in DM channels (for if we ever decide to enable commands there). -        if roles and any(r.id in roles for r in getattr(ctx.author, "roles", ())): -            log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they have a whitelisted role.") -            return True - -        log.trace(f"{ctx.author} may not use the `{ctx.command.name}` command within this context.") -        raise InWhitelistCheckFailure(redirect) +        """Check if command was issued in a whitelisted context.""" +        return in_whitelist_check(ctx, channels, categories, roles, redirect, fail_silently)      return commands.check(predicate) @@ -121,7 +78,7 @@ def locked() -> Callable:                  embed = Embed()                  embed.colour = Colour.red() -                log.debug(f"User tried to invoke a locked command.") +                log.debug("User tried to invoke a locked command.")                  embed.description = (                      "You're already using this command. Please wait until it is done before you use it again."                  ) diff --git a/bot/pagination.py b/bot/pagination.py index b0c4b70e2..2aa3590ba 100644 --- a/bot/pagination.py +++ b/bot/pagination.py @@ -147,7 +147,7 @@ class LinePaginator(Paginator):          if not lines:              if exception_on_empty_embed: -                log.exception(f"Pagination asked for empty lines iterable") +                log.exception("Pagination asked for empty lines iterable")                  raise EmptyPaginatorEmbed("No lines to paginate")              log.debug("No lines to add to paginator, adding '(nothing to display)' message") @@ -357,7 +357,7 @@ class ImagePaginator(Paginator):          if not pages:              if exception_on_empty_embed: -                log.exception(f"Pagination asked for empty image list") +                log.exception("Pagination asked for empty image list")                  raise EmptyPaginatorEmbed("No images to paginate")              log.debug("No images to add to paginator, adding '(no images to display)' message") diff --git a/bot/resources/tags/customcooldown.md b/bot/resources/tags/customcooldown.md new file mode 100644 index 000000000..ac7e70aee --- /dev/null +++ b/bot/resources/tags/customcooldown.md @@ -0,0 +1,20 @@ +**Cooldowns in discord.py** + +Cooldowns can be used in discord.py to rate-limit. In this example, we're using it in an on_message. + +```python +from discord.ext import commands + +message_cooldown = commands.CooldownMapping.from_cooldown(1.0, 60.0, commands.BucketType.user) + +async def on_message(message): +    bucket = message_cooldown.get_bucket(message) +    retry_after = bucket.update_rate_limit() +    if retry_after: +        await message.channel.send(f"Slow down! Try again in {retry_after} seconds.") +    else: +        await message.channel.send("Not ratelimited!") +``` + +`from_cooldown` takes the amount of `update_rate_limit()`s needed to trigger the cooldown, the time in which the cooldown is triggered, and a [`BucketType`](https://discordpy.readthedocs.io/en/latest/ext/commands/api.html#discord.discord.ext.commands.BucketType). diff --git a/bot/resources/tags/free.md b/bot/resources/tags/free.md index 582cca9da..1493076c7 100644 --- a/bot/resources/tags/free.md +++ b/bot/resources/tags/free.md @@ -1,5 +1,5 @@  **We have a new help channel system!** -We recently moved to a new help channel system. You can now use any channel in the **<#691405807388196926>** category to ask your question. +Please see <#704250143020417084> for further information. -For more information, check out [our website](https://pythondiscord.com/pages/resources/guides/help-channels/). +A more detailed guide can be found on [our website](https://pythondiscord.com/pages/resources/guides/help-channels/). diff --git a/bot/resources/tags/modmail.md b/bot/resources/tags/modmail.md new file mode 100644 index 000000000..7545419ee --- /dev/null +++ b/bot/resources/tags/modmail.md @@ -0,0 +1,9 @@ +**Contacting the moderation team via ModMail** + +<@!683001325440860340> is a bot that will relay your messages to our moderation team, so that you can start a conversation with the moderation team. Your messages will be relayed to the entire moderator team, who will be able to respond to you via the bot. + +It supports attachments, codeblocks, and reactions. As communication happens over direct messages, the conversation will stay between you and the mod team. + +**To use it, simply send a direct message to the bot.** + +Should there be an urgent and immediate need for a moderator or admin to look at a channel, feel free to ping the <@&267629731250176001> or <@&267628507062992896> role instead. diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 9b32e515d..5a6e1811b 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -2,8 +2,17 @@ from abc import ABCMeta  from discord.ext.commands import CogMeta +from bot.utils.redis_cache import RedisCache + +__all__ = ['RedisCache', 'CogABCMeta'] +  class CogABCMeta(CogMeta, ABCMeta):      """Metaclass for ABCs meant to be implemented as Cogs."""      pass + + +def pad_base64(data: str) -> str: +    """Return base64 `data` with padding characters to ensure its length is a multiple of 4.""" +    return data + "=" * (-len(data) % 4) diff --git a/bot/utils/checks.py b/bot/utils/checks.py index db56c347c..f0ef36302 100644 --- a/bot/utils/checks.py +++ b/bot/utils/checks.py @@ -1,12 +1,94 @@  import datetime  import logging -from typing import Callable, Iterable +from typing import Callable, Container, Iterable, Optional -from discord.ext.commands import BucketType, Cog, Command, CommandOnCooldown, Context, Cooldown, CooldownMapping +from discord.ext.commands import ( +    BucketType, +    CheckFailure, +    Cog, +    Command, +    CommandOnCooldown, +    Context, +    Cooldown, +    CooldownMapping, +) + +from bot import constants  log = logging.getLogger(__name__) +class InWhitelistCheckFailure(CheckFailure): +    """Raised when the `in_whitelist` check fails.""" + +    def __init__(self, redirect_channel: Optional[int]) -> None: +        self.redirect_channel = redirect_channel + +        if redirect_channel: +            redirect_message = f" here. Please use the <#{redirect_channel}> channel instead" +        else: +            redirect_message = "" + +        error_message = f"You are not allowed to use that command{redirect_message}." + +        super().__init__(error_message) + + +def in_whitelist_check( +    ctx: Context, +    channels: Container[int] = (), +    categories: Container[int] = (), +    roles: Container[int] = (), +    redirect: Optional[int] = constants.Channels.bot_commands, +    fail_silently: bool = False, +) -> bool: +    """ +    Check if a command was issued in a whitelisted context. + +    The whitelists that can be provided are: + +    - `channels`: a container with channel ids for whitelisted channels +    - `categories`: a container with category ids for whitelisted categories +    - `roles`: a container with with role ids for whitelisted roles + +    If the command was invoked in a context that was not whitelisted, the member is either +    redirected to the `redirect` channel that was passed (default: #bot-commands) or simply +    told that they're not allowed to use this particular command (if `None` was passed). +    """ +    if redirect and redirect not in channels: +        # It does not make sense for the channel whitelist to not contain the redirection +        # channel (if applicable). That's why we add the redirection channel to the `channels` +        # container if it's not already in it. As we allow any container type to be passed, +        # we first create a tuple in order to safely add the redirection channel. +        # +        # Note: It's possible for the redirect channel to be in a whitelisted category, but +        # there's no easy way to check that and as a channel can easily be moved in and out of +        # categories, it's probably not wise to rely on its category in any case. +        channels = tuple(channels) + (redirect,) + +    if channels and ctx.channel.id in channels: +        log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted channel.") +        return True + +    # Only check the category id if we have a category whitelist and the channel has a `category_id` +    if categories and hasattr(ctx.channel, "category_id") and ctx.channel.category_id in categories: +        log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they are in a whitelisted category.") +        return True + +    # Only check the roles whitelist if we have one and ensure the author's roles attribute returns +    # an iterable to prevent breakage in DM channels (for if we ever decide to enable commands there). +    if roles and any(r.id in roles for r in getattr(ctx.author, "roles", ())): +        log.trace(f"{ctx.author} may use the `{ctx.command.name}` command as they have a whitelisted role.") +        return True + +    log.trace(f"{ctx.author} may not use the `{ctx.command.name}` command within this context.") + +    # Some commands are secret, and should produce no feedback at all. +    if not fail_silently: +        raise InWhitelistCheckFailure(redirect) +    return False + +  def with_role_check(ctx: Context, *role_ids: int) -> bool:      """Returns True if the user has any one of the roles in role_ids."""      if not ctx.guild:  # Return False in a DM @@ -38,14 +120,6 @@ def without_role_check(ctx: Context, *role_ids: int) -> bool:      return check -def in_channel_check(ctx: Context, *channel_ids: int) -> bool: -    """Checks if the command was executed inside the list of specified channels.""" -    check = ctx.channel.id in channel_ids -    log.trace(f"{ctx.author} tried to call the '{ctx.command.name}' command. " -              f"The result of the in_channel check was {check}.") -    return check - -  def cooldown_with_role_bypass(rate: int, per: float, type: BucketType = BucketType.default, *,                                bypass_roles: Iterable[int]) -> Callable:      """ diff --git a/bot/utils/messages.py b/bot/utils/messages.py index e969ee590..a40a12e98 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -1,6 +1,7 @@  import asyncio  import contextlib  import logging +import re  from io import BytesIO  from typing import List, Optional, Sequence, Union @@ -86,7 +87,7 @@ async def send_attachments(                      else:                          await destination.send(                              file=attachment_file, -                            username=message.author.display_name, +                            username=sub_clyde(message.author.display_name),                              avatar_url=message.author.avatar_url                          )              elif link_large: @@ -97,10 +98,10 @@ async def send_attachments(              if link_large and e.status == 413:                  large.append(attachment)              else: -                log.warning(f"{failure_msg} with status {e.status}.") +                log.warning(f"{failure_msg} with status {e.status}.", exc_info=e)      if link_large and large: -        desc = f"\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large) +        desc = "\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large)          embed = Embed(description=desc)          embed.set_footer(text="Attachments exceed upload size limit.") @@ -109,8 +110,25 @@ async def send_attachments(          else:              await destination.send(                  embed=embed, -                username=message.author.display_name, +                username=sub_clyde(message.author.display_name),                  avatar_url=message.author.avatar_url              )      return urls + + +def sub_clyde(username: Optional[str]) -> Optional[str]: +    """ +    Replace "e"/"E" in any "clyde" in `username` with a Cyrillic "е"/"E" and return the new string. + +    Discord disallows "clyde" anywhere in the username for webhooks. It will return a 400. +    Return None only if `username` is None. +    """ +    def replace_e(match: re.Match) -> str: +        char = "е" if match[2] == "e" else "Е" +        return match[1] + char + +    if username: +        return re.sub(r"(clyd)(e)", replace_e, username, flags=re.I) +    else: +        return username  # Empty string or None diff --git a/bot/utils/redis_cache.py b/bot/utils/redis_cache.py new file mode 100644 index 000000000..58cfe1df5 --- /dev/null +++ b/bot/utils/redis_cache.py @@ -0,0 +1,415 @@ +from __future__ import annotations + +import asyncio +import logging +from functools import partialmethod +from typing import Any, Dict, ItemsView, Optional, Tuple, Union + +from bot.bot import Bot + +log = logging.getLogger(__name__) + +# Type aliases +RedisKeyType = Union[str, int] +RedisValueType = Union[str, int, float, bool] +RedisKeyOrValue = Union[RedisKeyType, RedisValueType] + +# Prefix tuples +_PrefixTuple = Tuple[Tuple[str, Any], ...] +_VALUE_PREFIXES = ( +    ("f|", float), +    ("i|", int), +    ("s|", str), +    ("b|", bool), +) +_KEY_PREFIXES = ( +    ("i|", int), +    ("s|", str), +) + + +class NoBotInstanceError(RuntimeError): +    """Raised when RedisCache is created without an available bot instance on the owner class.""" + + +class NoNamespaceError(RuntimeError): +    """Raised when RedisCache has no namespace, for example if it is not assigned to a class attribute.""" + + +class NoParentInstanceError(RuntimeError): +    """Raised when the parent instance is available, for example if called by accessing the parent class directly.""" + + +class RedisCache: +    """ +    A simplified interface for a Redis connection. + +    We implement several convenient methods that are fairly similar to have a dict +    behaves, and should be familiar to Python users. The biggest difference is that +    all the public methods in this class are coroutines, and must be awaited. + +    Because of limitations in Redis, this cache will only accept strings and integers for keys, +    and strings, integers, floats and booleans for values. + +    Please note that this class MUST be created as a class attribute, and that that class +    must also contain an attribute with an instance of our Bot. See `__get__` and `__set_name__` +    for more information about how this works. + +    Simple example for how to use this: + +    class SomeCog(Cog): +        # To initialize a valid RedisCache, just add it as a class attribute here. +        # Do not add it to the __init__ method or anywhere else, it MUST be a class +        # attribute. Do not pass any parameters. +        cache = RedisCache() + +        async def my_method(self): + +            # Now we're ready to use the RedisCache. +            # One thing to note here is that this will not work unless +            # we access self.cache through an _instance_ of this class. +            # +            # For example, attempting to use SomeCog.cache will _not_ work, +            # you _must_ instantiate the class first and use that instance. +            # +            # Now we can store some stuff in the cache just by doing this. +            # This data will persist through restarts! +            await self.cache.set("key", "value") + +            # To get the data, simply do this. +            value = await self.cache.get("key") + +            # Other methods work more or less like a dictionary. +            # Checking if something is in the cache +            await self.cache.contains("key") + +            # iterating the cache +            async for key, value in self.cache.items(): +                print(value) + +            # We can even iterate in a comprehension! +            consumed = [value async for key, value in self.cache.items()] +    """ + +    _namespaces = [] + +    def __init__(self) -> None: +        """Initialize the RedisCache.""" +        self._namespace = None +        self.bot = None +        self._increment_lock = None + +    def _set_namespace(self, namespace: str) -> None: +        """Try to set the namespace, but do not permit collisions.""" +        log.trace(f"RedisCache setting namespace to {namespace}") +        self._namespaces.append(namespace) +        self._namespace = namespace + +    @staticmethod +    def _to_typestring(key_or_value: RedisKeyOrValue, prefixes: _PrefixTuple) -> str: +        """Turn a valid Redis type into a typestring.""" +        for prefix, _type in prefixes: +            # Convert bools into integers before storing them. +            if type(key_or_value) is bool: +                bool_int = int(key_or_value) +                return f"{prefix}{bool_int}" + +            # isinstance is a bad idea here, because isintance(False, int) == True. +            if type(key_or_value) is _type: +                return f"{prefix}{key_or_value}" + +        raise TypeError(f"RedisCache._to_typestring only supports the following: {prefixes}.") + +    @staticmethod +    def _from_typestring(key_or_value: Union[bytes, str], prefixes: _PrefixTuple) -> RedisKeyOrValue: +        """Deserialize a typestring into a valid Redis type.""" +        # Stuff that comes out of Redis will be bytestrings, so let's decode those. +        if isinstance(key_or_value, bytes): +            key_or_value = key_or_value.decode('utf-8') + +        # Now we convert our unicode string back into the type it originally was. +        for prefix, _type in prefixes: +            if key_or_value.startswith(prefix): + +                # For booleans, we need special handling because bool("False") is True. +                if prefix == "b|": +                    value = key_or_value[len(prefix):] +                    return bool(int(value)) + +                # Otherwise we can just convert normally. +                return _type(key_or_value[len(prefix):]) +        raise TypeError(f"RedisCache._from_typestring only supports the following: {prefixes}.") + +    # Add some nice partials to call our generic typestring converters. +    # These are basically methods that will fill in some of the parameters for you, so that +    # any call to _key_to_typestring will be like calling _to_typestring with the two parameters +    # at `prefixes` and `types_string` pre-filled. +    # +    # See https://docs.python.org/3/library/functools.html#functools.partialmethod +    _key_to_typestring = partialmethod(_to_typestring, prefixes=_KEY_PREFIXES) +    _value_to_typestring = partialmethod(_to_typestring, prefixes=_VALUE_PREFIXES) +    _key_from_typestring = partialmethod(_from_typestring, prefixes=_KEY_PREFIXES) +    _value_from_typestring = partialmethod(_from_typestring, prefixes=_VALUE_PREFIXES) + +    def _dict_from_typestring(self, dictionary: Dict) -> Dict: +        """Turns all contents of a dict into valid Redis types.""" +        return {self._key_from_typestring(key): self._value_from_typestring(value) for key, value in dictionary.items()} + +    def _dict_to_typestring(self, dictionary: Dict) -> Dict: +        """Turns all contents of a dict into typestrings.""" +        return {self._key_to_typestring(key): self._value_to_typestring(value) for key, value in dictionary.items()} + +    async def _validate_cache(self) -> None: +        """Validate that the RedisCache is ready to be used.""" +        if self._namespace is None: +            error_message = ( +                "Critical error: RedisCache has no namespace. " +                "This object must be initialized as a class attribute." +            ) +            log.error(error_message) +            raise NoNamespaceError(error_message) + +        if self.bot is None: +            error_message = ( +                "Critical error: RedisCache has no `Bot` instance. " +                "This happens when the class RedisCache was created in doesn't " +                "have a Bot instance. Please make sure that you're instantiating " +                "the RedisCache inside a class that has a Bot instance attribute." +            ) +            log.error(error_message) +            raise NoBotInstanceError(error_message) + +        if not self.bot.redis_closed: +            await self.bot.redis_ready.wait() + +    def __set_name__(self, owner: Any, attribute_name: str) -> None: +        """ +        Set the namespace to Class.attribute_name. + +        Called automatically when this class is constructed inside a class as an attribute. + +        This class MUST be created as a class attribute in a class, otherwise it will raise +        exceptions whenever a method is used. This is because it uses this method to create +        a namespace like `MyCog.my_class_attribute` which is used as a hash name when we store +        stuff in Redis, to prevent collisions. +        """ +        self._set_namespace(f"{owner.__name__}.{attribute_name}") + +    def __get__(self, instance: RedisCache, owner: Any) -> RedisCache: +        """ +        This is called if the RedisCache is a class attribute, and is accessed. + +        The class this object is instantiated in must contain an attribute with an +        instance of Bot. This is because Bot contains our redis_session, which is +        the mechanism by which we will communicate with the Redis server. + +        Any attempt to use RedisCache in a class that does not have a Bot instance +        will fail. It is mostly intended to be used inside of a Cog, although theoretically +        it should work in any class that has a Bot instance. +        """ +        if self.bot: +            return self + +        if self._namespace is None: +            error_message = "RedisCache must be a class attribute." +            log.error(error_message) +            raise NoNamespaceError(error_message) + +        if instance is None: +            error_message = ( +                "You must access the RedisCache instance through the cog instance " +                "before accessing it using the cog's class object." +            ) +            log.error(error_message) +            raise NoParentInstanceError(error_message) + +        for attribute in vars(instance).values(): +            if isinstance(attribute, Bot): +                self.bot = attribute +                self._redis = self.bot.redis_session +                return self +        else: +            error_message = ( +                "Critical error: RedisCache has no `Bot` instance. " +                "This happens when the class RedisCache was created in doesn't " +                "have a Bot instance. Please make sure that you're instantiating " +                "the RedisCache inside a class that has a Bot instance attribute." +            ) +            log.error(error_message) +            raise NoBotInstanceError(error_message) + +    def __repr__(self) -> str: +        """Return a beautiful representation of this object instance.""" +        return f"RedisCache(namespace={self._namespace!r})" + +    async def set(self, key: RedisKeyType, value: RedisValueType) -> None: +        """Store an item in the Redis cache.""" +        await self._validate_cache() + +        # Convert to a typestring and then set it +        key = self._key_to_typestring(key) +        value = self._value_to_typestring(value) + +        log.trace(f"Setting {key} to {value}.") +        await self._redis.hset(self._namespace, key, value) + +    async def get(self, key: RedisKeyType, default: Optional[RedisValueType] = None) -> Optional[RedisValueType]: +        """Get an item from the Redis cache.""" +        await self._validate_cache() +        key = self._key_to_typestring(key) + +        log.trace(f"Attempting to retrieve {key}.") +        value = await self._redis.hget(self._namespace, key) + +        if value is None: +            log.trace(f"Value not found, returning default value {default}") +            return default +        else: +            value = self._value_from_typestring(value) +            log.trace(f"Value found, returning value {value}") +            return value + +    async def delete(self, key: RedisKeyType) -> None: +        """ +        Delete an item from the Redis cache. + +        If we try to delete a key that does not exist, it will simply be ignored. + +        See https://redis.io/commands/hdel for more info on how this works. +        """ +        await self._validate_cache() +        key = self._key_to_typestring(key) + +        log.trace(f"Attempting to delete {key}.") +        return await self._redis.hdel(self._namespace, key) + +    async def contains(self, key: RedisKeyType) -> bool: +        """ +        Check if a key exists in the Redis cache. + +        Return True if the key exists, otherwise False. +        """ +        await self._validate_cache() +        key = self._key_to_typestring(key) +        exists = await self._redis.hexists(self._namespace, key) + +        log.trace(f"Testing if {key} exists in the RedisCache - Result is {exists}") +        return exists + +    async def items(self) -> ItemsView: +        """ +        Fetch all the key/value pairs in the cache. + +        Returns a normal ItemsView, like you would get from dict.items(). + +        Keep in mind that these items are just a _copy_ of the data in the +        RedisCache - any changes you make to them will not be reflected +        into the RedisCache itself. If you want to change these, you need +        to make a .set call. + +        Example: +        items = await my_cache.items() +        for key, value in items: +            # Iterate like a normal dictionary +        """ +        await self._validate_cache() +        items = self._dict_from_typestring( +            await self._redis.hgetall(self._namespace) +        ).items() + +        log.trace(f"Retrieving all key/value pairs from cache, total of {len(items)} items.") +        return items + +    async def length(self) -> int: +        """Return the number of items in the Redis cache.""" +        await self._validate_cache() +        number_of_items = await self._redis.hlen(self._namespace) +        log.trace(f"Returning length. Result is {number_of_items}.") +        return number_of_items + +    async def to_dict(self) -> Dict: +        """Convert to dict and return.""" +        return {key: value for key, value in await self.items()} + +    async def clear(self) -> None: +        """Deletes the entire hash from the Redis cache.""" +        await self._validate_cache() +        log.trace("Clearing the cache of all key/value pairs.") +        await self._redis.delete(self._namespace) + +    async def pop(self, key: RedisKeyType, default: Optional[RedisValueType] = None) -> RedisValueType: +        """Get the item, remove it from the cache, and provide a default if not found.""" +        log.trace(f"Attempting to pop {key}.") +        value = await self.get(key, default) + +        log.trace( +            f"Attempting to delete item with key '{key}' from the cache. " +            "If this key doesn't exist, nothing will happen." +        ) +        await self.delete(key) + +        return value + +    async def update(self, items: Dict[RedisKeyType, RedisValueType]) -> None: +        """ +        Update the Redis cache with multiple values. + +        This works exactly like dict.update from a normal dictionary. You pass +        a dictionary with one or more key/value pairs into this method. If the keys +        do not exist in the RedisCache, they are created. If they do exist, the values +        are updated with the new ones from `items`. + +        Please note that keys and the values in the `items` dictionary +        must consist of valid RedisKeyTypes and RedisValueTypes. +        """ +        await self._validate_cache() +        log.trace(f"Updating the cache with the following items:\n{items}") +        await self._redis.hmset_dict(self._namespace, self._dict_to_typestring(items)) + +    async def increment(self, key: RedisKeyType, amount: Optional[int, float] = 1) -> None: +        """ +        Increment the value by `amount`. + +        This works for both floats and ints, but will raise a TypeError +        if you try to do it for any other type of value. + +        This also supports negative amounts, although it would provide better +        readability to use .decrement() for that. +        """ +        log.trace(f"Attempting to increment/decrement the value with the key {key} by {amount}.") + +        # We initialize the lock here, because we need to ensure we get it +        # running on the same loop as the calling coroutine. +        # +        # If we initialized the lock in the __init__, the loop that the coroutine this method +        # would be called from might not exist yet, and so the lock would be on a different +        # loop, which would raise RuntimeErrors. +        if self._increment_lock is None: +            self._increment_lock = asyncio.Lock() + +        # Since this has several API calls, we need a lock to prevent race conditions +        async with self._increment_lock: +            value = await self.get(key) + +            # Can't increment a non-existing value +            if value is None: +                error_message = "The provided key does not exist!" +                log.error(error_message) +                raise KeyError(error_message) + +            # If it does exist, and it's an int or a float, increment and set it. +            if isinstance(value, int) or isinstance(value, float): +                value += amount +                await self.set(key, value) +            else: +                error_message = "You may only increment or decrement values that are integers or floats." +                log.error(error_message) +                raise TypeError(error_message) + +    async def decrement(self, key: RedisKeyType, amount: Optional[int, float] = 1) -> None: +        """ +        Decrement the value by `amount`. + +        Basically just does the opposite of .increment. +        """ +        await self.increment(key, -amount) diff --git a/config-default.yml b/config-default.yml index 83ea59016..64c4e715b 100644 --- a/config-default.yml +++ b/config-default.yml @@ -3,6 +3,12 @@ bot:      token:       !ENV "BOT_TOKEN"      sentry_dsn:  !ENV "BOT_SENTRY_DSN" +    redis: +        host:  "redis" +        port:  6379 +        password: !ENV "REDIS_PASSWORD" +        use_fakeredis: false +      stats:          statsd_host: "graphite"          presence_update_timeout: 300 @@ -118,6 +124,7 @@ guild:          help_available:                     691405807388196926          help_in_use:                        696958401460043776          help_dormant:                       691405908919451718 +        modmail:                            714494672835444826      channels:          announcements:                              354619224620138496 @@ -135,6 +142,7 @@ guild:          # Python Help: Available          how_to_get_help:    704250143020417084 +        cooldown:           720603994149486673          # Logs          attachment_log:     &ATTACH_LOG     649243850006855680 @@ -164,6 +172,7 @@ guild:          mod_spam:           &MOD_SPAM       620607373828030464          organisation:       &ORGANISATION   551789653284356126          staff_lounge:       &STAFF_LOUNGE   464905259261755392 +        incidents:                          714214212200562749          # Voice          admins_voice:       &ADMINS_VOICE   500734494840717332 @@ -260,7 +269,8 @@ filter:      notify_user_domains:     false      # Filter configuration -    ping_everyone: true  # Ping @everyone when we send a mod-alert? +    ping_everyone:             true  # Ping @everyone when we send a mod-alert? +    offensive_msg_delete_days: 7     # How many days before deleting an offensive message?      guild_invite_whitelist:          - 280033776820813825  # Functional Programming @@ -288,6 +298,9 @@ filter:          - 81384788765712384   # Discord API          - 613425648685547541  # Discord Developers          - 185590609631903755  # Blender Hub +        - 420324994703163402  # /r/FlutterDev +        - 488751051629920277  # Python Atlanta +        - 143867839282020352  # C#      domain_blacklist:          - pornhub.com @@ -316,6 +329,10 @@ filter:          - poweredbydialup.online          - poweredbysecurity.org          - poweredbysecurity.online +        - ssteam.site +        - steamwalletgift.com +        - discord.gift +        - lmgtfy.com      word_watchlist:          - goo+ks* @@ -529,6 +546,10 @@ help_channels:      # Allowed duration of inactivity before making a channel dormant      idle_minutes: 30 +    # Allowed duration of inactivity when question message deleted +    # and no one other sent before message making channel dormant. +    deleted_idle_minutes: 5 +      # Maximum number of channels to put in the available category      max_available: 2 diff --git a/docker-compose.yml b/docker-compose.yml index 11deceae8..cff7d33d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,19 @@ services:        POSTGRES_PASSWORD: pysite        POSTGRES_USER: pysite +  redis: +    image: redis:5.0.9 +    ports: +      - "127.0.0.1:6379:6379" + +  snekbox: +    image: pythondiscord/snekbox:latest +    init: true +    ipc: none +    ports: +     - "127.0.0.1:8060:8060" +    privileged: true +    web:      image: pythondiscord/site:latest      command: ["run", "--debug"] @@ -41,6 +54,8 @@ services:      tty: true      depends_on:        - web +      - redis +      - snekbox      environment:        BOT_TOKEN: ${BOT_TOKEN}        BOT_API_KEY: badbot13m0n8f570f942013fc818f234916ca531 diff --git a/tests/bot/cogs/moderation/test_infractions.py b/tests/bot/cogs/moderation/test_infractions.py new file mode 100644 index 000000000..da4e92ccc --- /dev/null +++ b/tests/bot/cogs/moderation/test_infractions.py @@ -0,0 +1,55 @@ +import textwrap +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from bot.cogs.moderation.infractions import Infractions +from tests.helpers import MockBot, MockContext, MockGuild, MockMember, MockRole + + +class TruncationTests(unittest.IsolatedAsyncioTestCase): +    """Tests for ban and kick command reason truncation.""" + +    def setUp(self): +        self.bot = MockBot() +        self.cog = Infractions(self.bot) +        self.user = MockMember(id=1234, top_role=MockRole(id=3577, position=10)) +        self.target = MockMember(id=1265, top_role=MockRole(id=9876, position=0)) +        self.guild = MockGuild(id=4567) +        self.ctx = MockContext(bot=self.bot, author=self.user, guild=self.guild) + +    @patch("bot.cogs.moderation.utils.get_active_infraction") +    @patch("bot.cogs.moderation.utils.post_infraction") +    async def test_apply_ban_reason_truncation(self, post_infraction_mock, get_active_mock): +        """Should truncate reason for `ctx.guild.ban`.""" +        get_active_mock.return_value = None +        post_infraction_mock.return_value = {"foo": "bar"} + +        self.cog.apply_infraction = AsyncMock() +        self.bot.get_cog.return_value = AsyncMock() +        self.cog.mod_log.ignore = Mock() +        self.ctx.guild.ban = Mock() + +        await self.cog.apply_ban(self.ctx, self.target, "foo bar" * 3000) +        self.ctx.guild.ban.assert_called_once_with( +            self.target, +            reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="..."), +            delete_message_days=0 +        ) +        self.cog.apply_infraction.assert_awaited_once_with( +            self.ctx, {"foo": "bar"}, self.target, self.ctx.guild.ban.return_value +        ) + +    @patch("bot.cogs.moderation.utils.post_infraction") +    async def test_apply_kick_reason_truncation(self, post_infraction_mock): +        """Should truncate reason for `Member.kick`.""" +        post_infraction_mock.return_value = {"foo": "bar"} + +        self.cog.apply_infraction = AsyncMock() +        self.cog.mod_log.ignore = Mock() +        self.target.kick = Mock() + +        await self.cog.apply_kick(self.ctx, self.target, "foo bar" * 3000) +        self.target.kick.assert_called_once_with(reason=textwrap.shorten("foo bar" * 3000, 512, placeholder="...")) +        self.cog.apply_infraction.assert_awaited_once_with( +            self.ctx, {"foo": "bar"}, self.target, self.target.kick.return_value +        ) diff --git a/tests/bot/cogs/moderation/test_modlog.py b/tests/bot/cogs/moderation/test_modlog.py new file mode 100644 index 000000000..f2809f40a --- /dev/null +++ b/tests/bot/cogs/moderation/test_modlog.py @@ -0,0 +1,29 @@ +import unittest + +import discord + +from bot.cogs.moderation.modlog import ModLog +from tests.helpers import MockBot, MockTextChannel + + +class ModLogTests(unittest.IsolatedAsyncioTestCase): +    """Tests for moderation logs.""" + +    def setUp(self): +        self.bot = MockBot() +        self.cog = ModLog(self.bot) +        self.channel = MockTextChannel() + +    async def test_log_entry_description_truncation(self): +        """Test that embed description for ModLog entry is truncated.""" +        self.bot.get_channel.return_value = self.channel +        await self.cog.send_log_message( +            icon_url="foo", +            colour=discord.Colour.blue(), +            title="bar", +            text="foo bar" * 3000 +        ) +        embed = self.channel.send.call_args[1]["embed"] +        self.assertEqual( +            embed.description, ("foo bar" * 3000)[:2045] + "..." +        ) diff --git a/tests/bot/cogs/moderation/test_silence.py b/tests/bot/cogs/moderation/test_silence.py index 3fd149f04..ab3d0742a 100644 --- a/tests/bot/cogs/moderation/test_silence.py +++ b/tests/bot/cogs/moderation/test_silence.py @@ -127,10 +127,20 @@ class SilenceTests(unittest.IsolatedAsyncioTestCase):              self.ctx.reset_mock()      async def test_unsilence_sent_correct_discord_message(self): -        """Proper reply after a successful unsilence.""" -        with mock.patch.object(self.cog, "_unsilence", return_value=True): -            await self.cog.unsilence.callback(self.cog, self.ctx) -            self.ctx.send.assert_called_once_with(f"{Emojis.check_mark} unsilenced current channel.") +        """Check if proper message was sent when unsilencing channel.""" +        test_cases = ( +            (True, f"{Emojis.check_mark} unsilenced current channel."), +            (False, f"{Emojis.cross_mark} current channel was not silenced.") +        ) +        for _unsilence_patch_return, result_message in test_cases: +            with self.subTest( +                starting_silenced_state=_unsilence_patch_return, +                result_message=result_message +            ): +                with mock.patch.object(self.cog, "_unsilence", return_value=_unsilence_patch_return): +                    await self.cog.unsilence.callback(self.cog, self.ctx) +                    self.ctx.send.assert_called_once_with(result_message) +            self.ctx.reset_mock()      async def test_silence_private_for_false(self):          """Permissions are not set and `False` is returned in an already silenced channel.""" diff --git a/tests/bot/cogs/sync/test_cog.py b/tests/bot/cogs/sync/test_cog.py index 81398c61f..14fd909c4 100644 --- a/tests/bot/cogs/sync/test_cog.py +++ b/tests/bot/cogs/sync/test_cog.py @@ -247,14 +247,12 @@ class SyncCogListenerTests(SyncCogTestCase):          before_data = {              "name": "old name",              "discriminator": "1234", -            "avatar": "old avatar",              "bot": False,          }          subtests = (              (True, "name", "name", "new name", "new name"),              (True, "discriminator", "discriminator", "8765", 8765), -            (True, "avatar", "avatar_hash", "9j2e9", "9j2e9"),              (False, "bot", "bot", True, True),          ) @@ -295,7 +293,6 @@ class SyncCogListenerTests(SyncCogTestCase):          )          data = { -            "avatar_hash": member.avatar,              "discriminator": int(member.discriminator),              "id": member.id,              "in_guild": True, diff --git a/tests/bot/cogs/sync/test_users.py b/tests/bot/cogs/sync/test_users.py index 818883012..002a947ad 100644 --- a/tests/bot/cogs/sync/test_users.py +++ b/tests/bot/cogs/sync/test_users.py @@ -10,7 +10,6 @@ def fake_user(**kwargs):      kwargs.setdefault("id", 43)      kwargs.setdefault("name", "bob the test man")      kwargs.setdefault("discriminator", 1337) -    kwargs.setdefault("avatar_hash", None)      kwargs.setdefault("roles", (666,))      kwargs.setdefault("in_guild", True) @@ -32,7 +31,6 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase):          for member in members:              member = member.copy() -            member["avatar"] = member.pop("avatar_hash")              del member["in_guild"]              mock_member = helpers.MockMember(**member) diff --git a/tests/bot/cogs/test_antimalware.py b/tests/bot/cogs/test_antimalware.py new file mode 100644 index 000000000..f219fc1ba --- /dev/null +++ b/tests/bot/cogs/test_antimalware.py @@ -0,0 +1,159 @@ +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from discord import NotFound + +from bot.cogs import antimalware +from bot.constants import AntiMalware as AntiMalwareConfig, Channels, STAFF_ROLES +from tests.helpers import MockAttachment, MockBot, MockMessage, MockRole + +MODULE = "bot.cogs.antimalware" + + +@patch(f"{MODULE}.AntiMalwareConfig.whitelist", new=[".first", ".second", ".third"]) +class AntiMalwareCogTests(unittest.IsolatedAsyncioTestCase): +    """Test the AntiMalware cog.""" + +    def setUp(self): +        """Sets up fresh objects for each test.""" +        self.bot = MockBot() +        self.cog = antimalware.AntiMalware(self.bot) +        self.message = MockMessage() + +    async def test_message_with_allowed_attachment(self): +        """Messages with allowed extensions should not be deleted""" +        attachment = MockAttachment(filename=f"python{AntiMalwareConfig.whitelist[0]}") +        self.message.attachments = [attachment] + +        await self.cog.on_message(self.message) +        self.message.delete.assert_not_called() + +    async def test_message_without_attachment(self): +        """Messages without attachments should result in no action.""" +        await self.cog.on_message(self.message) +        self.message.delete.assert_not_called() + +    async def test_direct_message_with_attachment(self): +        """Direct messages should have no action taken.""" +        attachment = MockAttachment(filename="python.disallowed") +        self.message.attachments = [attachment] +        self.message.guild = None + +        await self.cog.on_message(self.message) + +        self.message.delete.assert_not_called() + +    async def test_message_with_illegal_extension_gets_deleted(self): +        """A message containing an illegal extension should send an embed.""" +        attachment = MockAttachment(filename="python.disallowed") +        self.message.attachments = [attachment] + +        await self.cog.on_message(self.message) + +        self.message.delete.assert_called_once() + +    async def test_message_send_by_staff(self): +        """A message send by a member of staff should be ignored.""" +        staff_role = MockRole(id=STAFF_ROLES[0]) +        self.message.author.roles.append(staff_role) +        attachment = MockAttachment(filename="python.disallowed") +        self.message.attachments = [attachment] + +        await self.cog.on_message(self.message) + +        self.message.delete.assert_not_called() + +    async def test_python_file_redirect_embed_description(self): +        """A message containing a .py file should result in an embed redirecting the user to our paste site""" +        attachment = MockAttachment(filename="python.py") +        self.message.attachments = [attachment] +        self.message.channel.send = AsyncMock() + +        await self.cog.on_message(self.message) +        self.message.channel.send.assert_called_once() +        args, kwargs = self.message.channel.send.call_args +        embed = kwargs.pop("embed") + +        self.assertEqual(embed.description, antimalware.PY_EMBED_DESCRIPTION) + +    async def test_txt_file_redirect_embed_description(self): +        """A message containing a .txt file should result in the correct embed.""" +        attachment = MockAttachment(filename="python.txt") +        self.message.attachments = [attachment] +        self.message.channel.send = AsyncMock() +        antimalware.TXT_EMBED_DESCRIPTION = Mock() +        antimalware.TXT_EMBED_DESCRIPTION.format.return_value = "test" + +        await self.cog.on_message(self.message) +        self.message.channel.send.assert_called_once() +        args, kwargs = self.message.channel.send.call_args +        embed = kwargs.pop("embed") +        cmd_channel = self.bot.get_channel(Channels.bot_commands) + +        self.assertEqual(embed.description, antimalware.TXT_EMBED_DESCRIPTION.format.return_value) +        antimalware.TXT_EMBED_DESCRIPTION.format.assert_called_with(cmd_channel_mention=cmd_channel.mention) + +    async def test_other_disallowed_extention_embed_description(self): +        """Test the description for a non .py/.txt disallowed extension.""" +        attachment = MockAttachment(filename="python.disallowed") +        self.message.attachments = [attachment] +        self.message.channel.send = AsyncMock() +        antimalware.DISALLOWED_EMBED_DESCRIPTION = Mock() +        antimalware.DISALLOWED_EMBED_DESCRIPTION.format.return_value = "test" + +        await self.cog.on_message(self.message) +        self.message.channel.send.assert_called_once() +        args, kwargs = self.message.channel.send.call_args +        embed = kwargs.pop("embed") +        meta_channel = self.bot.get_channel(Channels.meta) + +        self.assertEqual(embed.description, antimalware.DISALLOWED_EMBED_DESCRIPTION.format.return_value) +        antimalware.DISALLOWED_EMBED_DESCRIPTION.format.assert_called_with( +            blocked_extensions_str=".disallowed", +            meta_channel_mention=meta_channel.mention +        ) + +    async def test_removing_deleted_message_logs(self): +        """Removing an already deleted message logs the correct message""" +        attachment = MockAttachment(filename="python.disallowed") +        self.message.attachments = [attachment] +        self.message.delete = AsyncMock(side_effect=NotFound(response=Mock(status=""), message="")) + +        with self.assertLogs(logger=antimalware.log, level="INFO"): +            await self.cog.on_message(self.message) +        self.message.delete.assert_called_once() + +    async def test_message_with_illegal_attachment_logs(self): +        """Deleting a message with an illegal attachment should result in a log.""" +        attachment = MockAttachment(filename="python.disallowed") +        self.message.attachments = [attachment] + +        with self.assertLogs(logger=antimalware.log, level="INFO"): +            await self.cog.on_message(self.message) + +    async def test_get_disallowed_extensions(self): +        """The return value should include all non-whitelisted extensions.""" +        test_values = ( +            ([], []), +            (AntiMalwareConfig.whitelist, []), +            ([".first"], []), +            ([".first", ".disallowed"], [".disallowed"]), +            ([".disallowed"], [".disallowed"]), +            ([".disallowed", ".illegal"], [".disallowed", ".illegal"]), +        ) + +        for extensions, expected_disallowed_extensions in test_values: +            with self.subTest(extensions=extensions, expected_disallowed_extensions=expected_disallowed_extensions): +                self.message.attachments = [MockAttachment(filename=f"filename{extension}") for extension in extensions] +                disallowed_extensions = self.cog.get_disallowed_extensions(self.message) +                self.assertCountEqual(disallowed_extensions, expected_disallowed_extensions) + + +class AntiMalwareSetupTests(unittest.TestCase): +    """Tests setup of the `AntiMalware` cog.""" + +    def test_setup(self): +        """Setup of the extension should call add_cog.""" +        bot = MockBot() +        antimalware.setup(bot) +        bot.add_cog.assert_called_once() diff --git a/tests/bot/cogs/test_duck_pond.py b/tests/bot/cogs/test_duck_pond.py index 7e6bfc748..a8c0107c6 100644 --- a/tests/bot/cogs/test_duck_pond.py +++ b/tests/bot/cogs/test_duck_pond.py @@ -45,7 +45,7 @@ class DuckPondTests(base.LoggingTestsMixin, unittest.IsolatedAsyncioTestCase):          self.assertEqual(cog.bot, bot)          self.assertEqual(cog.webhook_id, constants.Webhooks.duck_pond) -        bot.loop.create_loop.called_once_with(cog.fetch_webhook()) +        bot.loop.create_task.assert_called_once_with(cog.fetch_webhook())      def test_fetch_webhook_succeeds_without_connectivity_issues(self):          """The `fetch_webhook` method waits until `READY` event and sets the `webhook` attribute.""" diff --git a/tests/bot/cogs/test_information.py b/tests/bot/cogs/test_information.py index b5f928dd6..79c0e0ad3 100644 --- a/tests/bot/cogs/test_information.py +++ b/tests/bot/cogs/test_information.py @@ -7,10 +7,9 @@ import discord  from bot import constants  from bot.cogs import information -from bot.decorators import InWhitelistCheckFailure +from bot.utils.checks import InWhitelistCheckFailure  from tests import helpers -  COG_PATH = "bot.cogs.information.Information" @@ -149,14 +148,18 @@ class InformationCogTests(unittest.TestCase):                  Voice region: {self.ctx.guild.region}                  Features: {', '.join(self.ctx.guild.features)} -                **Counts** -                Members: {self.ctx.guild.member_count:,} -                Roles: {len(self.ctx.guild.roles)} +                **Channel counts**                  Category channels: 1                  Text channels: 1                  Voice channels: 1 +                Staff channels: 0 + +                **Member counts** +                Members: {self.ctx.guild.member_count:,} +                Staff members: 0 +                Roles: {len(self.ctx.guild.roles)} -                **Members** +                **Member statuses**                  {constants.Emojis.status_online} 2                  {constants.Emojis.status_idle} 1                  {constants.Emojis.status_dnd} 4 diff --git a/tests/bot/cogs/test_snekbox.py b/tests/bot/cogs/test_snekbox.py index 8490b02ca..cf9adbee0 100644 --- a/tests/bot/cogs/test_snekbox.py +++ b/tests/bot/cogs/test_snekbox.py @@ -21,7 +21,10 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase):          """Post the eval code to the URLs.snekbox_eval_api endpoint."""          resp = MagicMock()          resp.json = AsyncMock(return_value="return") -        self.bot.http_session.post().__aenter__.return_value = resp + +        context_manager = MagicMock() +        context_manager.__aenter__.return_value = resp +        self.bot.http_session.post.return_value = context_manager          self.assertEqual(await self.cog.post_eval("import random"), "return")          self.bot.http_session.post.assert_called_with( @@ -41,7 +44,10 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase):          key = "MarkDiamond"          resp = MagicMock()          resp.json = AsyncMock(return_value={"key": key}) -        self.bot.http_session.post().__aenter__.return_value = resp + +        context_manager = MagicMock() +        context_manager.__aenter__.return_value = resp +        self.bot.http_session.post.return_value = context_manager          self.assertEqual(              await self.cog.upload_output("My awesome output"), @@ -57,7 +63,10 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase):          """Output upload gracefully fallback if the upload fail."""          resp = MagicMock()          resp.json = AsyncMock(side_effect=Exception) -        self.bot.http_session.post().__aenter__.return_value = resp + +        context_manager = MagicMock() +        context_manager.__aenter__.return_value = resp +        self.bot.http_session.post.return_value = context_manager          log = logging.getLogger("bot.cogs.snekbox")          with self.assertLogs(logger=log, level='ERROR'): @@ -290,7 +299,11 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase):          self.assertEqual(actual, expected)          self.bot.wait_for.assert_has_awaits(              ( -                call('message_edit', check=partial_mock(snekbox.predicate_eval_message_edit, ctx), timeout=10), +                call( +                    'message_edit', +                    check=partial_mock(snekbox.predicate_eval_message_edit, ctx), +                    timeout=snekbox.REEVAL_TIMEOUT, +                ),                  call('reaction_add', check=partial_mock(snekbox.predicate_eval_emoji_reaction, ctx), timeout=10)              )          ) diff --git a/tests/bot/cogs/test_token_remover.py b/tests/bot/cogs/test_token_remover.py index 33d1ec170..3349caa73 100644 --- a/tests/bot/cogs/test_token_remover.py +++ b/tests/bot/cogs/test_token_remover.py @@ -1,56 +1,89 @@ -import asyncio -import logging  import unittest -from unittest.mock import AsyncMock, MagicMock +from re import Match +from unittest import mock +from unittest.mock import MagicMock -from discord import Colour +from discord import Colour, NotFound -from bot.cogs.token_remover import ( -    DELETION_MESSAGE_TEMPLATE, -    TokenRemover, -    setup as setup_cog, -) -from bot.constants import Channels, Colours, Event, Icons -from tests.helpers import MockBot, MockMessage +from bot import constants +from bot.cogs import token_remover +from bot.cogs.moderation import ModLog +from bot.cogs.token_remover import Token, TokenRemover +from tests.helpers import MockBot, MockMessage, autospec -class TokenRemoverTests(unittest.TestCase): +class TokenRemoverTests(unittest.IsolatedAsyncioTestCase):      """Tests the `TokenRemover` cog."""      def setUp(self):          """Adds the cog, a bot, and a message to the instance for usage in tests."""          self.bot = MockBot() -        self.bot.get_cog.return_value = MagicMock() -        self.bot.get_cog.return_value.send_log_message = AsyncMock()          self.cog = TokenRemover(bot=self.bot) -        self.msg = MockMessage(id=555, content='') -        self.msg.author.__str__ = MagicMock() -        self.msg.author.__str__.return_value = 'lemon' -        self.msg.author.bot = False -        self.msg.author.avatar_url_as.return_value = 'picture-lemon.png' -        self.msg.author.id = 42 -        self.msg.author.mention = '@lemon' +        self.msg = MockMessage(id=555, content="hello world")          self.msg.channel.mention = "#lemonade-stand" +        self.msg.author.__str__ = MagicMock(return_value=self.msg.author.name) +        self.msg.author.avatar_url_as.return_value = "picture-lemon.png" -    def test_is_valid_user_id_is_true_for_numeric_content(self): -        """A string decoding to numeric characters is a valid user ID.""" -        # MTIz = base64(123) -        self.assertTrue(TokenRemover.is_valid_user_id('MTIz')) +    def test_is_valid_user_id_valid(self): +        """Should consider user IDs valid if they decode entirely to ASCII digits.""" +        ids = ( +            "NDcyMjY1OTQzMDYyNDEzMzMy", +            "NDc1MDczNjI5Mzk5NTQ3OTA0", +            "NDY3MjIzMjMwNjUwNzc3NjQx", +        ) + +        for user_id in ids: +            with self.subTest(user_id=user_id): +                result = TokenRemover.is_valid_user_id(user_id) +                self.assertTrue(result) -    def test_is_valid_user_id_is_false_for_alphabetic_content(self): -        """A string decoding to alphabetic characters is not a valid user ID.""" -        # YWJj = base64(abc) -        self.assertFalse(TokenRemover.is_valid_user_id('YWJj')) +    def test_is_valid_user_id_invalid(self): +        """Should consider non-digit and non-ASCII IDs invalid.""" +        ids = ( +            ("SGVsbG8gd29ybGQ", "non-digit ASCII"), +            ("0J_RgNC40LLQtdGCINC80LjRgA", "cyrillic text"), +            ("4pO14p6L4p6C4pG34p264pGl8J-EiOKSj-KCieKBsA", "Unicode digits"), +            ("4oaA4oaB4oWh4oWi4Lyz4Lyq4Lyr4LG9", "Unicode numerals"), +            ("8J2fjvCdn5nwnZ-k8J2fr_Cdn7rgravvvJngr6c", "Unicode decimals"), +            ("{hello}[world]&(bye!)", "ASCII invalid Base64"), +            ("Þíß-ï§-ňøẗ-våłìÐ", "Unicode invalid Base64"), +        ) -    def test_is_valid_timestamp_is_true_for_valid_timestamps(self): -        """A string decoding to a valid timestamp should be recognized as such.""" -        self.assertTrue(TokenRemover.is_valid_timestamp('DN9r_A')) +        for user_id, msg in ids: +            with self.subTest(msg=msg): +                result = TokenRemover.is_valid_user_id(user_id) +                self.assertFalse(result) -    def test_is_valid_timestamp_is_false_for_invalid_values(self): -        """A string not decoding to a valid timestamp should not be recognized as such.""" -        # MTIz = base64(123) -        self.assertFalse(TokenRemover.is_valid_timestamp('MTIz')) +    def test_is_valid_timestamp_valid(self): +        """Should consider timestamps valid if they're greater than the Discord epoch.""" +        timestamps = ( +            "XsyRkw", +            "Xrim9Q", +            "XsyR-w", +            "XsySD_", +            "Dn9r_A", +        ) + +        for timestamp in timestamps: +            with self.subTest(timestamp=timestamp): +                result = TokenRemover.is_valid_timestamp(timestamp) +                self.assertTrue(result) + +    def test_is_valid_timestamp_invalid(self): +        """Should consider timestamps invalid if they're before Discord epoch or can't be parsed.""" +        timestamps = ( +            ("B4Yffw", "DISCORD_EPOCH - TOKEN_EPOCH - 1"), +            ("ew", "123"), +            ("AoIKgA", "42076800"), +            ("{hello}[world]&(bye!)", "ASCII invalid Base64"), +            ("Þíß-ï§-ňøẗ-våłìÐ", "Unicode invalid Base64"), +        ) + +        for timestamp, msg in timestamps: +            with self.subTest(msg=msg): +                result = TokenRemover.is_valid_timestamp(timestamp) +                self.assertFalse(result)      def test_mod_log_property(self):          """The `mod_log` property should ask the bot to return the `ModLog` cog.""" @@ -58,74 +91,220 @@ class TokenRemoverTests(unittest.TestCase):          self.assertEqual(self.cog.mod_log, self.bot.get_cog.return_value)          self.bot.get_cog.assert_called_once_with('ModLog') -    def test_ignores_bot_messages(self): -        """When the message event handler is called with a bot message, nothing is done.""" -        self.msg.author.bot = True -        coroutine = self.cog.on_message(self.msg) -        self.assertIsNone(asyncio.run(coroutine)) - -    def test_ignores_messages_without_tokens(self): -        """Messages without anything looking like a token are ignored.""" -        for content in ('', 'lemon wins'): -            with self.subTest(content=content): -                self.msg.content = content -                coroutine = self.cog.on_message(self.msg) -                self.assertIsNone(asyncio.run(coroutine)) - -    def test_ignores_messages_with_invalid_tokens(self): -        """Messages with values that are invalid tokens are ignored.""" -        for content in ('foo.bar.baz', 'x.y.'): -            with self.subTest(content=content): -                self.msg.content = content -                coroutine = self.cog.on_message(self.msg) -                self.assertIsNone(asyncio.run(coroutine)) - -    def test_censors_valid_tokens(self): -        """Valid tokens are censored.""" -        cases = ( -            # (content, censored_token) -            ('MTIz.DN9R_A.xyz', 'MTIz.DN9R_A.xxx'), +    async def test_on_message_edit_uses_on_message(self): +        """The edit listener should delegate handling of the message to the normal listener.""" +        self.cog.on_message = mock.create_autospec(self.cog.on_message, spec_set=True) + +        await self.cog.on_message_edit(MockMessage(), self.msg) +        self.cog.on_message.assert_awaited_once_with(self.msg) + +    @autospec(TokenRemover, "find_token_in_message", "take_action") +    async def test_on_message_takes_action(self, find_token_in_message, take_action): +        """Should take action if a valid token is found when a message is sent.""" +        cog = TokenRemover(self.bot) +        found_token = "foobar" +        find_token_in_message.return_value = found_token + +        await cog.on_message(self.msg) + +        find_token_in_message.assert_called_once_with(self.msg) +        take_action.assert_awaited_once_with(cog, self.msg, found_token) + +    @autospec(TokenRemover, "find_token_in_message", "take_action") +    async def test_on_message_skips_missing_token(self, find_token_in_message, take_action): +        """Shouldn't take action if a valid token isn't found when a message is sent.""" +        cog = TokenRemover(self.bot) +        find_token_in_message.return_value = False + +        await cog.on_message(self.msg) + +        find_token_in_message.assert_called_once_with(self.msg) +        take_action.assert_not_awaited() + +    @autospec(TokenRemover, "find_token_in_message") +    async def test_on_message_ignores_dms_bots(self, find_token_in_message): +        """Shouldn't parse a message if it is a DM or authored by a bot.""" +        cog = TokenRemover(self.bot) +        dm_msg = MockMessage(guild=None) +        bot_msg = MockMessage(author=MagicMock(bot=True)) + +        for msg in (dm_msg, bot_msg): +            await cog.on_message(msg) +            find_token_in_message.assert_not_called() + +    @autospec("bot.cogs.token_remover", "TOKEN_RE") +    def test_find_token_no_matches(self, token_re): +        """None should be returned if the regex matches no tokens in a message.""" +        token_re.finditer.return_value = () + +        return_value = TokenRemover.find_token_in_message(self.msg) + +        self.assertIsNone(return_value) +        token_re.finditer.assert_called_once_with(self.msg.content) + +    @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") +    @autospec("bot.cogs.token_remover", "Token") +    @autospec("bot.cogs.token_remover", "TOKEN_RE") +    def test_find_token_valid_match(self, token_re, token_cls, is_valid_id, is_valid_timestamp): +        """The first match with a valid user ID and timestamp should be returned as a `Token`.""" +        matches = [ +            mock.create_autospec(Match, spec_set=True, instance=True), +            mock.create_autospec(Match, spec_set=True, instance=True), +        ] +        tokens = [ +            mock.create_autospec(Token, spec_set=True, instance=True), +            mock.create_autospec(Token, spec_set=True, instance=True), +        ] + +        token_re.finditer.return_value = matches +        token_cls.side_effect = tokens +        is_valid_id.side_effect = (False, True)  # The 1st match will be invalid, 2nd one valid. +        is_valid_timestamp.return_value = True + +        return_value = TokenRemover.find_token_in_message(self.msg) + +        self.assertEqual(tokens[1], return_value) +        token_re.finditer.assert_called_once_with(self.msg.content) + +    @autospec(TokenRemover, "is_valid_user_id", "is_valid_timestamp") +    @autospec("bot.cogs.token_remover", "Token") +    @autospec("bot.cogs.token_remover", "TOKEN_RE") +    def test_find_token_invalid_matches(self, token_re, token_cls, is_valid_id, is_valid_timestamp): +        """None should be returned if no matches have valid user IDs or timestamps.""" +        token_re.finditer.return_value = [mock.create_autospec(Match, spec_set=True, instance=True)] +        token_cls.return_value = mock.create_autospec(Token, spec_set=True, instance=True) +        is_valid_id.return_value = False +        is_valid_timestamp.return_value = False + +        return_value = TokenRemover.find_token_in_message(self.msg) + +        self.assertIsNone(return_value) +        token_re.finditer.assert_called_once_with(self.msg.content) + +    def test_regex_invalid_tokens(self): +        """Messages without anything looking like a token are not matched.""" +        tokens = ( +            "", +            "lemon wins", +            "..", +            "x.y", +            "x.y.", +            ".y.z", +            ".y.", +            "..z", +            "x..z", +            " . . ", +            "\n.\n.\n", +            "hellö.world.bye", +            "base64.nötbåse64.morebase64", +            "19jd3J.dfkm3d.€víł§tüff", +        ) + +        for token in tokens: +            with self.subTest(token=token): +                results = token_remover.TOKEN_RE.findall(token) +                self.assertEqual(len(results), 0) + +    def test_regex_valid_tokens(self): +        """Messages that look like tokens should be matched.""" +        # Don't worry, these tokens have been invalidated. +        tokens = ( +            "NDcyMjY1OTQzMDYy_DEzMz-y.XsyRkw.VXmErH7j511turNpfURmb0rVNm8", +            "NDcyMjY1OTQzMDYyNDEzMzMy.Xrim9Q.Ysnu2wacjaKs7qnoo46S8Dm2us8", +            "NDc1MDczNjI5Mzk5NTQ3OTA0.XsyR-w.sJf6omBPORBPju3WJEIAcwW9Zds", +            "NDY3MjIzMjMwNjUwNzc3NjQx.XsySD_.s45jqDV_Iisn-symw0yDRrk_jf4", +        ) + +        for token in tokens: +            with self.subTest(token=token): +                results = token_remover.TOKEN_RE.fullmatch(token) +                self.assertIsNotNone(results, f"{token} was not matched by the regex") + +    def test_regex_matches_multiple_valid(self): +        """Should support multiple matches in the middle of a string.""" +        token_1 = "NDY3MjIzMjMwNjUwNzc3NjQx.XsyWGg.uFNEQPCc4ePwGh7egG8UicQssz8" +        token_2 = "NDcyMjY1OTQzMDYyNDEzMzMy.XsyWMw.l8XPnDqb0lp-EiQ2g_0xVFT1pyc" +        message = f"garbage {token_1} hello {token_2} world" + +        results = token_remover.TOKEN_RE.finditer(message) +        results = [match[0] for match in results] +        self.assertCountEqual((token_1, token_2), results) + +    @autospec("bot.cogs.token_remover", "LOG_MESSAGE") +    def test_format_log_message(self, log_message): +        """Should correctly format the log message with info from the message and token.""" +        token = Token("NDY3MjIzMjMwNjUwNzc3NjQx", "XsySD_", "s45jqDV_Iisn-symw0yDRrk_jf4") +        log_message.format.return_value = "Howdy" + +        return_value = TokenRemover.format_log_message(self.msg, token) + +        self.assertEqual(return_value, log_message.format.return_value) +        log_message.format.assert_called_once_with( +            author=self.msg.author, +            author_id=self.msg.author.id, +            channel=self.msg.channel.mention, +            user_id=token.user_id, +            timestamp=token.timestamp, +            hmac="x" * len(token.hmac), +        ) + +    @mock.patch.object(TokenRemover, "mod_log", new_callable=mock.PropertyMock) +    @autospec("bot.cogs.token_remover", "log") +    @autospec(TokenRemover, "format_log_message") +    async def test_take_action(self, format_log_message, logger, mod_log_property): +        """Should delete the message and send a mod log.""" +        cog = TokenRemover(self.bot) +        mod_log = mock.create_autospec(ModLog, spec_set=True, instance=True) +        token = mock.create_autospec(Token, spec_set=True, instance=True) +        log_msg = "testing123" + +        mod_log_property.return_value = mod_log +        format_log_message.return_value = log_msg + +        await cog.take_action(self.msg, token) + +        self.msg.delete.assert_called_once_with() +        self.msg.channel.send.assert_called_once_with( +            token_remover.DELETION_MESSAGE_TEMPLATE.format(mention=self.msg.author.mention) +        ) + +        format_log_message.assert_called_once_with(self.msg, token) +        logger.debug.assert_called_with(log_msg) +        self.bot.stats.incr.assert_called_once_with("tokens.removed_tokens") + +        mod_log.ignore.assert_called_once_with(constants.Event.message_delete, self.msg.id) +        mod_log.send_log_message.assert_called_once_with( +            icon_url=constants.Icons.token_removed, +            colour=Colour(constants.Colours.soft_red), +            title="Token removed!", +            text=log_msg, +            thumbnail=self.msg.author.avatar_url_as.return_value, +            channel_id=constants.Channels.mod_alerts          ) -        for content, censored_token in cases: -            with self.subTest(content=content, censored_token=censored_token): -                self.msg.content = content -                coroutine = self.cog.on_message(self.msg) -                with self.assertLogs(logger='bot.cogs.token_remover', level=logging.DEBUG) as cm: -                    self.assertIsNone(asyncio.run(coroutine))  # no return value - -                [line] = cm.output -                log_message = ( -                    "Censored a seemingly valid token sent by " -                    "lemon (`42`) in #lemonade-stand, " -                    f"token was `{censored_token}`" -                ) -                self.assertIn(log_message, line) - -                self.msg.delete.assert_called_once_with() -                self.msg.channel.send.assert_called_once_with( -                    DELETION_MESSAGE_TEMPLATE.format(mention='@lemon') -                ) -                self.bot.get_cog.assert_called_with('ModLog') -                self.msg.author.avatar_url_as.assert_called_once_with(static_format='png') - -                mod_log = self.bot.get_cog.return_value -                mod_log.ignore.assert_called_once_with(Event.message_delete, self.msg.id) -                mod_log.send_log_message.assert_called_once_with( -                    icon_url=Icons.token_removed, -                    colour=Colour(Colours.soft_red), -                    title="Token removed!", -                    text=log_message, -                    thumbnail='picture-lemon.png', -                    channel_id=Channels.mod_alerts -                ) - - -class TokenRemoverSetupTests(unittest.TestCase): -    """Tests setup of the `TokenRemover` cog.""" - -    def test_setup(self): -        """Setup of the extension should call add_cog.""" +    @mock.patch.object(TokenRemover, "mod_log", new_callable=mock.PropertyMock) +    async def test_take_action_delete_failure(self, mod_log_property): +        """Shouldn't send any messages if the token message can't be deleted.""" +        cog = TokenRemover(self.bot) +        mod_log_property.return_value = mock.create_autospec(ModLog, spec_set=True, instance=True) +        self.msg.delete.side_effect = NotFound(MagicMock(), MagicMock()) + +        token = mock.create_autospec(Token, spec_set=True, instance=True) +        await cog.take_action(self.msg, token) + +        self.msg.delete.assert_called_once_with() +        self.msg.channel.send.assert_not_awaited() + + +class TokenRemoverExtensionTests(unittest.TestCase): +    """Tests for the token_remover extension.""" + +    @autospec("bot.cogs.token_remover", "TokenRemover") +    def test_extension_setup(self, cog): +        """The TokenRemover cog should be added."""          bot = MockBot() -        setup_cog(bot) +        token_remover.setup(bot) + +        cog.assert_called_once_with(bot)          bot.add_cog.assert_called_once() +        self.assertTrue(isinstance(bot.add_cog.call_args.args[0], TokenRemover)) diff --git a/tests/bot/test_constants.py b/tests/bot/test_constants.py index dae7c066c..f10d6fbe8 100644 --- a/tests/bot/test_constants.py +++ b/tests/bot/test_constants.py @@ -1,14 +1,40 @@  import inspect +import typing  import unittest  from bot import constants +def is_annotation_instance(value: typing.Any, annotation: typing.Any) -> bool: +    """ +    Return True if `value` is an instance of the type represented by `annotation`. + +    This doesn't account for things like Unions or checking for homogenous types in collections. +    """ +    origin = typing.get_origin(annotation) + +    # This is done in case a bare e.g. `typing.List` is used. +    # In such case, for the assertion to pass, the type needs to be normalised to e.g. `list`. +    # `get_origin()` does this normalisation for us. +    type_ = annotation if origin is None else origin + +    return isinstance(value, type_) + + +def is_any_instance(value: typing.Any, types: typing.Collection) -> bool: +    """Return True if `value` is an instance of any type in `types`.""" +    for type_ in types: +        if is_annotation_instance(value, type_): +            return True + +    return False + +  class ConstantsTests(unittest.TestCase):      """Tests for our constants."""      def test_section_configuration_matches_type_specification(self): -        """The section annotations should match the actual types of the sections.""" +        """"The section annotations should match the actual types of the sections."""          sections = (              cls @@ -17,10 +43,15 @@ class ConstantsTests(unittest.TestCase):          )          for section in sections:              for name, annotation in section.__annotations__.items(): -                with self.subTest(section=section, name=name, annotation=annotation): +                with self.subTest(section=section.__name__, name=name, annotation=annotation):                      value = getattr(section, name) +                    origin = typing.get_origin(annotation) +                    annotation_args = typing.get_args(annotation) +                    failure_msg = f"{value} is not an instance of {annotation}" -                    if getattr(annotation, '_name', None) in ('Dict', 'List'): -                        self.skipTest("Cannot validate containers yet.") - -                    self.assertIsInstance(value, annotation) +                    if origin is typing.Union: +                        is_instance = is_any_instance(value, annotation_args) +                        self.assertTrue(is_instance, failure_msg) +                    else: +                        is_instance = is_annotation_instance(value, annotation) +                        self.assertTrue(is_instance, failure_msg) diff --git a/tests/bot/test_converters.py b/tests/bot/test_converters.py index ca8cb6825..c42111f3f 100644 --- a/tests/bot/test_converters.py +++ b/tests/bot/test_converters.py @@ -1,5 +1,5 @@ -import asyncio  import datetime +import re  import unittest  from unittest.mock import MagicMock, patch @@ -16,7 +16,7 @@ from bot.converters import (  ) -class ConverterTests(unittest.TestCase): +class ConverterTests(unittest.IsolatedAsyncioTestCase):      """Tests our custom argument converters."""      @classmethod @@ -26,7 +26,7 @@ class ConverterTests(unittest.TestCase):          cls.fixed_utc_now = datetime.datetime.fromisoformat('2019-01-01T00:00:00') -    def test_tag_content_converter_for_valid(self): +    async def test_tag_content_converter_for_valid(self):          """TagContentConverter should return correct values for valid input."""          test_values = (              ('hello', 'hello'), @@ -35,10 +35,10 @@ class ConverterTests(unittest.TestCase):          for content, expected_conversion in test_values:              with self.subTest(content=content, expected_conversion=expected_conversion): -                conversion = asyncio.run(TagContentConverter.convert(self.context, content)) +                conversion = await TagContentConverter.convert(self.context, content)                  self.assertEqual(conversion, expected_conversion) -    def test_tag_content_converter_for_invalid(self): +    async def test_tag_content_converter_for_invalid(self):          """TagContentConverter should raise the proper exception for invalid input."""          test_values = (              ('', "Tag contents should not be empty, or filled with whitespace."), @@ -47,10 +47,10 @@ class ConverterTests(unittest.TestCase):          for value, exception_message in test_values:              with self.subTest(tag_content=value, exception_message=exception_message): -                with self.assertRaises(BadArgument, msg=exception_message): -                    asyncio.run(TagContentConverter.convert(self.context, value)) +                with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): +                    await TagContentConverter.convert(self.context, value) -    def test_tag_name_converter_for_valid(self): +    async def test_tag_name_converter_for_valid(self):          """TagNameConverter should return the correct values for valid tag names."""          test_values = (              ('tracebacks', 'tracebacks'), @@ -60,10 +60,10 @@ class ConverterTests(unittest.TestCase):          for name, expected_conversion in test_values:              with self.subTest(name=name, expected_conversion=expected_conversion): -                conversion = asyncio.run(TagNameConverter.convert(self.context, name)) +                conversion = await TagNameConverter.convert(self.context, name)                  self.assertEqual(conversion, expected_conversion) -    def test_tag_name_converter_for_invalid(self): +    async def test_tag_name_converter_for_invalid(self):          """TagNameConverter should raise the correct exception for invalid tag names."""          test_values = (              ('👋', "Don't be ridiculous, you can't use that character!"), @@ -75,29 +75,29 @@ class ConverterTests(unittest.TestCase):          for invalid_name, exception_message in test_values:              with self.subTest(invalid_name=invalid_name, exception_message=exception_message): -                with self.assertRaises(BadArgument, msg=exception_message): -                    asyncio.run(TagNameConverter.convert(self.context, invalid_name)) +                with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): +                    await TagNameConverter.convert(self.context, invalid_name) -    def test_valid_python_identifier_for_valid(self): +    async def test_valid_python_identifier_for_valid(self):          """ValidPythonIdentifier returns valid identifiers unchanged."""          test_values = ('foo', 'lemon')          for name in test_values:              with self.subTest(identifier=name): -                conversion = asyncio.run(ValidPythonIdentifier.convert(self.context, name)) +                conversion = await ValidPythonIdentifier.convert(self.context, name)                  self.assertEqual(name, conversion) -    def test_valid_python_identifier_for_invalid(self): +    async def test_valid_python_identifier_for_invalid(self):          """ValidPythonIdentifier raises the proper exception for invalid identifiers."""          test_values = ('nested.stuff', '#####')          for name in test_values:              with self.subTest(identifier=name):                  exception_message = f'`{name}` is not a valid Python identifier' -                with self.assertRaises(BadArgument, msg=exception_message): -                    asyncio.run(ValidPythonIdentifier.convert(self.context, name)) +                with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): +                    await ValidPythonIdentifier.convert(self.context, name) -    def test_duration_converter_for_valid(self): +    async def test_duration_converter_for_valid(self):          """Duration returns the correct `datetime` for valid duration strings."""          test_values = (              # Simple duration strings @@ -159,35 +159,35 @@ class ConverterTests(unittest.TestCase):                  mock_datetime.utcnow.return_value = self.fixed_utc_now                  with self.subTest(duration=duration, duration_dict=duration_dict): -                    converted_datetime = asyncio.run(converter.convert(self.context, duration)) +                    converted_datetime = await converter.convert(self.context, duration)                      self.assertEqual(converted_datetime, expected_datetime) -    def test_duration_converter_for_invalid(self): +    async def test_duration_converter_for_invalid(self):          """Duration raises the right exception for invalid duration strings."""          test_values = (              # Units in wrong order -            ('1d1w'), -            ('1s1y'), +            '1d1w', +            '1s1y',              # Duplicated units -            ('1 year 2 years'), -            ('1 M 10 minutes'), +            '1 year 2 years', +            '1 M 10 minutes',              # Unknown substrings -            ('1MVes'), -            ('1y3breads'), +            '1MVes', +            '1y3breads',              # Missing amount -            ('ym'), +            'ym',              # Incorrect whitespace -            (" 1y"), -            ("1S "), -            ("1y  1m"), +            " 1y", +            "1S ", +            "1y  1m",              # Garbage -            ('Guido van Rossum'), -            ('lemon lemon lemon lemon lemon lemon lemon'), +            'Guido van Rossum', +            'lemon lemon lemon lemon lemon lemon lemon',          )          converter = Duration() @@ -195,10 +195,21 @@ class ConverterTests(unittest.TestCase):          for invalid_duration in test_values:              with self.subTest(invalid_duration=invalid_duration):                  exception_message = f'`{invalid_duration}` is not a valid duration string.' -                with self.assertRaises(BadArgument, msg=exception_message): -                    asyncio.run(converter.convert(self.context, invalid_duration)) +                with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): +                    await converter.convert(self.context, invalid_duration) -    def test_isodatetime_converter_for_valid(self): +    @patch("bot.converters.datetime") +    async def test_duration_converter_out_of_range(self, mock_datetime): +        """Duration converter should raise BadArgument if datetime raises a ValueError.""" +        mock_datetime.__add__.side_effect = ValueError +        mock_datetime.utcnow.return_value = mock_datetime + +        duration = f"{datetime.MAXYEAR}y" +        exception_message = f"`{duration}` results in a datetime outside the supported range." +        with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): +            await Duration().convert(self.context, duration) + +    async def test_isodatetime_converter_for_valid(self):          """ISODateTime converter returns correct datetime for valid datetime string."""          test_values = (              # `YYYY-mm-ddTHH:MM:SSZ` | `YYYY-mm-dd HH:MM:SSZ` @@ -243,37 +254,37 @@ class ConverterTests(unittest.TestCase):          for datetime_string, expected_dt in test_values:              with self.subTest(datetime_string=datetime_string, expected_dt=expected_dt): -                converted_dt = asyncio.run(converter.convert(self.context, datetime_string)) +                converted_dt = await converter.convert(self.context, datetime_string)                  self.assertIsNone(converted_dt.tzinfo)                  self.assertEqual(converted_dt, expected_dt) -    def test_isodatetime_converter_for_invalid(self): +    async def test_isodatetime_converter_for_invalid(self):          """ISODateTime converter raises the correct exception for invalid datetime strings."""          test_values = (              # Make sure it doesn't interfere with the Duration converter -            ('1Y'), -            ('1d'), -            ('1H'), +            '1Y', +            '1d', +            '1H',              # Check if it fails when only providing the optional time part -            ('10:10:10'), -            ('10:00'), +            '10:10:10', +            '10:00',              # Invalid date format -            ('19-01-01'), +            '19-01-01',              # Other non-valid strings -            ('fisk the tag master'), +            'fisk the tag master',          )          converter = ISODateTime()          for datetime_string in test_values:              with self.subTest(datetime_string=datetime_string):                  exception_message = f"`{datetime_string}` is not a valid ISO-8601 datetime string" -                with self.assertRaises(BadArgument, msg=exception_message): -                    asyncio.run(converter.convert(self.context, datetime_string)) +                with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): +                    await converter.convert(self.context, datetime_string) -    def test_hush_duration_converter_for_valid(self): +    async def test_hush_duration_converter_for_valid(self):          """HushDurationConverter returns correct value for minutes duration or `"forever"` strings."""          test_values = (              ("0", 0), @@ -286,10 +297,10 @@ class ConverterTests(unittest.TestCase):          converter = HushDurationConverter()          for minutes_string, expected_minutes in test_values:              with self.subTest(minutes_string=minutes_string, expected_minutes=expected_minutes): -                converted = asyncio.run(converter.convert(self.context, minutes_string)) +                converted = await converter.convert(self.context, minutes_string)                  self.assertEqual(expected_minutes, converted) -    def test_hush_duration_converter_for_invalid(self): +    async def test_hush_duration_converter_for_invalid(self):          """HushDurationConverter raises correct exception for invalid minutes duration strings."""          test_values = (              ("16", "Duration must be at most 15 minutes."), @@ -299,5 +310,5 @@ class ConverterTests(unittest.TestCase):          converter = HushDurationConverter()          for invalid_minutes_string, exception_message in test_values:              with self.subTest(invalid_minutes_string=invalid_minutes_string, exception_message=exception_message): -                with self.assertRaisesRegex(BadArgument, exception_message): -                    asyncio.run(converter.convert(self.context, invalid_minutes_string)) +                with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): +                    await converter.convert(self.context, invalid_minutes_string) diff --git a/tests/bot/test_decorators.py b/tests/bot/test_decorators.py index a17dd3e16..3d450caa0 100644 --- a/tests/bot/test_decorators.py +++ b/tests/bot/test_decorators.py @@ -3,10 +3,10 @@ import unittest  import unittest.mock  from bot import constants -from bot.decorators import InWhitelistCheckFailure, in_whitelist +from bot.decorators import in_whitelist +from bot.utils.checks import InWhitelistCheckFailure  from tests import helpers -  InWhitelistTestCase = collections.namedtuple("WhitelistedContextTestCase", ("kwargs", "ctx", "description")) diff --git a/tests/bot/utils/test_checks.py b/tests/bot/utils/test_checks.py index 9610771e5..de72e5748 100644 --- a/tests/bot/utils/test_checks.py +++ b/tests/bot/utils/test_checks.py @@ -1,6 +1,8 @@  import unittest +from unittest.mock import MagicMock  from bot.utils import checks +from bot.utils.checks import InWhitelistCheckFailure  from tests.helpers import MockContext, MockRole @@ -42,10 +44,48 @@ class ChecksTests(unittest.TestCase):          self.ctx.author.roles.append(MockRole(id=role_id))          self.assertTrue(checks.without_role_check(self.ctx, role_id + 10)) -    def test_in_channel_check_for_correct_channel(self): -        self.ctx.channel.id = 42 -        self.assertTrue(checks.in_channel_check(self.ctx, *[42])) +    def test_in_whitelist_check_correct_channel(self): +        """`in_whitelist_check` returns `True` if `Context.channel.id` is in the channel list.""" +        channel_id = 3 +        self.ctx.channel.id = channel_id +        self.assertTrue(checks.in_whitelist_check(self.ctx, [channel_id])) -    def test_in_channel_check_for_incorrect_channel(self): -        self.ctx.channel.id = 42 + 10 -        self.assertFalse(checks.in_channel_check(self.ctx, *[42])) +    def test_in_whitelist_check_incorrect_channel(self): +        """`in_whitelist_check` raises InWhitelistCheckFailure if there's no channel match.""" +        self.ctx.channel.id = 3 +        with self.assertRaises(InWhitelistCheckFailure): +            checks.in_whitelist_check(self.ctx, [4]) + +    def test_in_whitelist_check_correct_category(self): +        """`in_whitelist_check` returns `True` if `Context.channel.category_id` is in the category list.""" +        category_id = 3 +        self.ctx.channel.category_id = category_id +        self.assertTrue(checks.in_whitelist_check(self.ctx, categories=[category_id])) + +    def test_in_whitelist_check_incorrect_category(self): +        """`in_whitelist_check` raises InWhitelistCheckFailure if there's no category match.""" +        self.ctx.channel.category_id = 3 +        with self.assertRaises(InWhitelistCheckFailure): +            checks.in_whitelist_check(self.ctx, categories=[4]) + +    def test_in_whitelist_check_correct_role(self): +        """`in_whitelist_check` returns `True` if any of the `Context.author.roles` are in the roles list.""" +        self.ctx.author.roles = (MagicMock(id=1), MagicMock(id=2)) +        self.assertTrue(checks.in_whitelist_check(self.ctx, roles=[2, 6])) + +    def test_in_whitelist_check_incorrect_role(self): +        """`in_whitelist_check` raises InWhitelistCheckFailure if there's no role match.""" +        self.ctx.author.roles = (MagicMock(id=1), MagicMock(id=2)) +        with self.assertRaises(InWhitelistCheckFailure): +            checks.in_whitelist_check(self.ctx, roles=[4]) + +    def test_in_whitelist_check_fail_silently(self): +        """`in_whitelist_check` test no exception raised if `fail_silently` is `True`""" +        self.assertFalse(checks.in_whitelist_check(self.ctx, roles=[2, 6], fail_silently=True)) + +    def test_in_whitelist_check_complex(self): +        """`in_whitelist_check` test with multiple parameters""" +        self.ctx.author.roles = (MagicMock(id=1), MagicMock(id=2)) +        self.ctx.channel.category_id = 3 +        self.ctx.channel.id = 5 +        self.assertTrue(checks.in_whitelist_check(self.ctx, channels=[1], categories=[8], roles=[2])) diff --git a/tests/bot/utils/test_messages.py b/tests/bot/utils/test_messages.py new file mode 100644 index 000000000..9c22c9751 --- /dev/null +++ b/tests/bot/utils/test_messages.py @@ -0,0 +1,27 @@ +import unittest + +from bot.utils import messages + + +class TestMessages(unittest.TestCase): +    """Tests for functions in the `bot.utils.messages` module.""" + +    def test_sub_clyde(self): +        """Uppercase E's and lowercase e's are substituted with their cyrillic counterparts.""" +        sub_e = "\u0435" +        sub_E = "\u0415"  # noqa: N806: Uppercase E in variable name + +        test_cases = ( +            (None, None), +            ("", ""), +            ("clyde", f"clyd{sub_e}"), +            ("CLYDE", f"CLYD{sub_E}"), +            ("cLyDe", f"cLyD{sub_e}"), +            ("BIGclyde", f"BIGclyd{sub_e}"), +            ("small clydeus the unholy", f"small clyd{sub_e}us the unholy"), +            ("BIGCLYDE, babyclyde", f"BIGCLYD{sub_E}, babyclyd{sub_e}"), +        ) + +        for username_in, username_out in test_cases: +            with self.subTest(input=username_in, expected_output=username_out): +                self.assertEqual(messages.sub_clyde(username_in), username_out) diff --git a/tests/bot/utils/test_redis_cache.py b/tests/bot/utils/test_redis_cache.py new file mode 100644 index 000000000..a2f0fe55d --- /dev/null +++ b/tests/bot/utils/test_redis_cache.py @@ -0,0 +1,265 @@ +import asyncio +import unittest + +import fakeredis.aioredis + +from bot.utils import RedisCache +from bot.utils.redis_cache import NoBotInstanceError, NoNamespaceError, NoParentInstanceError +from tests import helpers + + +class RedisCacheTests(unittest.IsolatedAsyncioTestCase): +    """Tests the RedisCache class from utils.redis_dict.py.""" + +    async def asyncSetUp(self):  # noqa: N802 +        """Sets up the objects that only have to be initialized once.""" +        self.bot = helpers.MockBot() +        self.bot.redis_session = await fakeredis.aioredis.create_redis_pool() + +        # Okay, so this is necessary so that we can create a clean new +        # class for every test method, and we want that because it will +        # ensure we get a fresh loop, which is necessary for test_increment_lock +        # to be able to pass. +        class DummyCog: +            """A dummy cog, for dummies.""" + +            redis = RedisCache() + +            def __init__(self, bot: helpers.MockBot): +                self.bot = bot + +        self.cog = DummyCog(self.bot) + +        await self.cog.redis.clear() + +    def test_class_attribute_namespace(self): +        """Test that RedisDict creates a namespace automatically for class attributes.""" +        self.assertEqual(self.cog.redis._namespace, "DummyCog.redis") + +    async def test_class_attribute_required(self): +        """Test that errors are raised when not assigned as a class attribute.""" +        bad_cache = RedisCache() +        self.assertIs(bad_cache._namespace, None) + +        with self.assertRaises(RuntimeError): +            await bad_cache.set("test", "me_up_deadman") + +    async def test_set_get_item(self): +        """Test that users can set and get items from the RedisDict.""" +        test_cases = ( +            ('favorite_fruit', 'melon'), +            ('favorite_number', 86), +            ('favorite_fraction', 86.54), +            ('favorite_boolean', False), +            ('other_boolean', True), +        ) + +        # Test that we can get and set different types. +        for test in test_cases: +            await self.cog.redis.set(*test) +            self.assertEqual(await self.cog.redis.get(test[0]), test[1]) + +        # Test that .get allows a default value +        self.assertEqual(await self.cog.redis.get('favorite_nothing', "bearclaw"), "bearclaw") + +    async def test_set_item_type(self): +        """Test that .set rejects keys and values that are not permitted.""" +        fruits = ["lemon", "melon", "apple"] + +        with self.assertRaises(TypeError): +            await self.cog.redis.set(fruits, "nice") + +        with self.assertRaises(TypeError): +            await self.cog.redis.set(4.23, "nice") + +    async def test_delete_item(self): +        """Test that .delete allows us to delete stuff from the RedisCache.""" +        # Add an item and verify that it gets added +        await self.cog.redis.set("internet", "firetruck") +        self.assertEqual(await self.cog.redis.get("internet"), "firetruck") + +        # Delete that item and verify that it gets deleted +        await self.cog.redis.delete("internet") +        self.assertIs(await self.cog.redis.get("internet"), None) + +    async def test_contains(self): +        """Test that we can check membership with .contains.""" +        await self.cog.redis.set('favorite_country', "Burkina Faso") + +        self.assertIs(await self.cog.redis.contains('favorite_country'), True) +        self.assertIs(await self.cog.redis.contains('favorite_dentist'), False) + +    async def test_items(self): +        """Test that the RedisDict can be iterated.""" +        # Set up our test cases in the Redis cache +        test_cases = [ +            ('favorite_turtle', 'Donatello'), +            ('second_favorite_turtle', 'Leonardo'), +            ('third_favorite_turtle', 'Raphael'), +        ] +        for key, value in test_cases: +            await self.cog.redis.set(key, value) + +        # Consume the AsyncIterator into a regular list, easier to compare that way. +        redis_items = [item for item in await self.cog.redis.items()] + +        # These sequences are probably in the same order now, but probably +        # isn't good enough for tests. Let's not rely on .hgetall always +        # returning things in sequence, and just sort both lists to be safe. +        redis_items = sorted(redis_items) +        test_cases = sorted(test_cases) + +        # If these are equal now, everything works fine. +        self.assertSequenceEqual(test_cases, redis_items) + +    async def test_length(self): +        """Test that we can get the correct .length from the RedisDict.""" +        await self.cog.redis.set('one', 1) +        await self.cog.redis.set('two', 2) +        await self.cog.redis.set('three', 3) +        self.assertEqual(await self.cog.redis.length(), 3) + +        await self.cog.redis.set('four', 4) +        self.assertEqual(await self.cog.redis.length(), 4) + +    async def test_to_dict(self): +        """Test that the .to_dict method returns a workable dictionary copy.""" +        copy = await self.cog.redis.to_dict() +        local_copy = {key: value for key, value in await self.cog.redis.items()} +        self.assertIs(type(copy), dict) +        self.assertDictEqual(copy, local_copy) + +    async def test_clear(self): +        """Test that the .clear method removes the entire hash.""" +        await self.cog.redis.set('teddy', 'with me') +        await self.cog.redis.set('in my dreams', 'you have a weird hat') +        self.assertEqual(await self.cog.redis.length(), 2) + +        await self.cog.redis.clear() +        self.assertEqual(await self.cog.redis.length(), 0) + +    async def test_pop(self): +        """Test that we can .pop an item from the RedisDict.""" +        await self.cog.redis.set('john', 'was afraid') + +        self.assertEqual(await self.cog.redis.pop('john'), 'was afraid') +        self.assertEqual(await self.cog.redis.pop('pete', 'breakneck'), 'breakneck') +        self.assertEqual(await self.cog.redis.length(), 0) + +    async def test_update(self): +        """Test that we can .update the RedisDict with multiple items.""" +        await self.cog.redis.set("reckfried", "lona") +        await self.cog.redis.set("bel air", "prince") +        await self.cog.redis.update({ +            "reckfried": "jona", +            "mega": "hungry, though", +        }) + +        result = { +            "reckfried": "jona", +            "bel air": "prince", +            "mega": "hungry, though", +        } +        self.assertDictEqual(await self.cog.redis.to_dict(), result) + +    def test_typestring_conversion(self): +        """Test the typestring-related helper functions.""" +        conversion_tests = ( +            (12, "i|12"), +            (12.4, "f|12.4"), +            ("cowabunga", "s|cowabunga"), +        ) + +        # Test conversion to typestring +        for _input, expected in conversion_tests: +            self.assertEqual(self.cog.redis._value_to_typestring(_input), expected) + +        # Test conversion from typestrings +        for _input, expected in conversion_tests: +            self.assertEqual(self.cog.redis._value_from_typestring(expected), _input) + +        # Test that exceptions are raised on invalid input +        with self.assertRaises(TypeError): +            self.cog.redis._value_to_typestring(["internet"]) +            self.cog.redis._value_from_typestring("o|firedog") + +    async def test_increment_decrement(self): +        """Test .increment and .decrement methods.""" +        await self.cog.redis.set("entropic", 5) +        await self.cog.redis.set("disentropic", 12.5) + +        # Test default increment +        await self.cog.redis.increment("entropic") +        self.assertEqual(await self.cog.redis.get("entropic"), 6) + +        # Test default decrement +        await self.cog.redis.decrement("entropic") +        self.assertEqual(await self.cog.redis.get("entropic"), 5) + +        # Test float increment with float +        await self.cog.redis.increment("disentropic", 2.0) +        self.assertEqual(await self.cog.redis.get("disentropic"), 14.5) + +        # Test float increment with int +        await self.cog.redis.increment("disentropic", 2) +        self.assertEqual(await self.cog.redis.get("disentropic"), 16.5) + +        # Test negative increments, because why not. +        await self.cog.redis.increment("entropic", -5) +        self.assertEqual(await self.cog.redis.get("entropic"), 0) + +        # Negative decrements? Sure. +        await self.cog.redis.decrement("entropic", -5) +        self.assertEqual(await self.cog.redis.get("entropic"), 5) + +        # What about if we use a negative float to decrement an int? +        # This should convert the type into a float. +        await self.cog.redis.decrement("entropic", -2.5) +        self.assertEqual(await self.cog.redis.get("entropic"), 7.5) + +        # Let's test that they raise the right errors +        with self.assertRaises(KeyError): +            await self.cog.redis.increment("doesn't_exist!") + +        await self.cog.redis.set("stringthing", "stringthing") +        with self.assertRaises(TypeError): +            await self.cog.redis.increment("stringthing") + +    async def test_increment_lock(self): +        """Test that we can't produce a race condition in .increment.""" +        await self.cog.redis.set("test_key", 0) +        tasks = [] + +        # Increment this a lot in different tasks +        for _ in range(100): +            task = asyncio.create_task( +                self.cog.redis.increment("test_key", 1) +            ) +            tasks.append(task) +        await asyncio.gather(*tasks) + +        # Confirm that the value has been incremented the exact right number of times. +        value = await self.cog.redis.get("test_key") +        self.assertEqual(value, 100) + +    async def test_exceptions_raised(self): +        """Testing that the various RuntimeErrors are reachable.""" +        class MyCog: +            cache = RedisCache() + +            def __init__(self): +                self.other_cache = RedisCache() + +        cog = MyCog() + +        # Raises "No Bot instance" +        with self.assertRaises(NoBotInstanceError): +            await cog.cache.get("john") + +        # Raises "RedisCache has no namespace" +        with self.assertRaises(NoNamespaceError): +            await cog.other_cache.get("was") + +        # Raises "You must access the RedisCache instance through the cog instance" +        with self.assertRaises(NoParentInstanceError): +            await MyCog.cache.get("afraid") diff --git a/tests/helpers.py b/tests/helpers.py index 2b79a6c2a..facc4e1af 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -4,12 +4,15 @@ import collections  import itertools  import logging  import unittest.mock -from typing import Iterable, Optional +from asyncio import AbstractEventLoop +from typing import Callable, Iterable, Optional  import discord +from aiohttp import ClientSession  from discord.ext.commands import Context  from bot.api import APIClient +from bot.async_stats import AsyncStatsClient  from bot.bot import Bot @@ -23,6 +26,24 @@ for logger in logging.Logger.manager.loggerDict.values():      logger.setLevel(logging.CRITICAL) +def autospec(target, *attributes: str, **kwargs) -> Callable: +    """Patch multiple `attributes` of a `target` with autospecced mocks and `spec_set` as True.""" +    # Caller's kwargs should take priority and overwrite the defaults. +    kwargs = {'spec_set': True, 'autospec': True, **kwargs} + +    # Import the target if it's a string. +    # This is to support both object and string targets like patch.multiple. +    if type(target) is str: +        target = unittest.mock._importer(target) + +    def decorator(func): +        for attribute in attributes: +            patcher = unittest.mock.patch.object(target, attribute, **kwargs) +            func = patcher(func) +        return func +    return decorator + +  class HashableMixin(discord.mixins.EqualityComparable):      """      Mixin that provides similar hashing and equality functionality as discord.py's `Hashable` mixin. @@ -205,6 +226,10 @@ class MockRole(CustomMockMixin, unittest.mock.Mock, ColourMixin, HashableMixin):          """Simplified position-based comparisons similar to those of `discord.Role`."""          return self.position < other.position +    def __ge__(self, other): +        """Simplified position-based comparisons similar to those of `discord.Role`.""" +        return self.position >= other.position +  # Create a Member instance to get a realistic Mock of `discord.Member`  member_data = {'user': 'lemon', 'roles': [1]} @@ -264,10 +289,16 @@ class MockAPIClient(CustomMockMixin, unittest.mock.MagicMock):      spec_set = APIClient -# Create a Bot instance to get a realistic MagicMock of `discord.ext.commands.Bot` -bot_instance = Bot(command_prefix=unittest.mock.MagicMock()) -bot_instance.http_session = None -bot_instance.api_client = None +def _get_mock_loop() -> unittest.mock.Mock: +    """Return a mocked asyncio.AbstractEventLoop.""" +    loop = unittest.mock.create_autospec(spec=AbstractEventLoop, spec_set=True) + +    # Since calling `create_task` on our MockBot does not actually schedule the coroutine object +    # as a task in the asyncio loop, this `side_effect` calls `close()` on the coroutine object +    # to prevent "has not been awaited"-warnings. +    loop.create_task.side_effect = lambda coroutine: coroutine.close() + +    return loop  class MockBot(CustomMockMixin, unittest.mock.MagicMock): @@ -277,17 +308,16 @@ class MockBot(CustomMockMixin, unittest.mock.MagicMock):      Instances of this class will follow the specifications of `discord.ext.commands.Bot` instances.      For more information, see the `MockGuild` docstring.      """ -    spec_set = bot_instance -    additional_spec_asyncs = ("wait_for",) +    spec_set = Bot(command_prefix=unittest.mock.MagicMock(), loop=_get_mock_loop()) +    additional_spec_asyncs = ("wait_for", "redis_ready")      def __init__(self, **kwargs) -> None:          super().__init__(**kwargs) -        self.api_client = MockAPIClient() -        # Since calling `create_task` on our MockBot does not actually schedule the coroutine object -        # as a task in the asyncio loop, this `side_effect` calls `close()` on the coroutine object -        # to prevent "has not been awaited"-warnings. -        self.loop.create_task.side_effect = lambda coroutine: coroutine.close() +        self.loop = _get_mock_loop() +        self.api_client = MockAPIClient(loop=self.loop) +        self.http_session = unittest.mock.create_autospec(spec=ClientSession, spec_set=True) +        self.stats = unittest.mock.create_autospec(spec=AsyncStatsClient, spec_set=True)  # Create a TextChannel instance to get a realistic MagicMock of `discord.TextChannel` | 
