diff options
author | 2021-04-12 16:38:08 +0100 | |
---|---|---|
committer | 2021-04-12 16:38:08 +0100 | |
commit | e4cf3afa2629bdcc287ca2b363505e5a5d1c0e1a (patch) | |
tree | 5511c136833aa07efc89c3eac98a9eb013cb53ca | |
parent | Merge pull request #1516 from ToxicKidz/sorted-available-channels (diff) | |
parent | Merge branch 'main' into doc-imp (diff) |
Merge pull request #1014 from Numerlor/doc-imp
-rw-r--r-- | LICENSE-THIRD-PARTY | 30 | ||||
-rw-r--r-- | Pipfile | 4 | ||||
-rw-r--r-- | Pipfile.lock | 241 | ||||
-rw-r--r-- | bot/converters.py | 44 | ||||
-rw-r--r-- | bot/decorators.py | 12 | ||||
-rw-r--r-- | bot/exts/info/doc.py | 485 | ||||
-rw-r--r-- | bot/exts/info/doc/__init__.py | 16 | ||||
-rw-r--r-- | bot/exts/info/doc/_batch_parser.py | 186 | ||||
-rw-r--r-- | bot/exts/info/doc/_cog.py | 442 | ||||
-rw-r--r-- | bot/exts/info/doc/_html.py | 136 | ||||
-rw-r--r-- | bot/exts/info/doc/_inventory_parser.py | 126 | ||||
-rw-r--r-- | bot/exts/info/doc/_markdown.py | 58 | ||||
-rw-r--r-- | bot/exts/info/doc/_parsing.py | 256 | ||||
-rw-r--r-- | bot/exts/info/doc/_redis_cache.py | 70 | ||||
-rw-r--r-- | bot/exts/info/source.py | 3 | ||||
-rw-r--r-- | bot/utils/function.py | 72 | ||||
-rw-r--r-- | bot/utils/lock.py | 37 | ||||
-rw-r--r-- | bot/utils/messages.py | 4 | ||||
-rw-r--r-- | tests/bot/exts/info/doc/__init__.py | 0 | ||||
-rw-r--r-- | tests/bot/exts/info/doc/test_parsing.py | 66 | ||||
-rw-r--r-- | tests/bot/test_converters.py | 21 |
21 files changed, 1561 insertions, 748 deletions
diff --git a/LICENSE-THIRD-PARTY b/LICENSE-THIRD-PARTY index eacd9b952..ab715630d 100644 --- a/LICENSE-THIRD-PARTY +++ b/LICENSE-THIRD-PARTY @@ -35,6 +35,36 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------------------------- + BSD 2-Clause License +Applies to: + - Copyright (c) 2007-2020 by the Sphinx team (see AUTHORS file). All rights reserved. + - bot/cogs/doc/inventory_parser.py: _load_v1, _load_v2 and ZlibStreamReader.__aiter__. +--------------------------------------------------------------------------------------------------- + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--------------------------------------------------------------------------------------------------- PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 Applies to: - Copyright © 2001-2020 Python Software Foundation. All rights reserved. @@ -20,15 +20,13 @@ emoji = "~=0.6" feedparser = "~=5.2" fuzzywuzzy = "~=0.17" lxml = "~=4.4" -markdownify = "==0.5.3" +markdownify = "==0.6.1" more_itertools = "~=8.2" python-dateutil = "~=2.8" python-frontmatter = "~=1.0.0" pyyaml = "~=5.1" regex = "==2021.4.4" -requests = "~=2.22" sentry-sdk = "~=0.19" -sphinx = "~=2.2" statsd = "~=3.3" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index d6792ac35..1e1a8167b 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "fc3421fc4c95d73b620f2b8b0a7dea288d4fc559e0d288ed4ad6cf4eb312f630" + "sha256": "e35c9bad81b01152ad3e10b85f1abf5866aa87b9d87e03bc30bdb9d37668ccae" }, "pipfile-spec": 6, "requires": { @@ -99,13 +99,6 @@ "markers": "python_version >= '3.6'", "version": "==3.3.1" }, - "alabaster": { - "hashes": [ - "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359", - "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02" - ], - "version": "==0.7.12" - }, "arrow": { "hashes": [ "sha256:3515630f11a15c61dcb4cdd245883270dd334c83f3e639824e65a4b79cc48543", @@ -142,14 +135,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==20.3.0" }, - "babel": { - "hashes": [ - "sha256:9d35c22fcc79893c3ecc85ac4a56cde1ecf3f19c540bba0922308a6c06ca6fa5", - "sha256:da031ab54472314f210b0adcff1588ee5d1d1d0ba4dbd07b94dba82bde791e05" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.9.0" - }, "beautifulsoup4": { "hashes": [ "sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35", @@ -221,7 +206,6 @@ "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" ], - "index": "pypi", "markers": "sys_platform == 'win32'", "version": "==0.4.4" }, @@ -249,14 +233,6 @@ "index": "pypi", "version": "==1.6.0" }, - "docutils": { - "hashes": [ - "sha256:a71042bb7207c03d5647f280427f14bfbd1a65c9eb84f4b341d85fafb6bb4bdf", - "sha256:e2ffeea817964356ba4470efba7c2f42b6b0de0b04e66378507e3e2504bbff4c" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==0.17" - }, "emoji": { "hashes": [ "sha256:e42da4f8d648f8ef10691bc246f682a1ec6b18373abfd9be10ec0b398823bd11" @@ -345,27 +321,11 @@ }, "idna": { "hashes": [ - "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", - "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + "sha256:5205d03e7bcbb919cc9c19885f9920d622ca52448306f2377daede5cf3faac16", + "sha256:c5b02147e01ea9920e6b0a3f1f7bb833612d507592c837a6c49552768f4054e1" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.10" - }, - "imagesize": { - "hashes": [ - "sha256:6965f19a6a2039c7d48bca7dba2473069ff854c36ae6f19d2cde309d998228a1", - "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.2.0" - }, - "jinja2": { - "hashes": [ - "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419", - "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==2.11.3" + "markers": "python_version >= '3.4'", + "version": "==3.1" }, "lxml": { "hashes": [ @@ -411,69 +371,11 @@ }, "markdownify": { "hashes": [ - "sha256:30be8340724e706c9e811c27fe8c1542cf74a15b46827924fff5c54b40dd9b0d", - "sha256:a69588194fd76634f0139d6801b820fd652dc5eeba9530e90d323dfdc0155252" + "sha256:31d7c13ac2ada8bfc7535a25fee6622ca720e1b5f2d4a9cbc429d167c21f886d", + "sha256:7489fd5c601536996a376c4afbcd1dd034db7690af807120681461e82fbc0acc" ], "index": "pypi", - "version": "==0.5.3" - }, - "markupsafe": { - "hashes": [ - "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", - "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", - "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", - "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", - "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", - "sha256:195d7d2c4fbb0ee8139a6cf67194f3973a6b3042d742ebe0a9ed36d8b6f0c07f", - "sha256:22c178a091fc6630d0d045bdb5992d2dfe14e3259760e713c490da5323866c39", - "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", - "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", - "sha256:2beec1e0de6924ea551859edb9e7679da6e4870d32cb766240ce17e0a0ba2014", - "sha256:3b8a6499709d29c2e2399569d96719a1b21dcd94410a586a18526b143ec8470f", - "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", - "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", - "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", - "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", - "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b", - "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", - "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", - "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", - "sha256:6f1e273a344928347c1290119b493a1f0303c52f5a5eae5f16d74f48c15d4a85", - "sha256:6fffc775d90dcc9aed1b89219549b329a9250d918fd0b8fa8d93d154918422e1", - "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", - "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", - "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", - "sha256:7fed13866cf14bba33e7176717346713881f56d9d2bcebab207f7a036f41b850", - "sha256:84dee80c15f1b560d55bcfe6d47b27d070b4681c699c572af2e3c7cc90a3b8e0", - "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", - "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", - "sha256:98bae9582248d6cf62321dcb52aaf5d9adf0bad3b40582925ef7c7f0ed85fceb", - "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", - "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", - "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", - "sha256:a6a744282b7718a2a62d2ed9d993cad6f5f585605ad352c11de459f4108df0a1", - "sha256:acf08ac40292838b3cbbb06cfe9b2cb9ec78fce8baca31ddb87aaac2e2dc3bc2", - "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", - "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", - "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", - "sha256:b1dba4527182c95a0db8b6060cc98ac49b9e2f5e64320e2b56e47cb2831978c7", - "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", - "sha256:b7d644ddb4dbd407d31ffb699f1d140bc35478da613b441c582aeb7c43838dd8", - "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", - "sha256:bf5aa3cbcfdf57fa2ee9cd1822c862ef23037f5c832ad09cfea57fa846dec193", - "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", - "sha256:caabedc8323f1e93231b52fc32bdcde6db817623d33e100708d9a68e1f53b26b", - "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", - "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2", - "sha256:d53bc011414228441014aa71dbec320c66468c1030aae3a6e29778a3382d96e5", - "sha256:d73a845f227b0bfe8a7455ee623525ee656a9e2e749e4742706d80a6065d5e2c", - "sha256:d9be0ba6c527163cbed5e0857c451fcd092ce83947944d6c14bc95441203f032", - "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", - "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be", - "sha256:feb7b34d6325451ef96bc0e36e1a6c0c1c64bc1fbec4b854f4529e51887b1621" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.1.1" + "version": "==0.6.1" }, "more-itertools": { "hashes": [ @@ -533,14 +435,6 @@ "markers": "python_version >= '3.5'", "version": "==4.0.2" }, - "packaging": { - "hashes": [ - "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5", - "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.9" - }, "pamqp": { "hashes": [ "sha256:2f81b5c186f668a67f165193925b6bfd83db4363a6222f599517f29ecee60b02", @@ -590,31 +484,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.20" }, - "pygments": { - "hashes": [ - "sha256:2656e1a6edcdabf4275f9a3640db59fd5de107d88e8663c5d4e9a0fa62f77f94", - "sha256:534ef71d539ae97d4c3a4cf7d6f110f214b0e687e92f9cb9d2a3b0d3101289c8" - ], - "markers": "python_version >= '3.5'", - "version": "==2.8.1" - }, - "pyparsing": { - "hashes": [ - "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", - "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" - ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.4.7" - }, - "pyreadline": { - "hashes": [ - "sha256:4530592fc2e85b25b1a9f79664433da09237c1a270e4d78ea5aa3a2c7229e2d1", - "sha256:65540c21bfe14405a3a77e4c085ecfce88724743a4ead47c66b84defcf82c32e", - "sha256:9ce5fa65b8992dfa373bddc5b6e0864ead8f291c94fbfec05fbd5c836162e67b" - ], - "markers": "sys_platform == 'win32'", - "version": "==2.1" - }, "python-dateutil": { "hashes": [ "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", @@ -631,13 +500,6 @@ "index": "pypi", "version": "==1.0.0" }, - "pytz": { - "hashes": [ - "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da", - "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798" - ], - "version": "==2021.1" - }, "pyyaml": { "hashes": [ "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", @@ -728,14 +590,6 @@ "index": "pypi", "version": "==2021.4.4" }, - "requests": { - "hashes": [ - "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", - "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" - ], - "index": "pypi", - "version": "==2.25.1" - }, "sentry-sdk": { "hashes": [ "sha256:4ae8d1ced6c67f1c8ea51d82a16721c166c489b76876c9f2c202b8a50334b237", @@ -749,16 +603,9 @@ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.15.0" }, - "snowballstemmer": { - "hashes": [ - "sha256:b51b447bea85f9968c13b650126a888aabd4cb4463fca868ec596826325dedc2", - "sha256:e997baa4f2e9139951b6f4c631bad912dfd3c792467e2f03d7239464af90e914" - ], - "version": "==2.1.0" - }, "sortedcontainers": { "hashes": [ "sha256:37257a32add0a3ee490bb170b599e93095eed89a55da91fa9f48753ea12fd73f", @@ -774,62 +621,6 @@ "markers": "python_version >= '3.0'", "version": "==2.2.1" }, - "sphinx": { - "hashes": [ - "sha256:b4c750d546ab6d7e05bdff6ac24db8ae3e8b8253a3569b754e445110a0a12b66", - "sha256:fc312670b56cb54920d6cc2ced455a22a547910de10b3142276495ced49231cb" - ], - "index": "pypi", - "version": "==2.4.4" - }, - "sphinxcontrib-applehelp": { - "hashes": [ - "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a", - "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58" - ], - "markers": "python_version >= '3.5'", - "version": "==1.0.2" - }, - "sphinxcontrib-devhelp": { - "hashes": [ - "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e", - "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4" - ], - "markers": "python_version >= '3.5'", - "version": "==1.0.2" - }, - "sphinxcontrib-htmlhelp": { - "hashes": [ - "sha256:3c0bc24a2c41e340ac37c85ced6dafc879ab485c095b1d65d2461ac2f7cca86f", - "sha256:e8f5bb7e31b2dbb25b9cc435c8ab7a79787ebf7f906155729338f3156d93659b" - ], - "markers": "python_version >= '3.5'", - "version": "==1.0.3" - }, - "sphinxcontrib-jsmath": { - "hashes": [ - "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", - "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8" - ], - "markers": "python_version >= '3.5'", - "version": "==1.0.1" - }, - "sphinxcontrib-qthelp": { - "hashes": [ - "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72", - "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6" - ], - "markers": "python_version >= '3.5'", - "version": "==1.0.3" - }, - "sphinxcontrib-serializinghtml": { - "hashes": [ - "sha256:eaa0eccc86e982a9b939b2b82d12cc5d013385ba5eadcc7e4fed23f4405f77bc", - "sha256:f242a81d423f59617a8e5cf16f5d4d74e28ee9a66f9e5b637a18082991db5a9a" - ], - "markers": "python_version >= '3.5'", - "version": "==1.1.4" - }, "statsd": { "hashes": [ "sha256:c610fb80347fca0ef62666d241bce64184bd7cc1efe582f9690e045c25535eaa", @@ -1103,11 +894,11 @@ }, "idna": { "hashes": [ - "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", - "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + "sha256:5205d03e7bcbb919cc9c19885f9920d622ca52448306f2377daede5cf3faac16", + "sha256:c5b02147e01ea9920e6b0a3f1f7bb833612d507592c837a6c49552768f4054e1" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.10" + "markers": "python_version >= '3.4'", + "version": "==3.1" }, "mccabe": { "hashes": [ @@ -1203,7 +994,7 @@ "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" ], - "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==2.25.1" }, "six": { @@ -1211,7 +1002,7 @@ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.15.0" }, "snowballstemmer": { @@ -1226,7 +1017,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", "version": "==0.10.2" }, "urllib3": { diff --git a/bot/converters.py b/bot/converters.py index 67525cd4d..3bf05cfb3 100644 --- a/bot/converters.py +++ b/bot/converters.py @@ -15,6 +15,7 @@ from discord.utils import DISCORD_EPOCH, snowflake_time from bot.api import ResponseCodeError from bot.constants import URLs +from bot.exts.info.doc import _inventory_parser from bot.utils.regex import INVITE_RE from bot.utils.time import parse_duration_string @@ -127,22 +128,20 @@ class ValidFilterListType(Converter): return list_type -class ValidPythonIdentifier(Converter): +class PackageName(Converter): """ - A converter that checks whether the given string is a valid Python identifier. + A converter that checks whether the given string is a valid package name. - This is used to have package names that correspond to how you would use the package in your - code, e.g. `import package`. - - Raises `BadArgument` if the argument is not a valid Python identifier, and simply passes through - the given argument otherwise. + Package names are used for stats and are restricted to the a-z and _ characters. """ - @staticmethod - async def convert(ctx: Context, argument: str) -> str: - """Checks whether the given string is a valid Python identifier.""" - if not argument.isidentifier(): - raise BadArgument(f"`{argument}` is not a valid Python identifier") + PACKAGE_NAME_RE = re.compile(r"[^a-z0-9_]") + + @classmethod + async def convert(cls, ctx: Context, argument: str) -> str: + """Checks whether the given string is a valid package name.""" + if cls.PACKAGE_NAME_RE.search(argument): + raise BadArgument("The provided package name is not valid; please only use the _, 0-9, and a-z characters.") return argument @@ -178,6 +177,27 @@ class ValidURL(Converter): return url +class Inventory(Converter): + """ + Represents an Intersphinx inventory URL. + + This converter checks whether intersphinx accepts the given inventory URL, and raises + `BadArgument` if that is not the case or if the url is unreachable. + + Otherwise, it returns the url and the fetched inventory dict in a tuple. + """ + + @staticmethod + async def convert(ctx: Context, url: str) -> t.Tuple[str, _inventory_parser.InventoryDict]: + """Convert url to Intersphinx inventory URL.""" + await ctx.trigger_typing() + if (inventory := await _inventory_parser.fetch_inventory(url)) is None: + raise BadArgument( + f"Failed to fetch inventory file after {_inventory_parser.FAILED_REQUEST_ATTEMPTS} attempts." + ) + return url, inventory + + class Snowflake(IDConverter): """ Converts to an int if the argument is a valid Discord snowflake. diff --git a/bot/decorators.py b/bot/decorators.py index 0b50cc365..1d30317ef 100644 --- a/bot/decorators.py +++ b/bot/decorators.py @@ -1,9 +1,9 @@ import asyncio import functools import logging +import types import typing as t from contextlib import suppress -from functools import wraps from discord import Member, NotFound from discord.ext import commands @@ -12,6 +12,7 @@ from discord.ext.commands import Cog, Context from bot.constants import Channels, DEBUG_MODE, RedirectOutput from bot.utils import function from bot.utils.checks import in_whitelist_check +from bot.utils.function import command_wraps log = logging.getLogger(__name__) @@ -71,8 +72,8 @@ def redirect_output(destination_channel: int, bypass_roles: t.Container[int] = N This decorator must go before (below) the `command` decorator. """ - def wrap(func: t.Callable) -> t.Callable: - @wraps(func) + def wrap(func: types.FunctionType) -> types.FunctionType: + @command_wraps(func) async def inner(self: Cog, ctx: Context, *args, **kwargs) -> None: if ctx.channel.id == destination_channel: log.trace(f"Command {ctx.command.name} was invoked in destination_channel, not redirecting") @@ -106,7 +107,6 @@ def redirect_output(destination_channel: int, bypass_roles: t.Container[int] = N with suppress(NotFound): await ctx.message.delete() log.trace("Redirect output: Deleted invocation message") - return inner return wrap @@ -123,8 +123,8 @@ def respect_role_hierarchy(member_arg: function.Argument) -> t.Callable: This decorator must go before (below) the `command` decorator. """ - def decorator(func: t.Callable) -> t.Callable: - @wraps(func) + def decorator(func: types.FunctionType) -> types.FunctionType: + @command_wraps(func) async def wrapper(*args, **kwargs) -> None: log.trace(f"{func.__name__}: respect role hierarchy decorator called") diff --git a/bot/exts/info/doc.py b/bot/exts/info/doc.py deleted file mode 100644 index 9b5bd6504..000000000 --- a/bot/exts/info/doc.py +++ /dev/null @@ -1,485 +0,0 @@ -import asyncio -import functools -import logging -import re -import textwrap -from contextlib import suppress -from types import SimpleNamespace -from typing import Optional, Tuple - -import discord -from bs4 import BeautifulSoup -from bs4.element import PageElement, Tag -from discord.errors import NotFound -from discord.ext import commands -from markdownify import MarkdownConverter -from requests import ConnectTimeout, ConnectionError, HTTPError -from sphinx.ext import intersphinx -from urllib3.exceptions import ProtocolError - -from bot.bot import Bot -from bot.constants import MODERATION_ROLES, RedirectOutput -from bot.converters import ValidPythonIdentifier, ValidURL -from bot.pagination import LinePaginator -from bot.utils.cache import AsyncCache -from bot.utils.messages import wait_for_deletion - - -log = logging.getLogger(__name__) -logging.getLogger('urllib3').setLevel(logging.WARNING) - -# Since Intersphinx is intended to be used with Sphinx, -# we need to mock its configuration. -SPHINX_MOCK_APP = SimpleNamespace( - config=SimpleNamespace( - intersphinx_timeout=3, - tls_verify=True, - user_agent="python3:python-discord/bot:1.0.0" - ) -) - -NO_OVERRIDE_GROUPS = ( - "2to3fixer", - "token", - "label", - "pdbcommand", - "term", -) -NO_OVERRIDE_PACKAGES = ( - "python", -) - -SEARCH_END_TAG_ATTRS = ( - "data", - "function", - "class", - "exception", - "seealso", - "section", - "rubric", - "sphinxsidebar", -) -UNWANTED_SIGNATURE_SYMBOLS_RE = re.compile(r"\[source]|\\\\|¶") -WHITESPACE_AFTER_NEWLINES_RE = re.compile(r"(?<=\n\n)(\s+)") - -FAILED_REQUEST_RETRY_AMOUNT = 3 -NOT_FOUND_DELETE_DELAY = RedirectOutput.delete_delay - -symbol_cache = AsyncCache() - - -class DocMarkdownConverter(MarkdownConverter): - """Subclass markdownify's MarkdownCoverter to provide custom conversion methods.""" - - def convert_code(self, el: PageElement, text: str) -> str: - """Undo `markdownify`s underscore escaping.""" - return f"`{text}`".replace('\\', '') - - def convert_pre(self, el: PageElement, text: str) -> str: - """Wrap any codeblocks in `py` for syntax highlighting.""" - code = ''.join(el.strings) - return f"```py\n{code}```" - - -def markdownify(html: str) -> DocMarkdownConverter: - """Create a DocMarkdownConverter object from the input html.""" - return DocMarkdownConverter(bullets='•').convert(html) - - -class InventoryURL(commands.Converter): - """ - Represents an Intersphinx inventory URL. - - This converter checks whether intersphinx accepts the given inventory URL, and raises - `BadArgument` if that is not the case. - - Otherwise, it simply passes through the given URL. - """ - - @staticmethod - async def convert(ctx: commands.Context, url: str) -> str: - """Convert url to Intersphinx inventory URL.""" - try: - intersphinx.fetch_inventory(SPHINX_MOCK_APP, '', url) - except AttributeError: - raise commands.BadArgument(f"Failed to fetch Intersphinx inventory from URL `{url}`.") - except ConnectionError: - if url.startswith('https'): - raise commands.BadArgument( - f"Cannot establish a connection to `{url}`. Does it support HTTPS?" - ) - raise commands.BadArgument(f"Cannot connect to host with URL `{url}`.") - except ValueError: - raise commands.BadArgument( - f"Failed to read Intersphinx inventory from URL `{url}`. " - "Are you sure that it's a valid inventory file?" - ) - return url - - -class Doc(commands.Cog): - """A set of commands for querying & displaying documentation.""" - - def __init__(self, bot: Bot): - self.base_urls = {} - self.bot = bot - self.inventories = {} - self.renamed_symbols = set() - - self.bot.loop.create_task(self.init_refresh_inventory()) - - async def init_refresh_inventory(self) -> None: - """Refresh documentation inventory on cog initialization.""" - await self.bot.wait_until_guild_available() - await self.refresh_inventory() - - async def update_single( - self, package_name: str, base_url: str, inventory_url: str - ) -> None: - """ - Rebuild the inventory for a single package. - - Where: - * `package_name` is the package name to use, appears in the log - * `base_url` is the root documentation URL for the specified package, used to build - absolute paths that link to specific symbols - * `inventory_url` is the absolute URL to the intersphinx inventory, fetched by running - `intersphinx.fetch_inventory` in an executor on the bot's event loop - """ - self.base_urls[package_name] = base_url - - package = await self._fetch_inventory(inventory_url) - if not package: - return None - - for group, value in package.items(): - for symbol, (package_name, _version, relative_doc_url, _) in value.items(): - absolute_doc_url = base_url + relative_doc_url - - if symbol in self.inventories: - group_name = group.split(":")[1] - symbol_base_url = self.inventories[symbol].split("/", 3)[2] - if ( - group_name in NO_OVERRIDE_GROUPS - or any(package in symbol_base_url for package in NO_OVERRIDE_PACKAGES) - ): - - symbol = f"{group_name}.{symbol}" - # If renamed `symbol` already exists, add library name in front to differentiate between them. - if symbol in self.renamed_symbols: - # Split `package_name` because of packages like Pillow that have spaces in them. - symbol = f"{package_name.split()[0]}.{symbol}" - - self.inventories[symbol] = absolute_doc_url - self.renamed_symbols.add(symbol) - continue - - self.inventories[symbol] = absolute_doc_url - - log.trace(f"Fetched inventory for {package_name}.") - - async def refresh_inventory(self) -> None: - """Refresh internal documentation inventory.""" - log.debug("Refreshing documentation inventory...") - - # Clear the old base URLS and inventories to ensure - # that we start from a fresh local dataset. - # Also, reset the cache used for fetching documentation. - self.base_urls.clear() - self.inventories.clear() - self.renamed_symbols.clear() - symbol_cache.clear() - - # Run all coroutines concurrently - since each of them performs a HTTP - # request, this speeds up fetching the inventory data heavily. - coros = [ - self.update_single( - package["package"], package["base_url"], package["inventory_url"] - ) for package in await self.bot.api_client.get('bot/documentation-links') - ] - await asyncio.gather(*coros) - - async def get_symbol_html(self, symbol: str) -> Optional[Tuple[list, str]]: - """ - Given a Python symbol, return its signature and description. - - The first tuple element is the signature of the given symbol as a markup-free string, and - the second tuple element is the description of the given symbol with HTML markup included. - - If the given symbol is a module, returns a tuple `(None, str)` - else if the symbol could not be found, returns `None`. - """ - url = self.inventories.get(symbol) - if url is None: - return None - - async with self.bot.http_session.get(url) as response: - html = await response.text(encoding='utf-8') - - # Find the signature header and parse the relevant parts. - symbol_id = url.split('#')[-1] - soup = BeautifulSoup(html, 'lxml') - symbol_heading = soup.find(id=symbol_id) - search_html = str(soup) - - if symbol_heading is None: - return None - - if symbol_id == f"module-{symbol}": - # Get page content from the module headerlink to the - # first tag that has its class in `SEARCH_END_TAG_ATTRS` - start_tag = symbol_heading.find("a", attrs={"class": "headerlink"}) - if start_tag is None: - return [], "" - - end_tag = start_tag.find_next(self._match_end_tag) - if end_tag is None: - return [], "" - - description_start_index = search_html.find(str(start_tag.parent)) + len(str(start_tag.parent)) - description_end_index = search_html.find(str(end_tag)) - description = search_html[description_start_index:description_end_index] - signatures = None - - else: - signatures = [] - description = str(symbol_heading.find_next_sibling("dd")) - description_pos = search_html.find(description) - # Get text of up to 3 signatures, remove unwanted symbols - for element in [symbol_heading] + symbol_heading.find_next_siblings("dt", limit=2): - signature = UNWANTED_SIGNATURE_SYMBOLS_RE.sub("", element.text) - if signature and search_html.find(str(element)) < description_pos: - signatures.append(signature) - - return signatures, description.replace('¶', '') - - @symbol_cache(arg_offset=1) - async def get_symbol_embed(self, symbol: str) -> Optional[discord.Embed]: - """ - Attempt to scrape and fetch the data for the given `symbol`, and build an embed from its contents. - - If the symbol is known, an Embed with documentation about it is returned. - """ - scraped_html = await self.get_symbol_html(symbol) - if scraped_html is None: - return None - - signatures = scraped_html[0] - permalink = self.inventories[symbol] - description = markdownify(scraped_html[1]) - - # Truncate the description of the embed to the last occurrence - # of a double newline (interpreted as a paragraph) before index 1000. - if len(description) > 1000: - shortened = description[:1000] - description_cutoff = shortened.rfind('\n\n', 100) - if description_cutoff == -1: - # Search the shortened version for cutoff points in decreasing desirability, - # cutoff at 1000 if none are found. - for string in (". ", ", ", ",", " "): - description_cutoff = shortened.rfind(string) - if description_cutoff != -1: - break - else: - description_cutoff = 1000 - description = description[:description_cutoff] - - # If there is an incomplete code block, cut it out - if description.count("```") % 2: - codeblock_start = description.rfind('```py') - description = description[:codeblock_start].rstrip() - description += f"... [read more]({permalink})" - - description = WHITESPACE_AFTER_NEWLINES_RE.sub('', description) - if signatures is None: - # If symbol is a module, don't show signature. - embed_description = description - - elif not signatures: - # It's some "meta-page", for example: - # https://docs.djangoproject.com/en/dev/ref/views/#module-django.views - embed_description = "This appears to be a generic page not tied to a specific symbol." - - else: - embed_description = "".join(f"```py\n{textwrap.shorten(signature, 500)}```" for signature in signatures) - embed_description += f"\n{description}" - - embed = discord.Embed( - title=f'`{symbol}`', - url=permalink, - description=embed_description - ) - # Show all symbols with the same name that were renamed in the footer. - embed.set_footer( - text=", ".join(renamed for renamed in self.renamed_symbols - {symbol} if renamed.endswith(f".{symbol}")) - ) - return embed - - @commands.group(name='docs', aliases=('doc', 'd'), invoke_without_command=True) - async def docs_group(self, ctx: commands.Context, symbol: commands.clean_content = None) -> None: - """Lookup documentation for Python symbols.""" - await self.get_command(ctx, symbol) - - @docs_group.command(name='get', aliases=('g',)) - async def get_command(self, ctx: commands.Context, symbol: commands.clean_content = None) -> None: - """ - Return a documentation embed for a given symbol. - - If no symbol is given, return a list of all available inventories. - - Examples: - !docs - !docs aiohttp - !docs aiohttp.ClientSession - !docs get aiohttp.ClientSession - """ - if symbol is None: - inventory_embed = discord.Embed( - title=f"All inventories (`{len(self.base_urls)}` total)", - colour=discord.Colour.blue() - ) - - lines = sorted(f"• [`{name}`]({url})" for name, url in self.base_urls.items()) - if self.base_urls: - await LinePaginator.paginate(lines, ctx, inventory_embed, max_size=400, empty=False) - - else: - inventory_embed.description = "Hmmm, seems like there's nothing here yet." - await ctx.send(embed=inventory_embed) - - else: - # Fetching documentation for a symbol (at least for the first time, since - # caching is used) takes quite some time, so let's send typing to indicate - # that we got the command, but are still working on it. - async with ctx.typing(): - doc_embed = await self.get_symbol_embed(symbol) - - if doc_embed is None: - error_embed = discord.Embed( - description=f"Sorry, I could not find any documentation for `{symbol}`.", - colour=discord.Colour.red() - ) - error_message = await ctx.send(embed=error_embed) - with suppress(NotFound): - await error_message.delete(delay=NOT_FOUND_DELETE_DELAY) - await ctx.message.delete(delay=NOT_FOUND_DELETE_DELAY) - else: - msg = await ctx.send(embed=doc_embed) - await wait_for_deletion(msg, (ctx.author.id,)) - - @docs_group.command(name='set', aliases=('s',)) - @commands.has_any_role(*MODERATION_ROLES) - async def set_command( - self, ctx: commands.Context, package_name: ValidPythonIdentifier, - base_url: ValidURL, inventory_url: InventoryURL - ) -> None: - """ - Adds a new documentation metadata object to the site's database. - - The database will update the object, should an existing item with the specified `package_name` already exist. - - Example: - !docs set \ - python \ - https://docs.python.org/3/ \ - https://docs.python.org/3/objects.inv - """ - body = { - 'package': package_name, - 'base_url': base_url, - 'inventory_url': inventory_url - } - await self.bot.api_client.post('bot/documentation-links', json=body) - - log.info( - f"User @{ctx.author} ({ctx.author.id}) added a new documentation package:\n" - f"Package name: {package_name}\n" - f"Base url: {base_url}\n" - f"Inventory URL: {inventory_url}" - ) - - # Rebuilding the inventory can take some time, so lets send out a - # typing event to show that the Bot is still working. - async with ctx.typing(): - await self.refresh_inventory() - await ctx.send(f"Added package `{package_name}` to database and refreshed inventory.") - - @docs_group.command(name='delete', aliases=('remove', 'rm', 'd')) - @commands.has_any_role(*MODERATION_ROLES) - async def delete_command(self, ctx: commands.Context, package_name: ValidPythonIdentifier) -> None: - """ - Removes the specified package from the database. - - Examples: - !docs delete aiohttp - """ - await self.bot.api_client.delete(f'bot/documentation-links/{package_name}') - - async with ctx.typing(): - # Rebuild the inventory to ensure that everything - # that was from this package is properly deleted. - await self.refresh_inventory() - await ctx.send(f"Successfully deleted `{package_name}` and refreshed inventory.") - - @docs_group.command(name="refresh", aliases=("rfsh", "r")) - @commands.has_any_role(*MODERATION_ROLES) - async def refresh_command(self, ctx: commands.Context) -> None: - """Refresh inventories and send differences to channel.""" - old_inventories = set(self.base_urls) - with ctx.typing(): - await self.refresh_inventory() - # Get differences of added and removed inventories - added = ', '.join(inv for inv in self.base_urls if inv not in old_inventories) - if added: - added = f"+ {added}" - - removed = ', '.join(inv for inv in old_inventories if inv not in self.base_urls) - if removed: - removed = f"- {removed}" - - embed = discord.Embed( - title="Inventories refreshed", - description=f"```diff\n{added}\n{removed}```" if added or removed else "" - ) - await ctx.send(embed=embed) - - async def _fetch_inventory(self, inventory_url: str) -> Optional[dict]: - """Get and return inventory from `inventory_url`. If fetching fails, return None.""" - fetch_func = functools.partial(intersphinx.fetch_inventory, SPHINX_MOCK_APP, '', inventory_url) - for retry in range(1, FAILED_REQUEST_RETRY_AMOUNT+1): - try: - package = await self.bot.loop.run_in_executor(None, fetch_func) - except ConnectTimeout: - log.error( - f"Fetching of inventory {inventory_url} timed out," - f" trying again. ({retry}/{FAILED_REQUEST_RETRY_AMOUNT})" - ) - except ProtocolError: - log.error( - f"Connection lost while fetching inventory {inventory_url}," - f" trying again. ({retry}/{FAILED_REQUEST_RETRY_AMOUNT})" - ) - except HTTPError as e: - log.error(f"Fetching of inventory {inventory_url} failed with status code {e.response.status_code}.") - return None - except ConnectionError: - log.error(f"Couldn't establish connection to inventory {inventory_url}.") - return None - else: - return package - log.error(f"Fetching of inventory {inventory_url} failed.") - return None - - @staticmethod - def _match_end_tag(tag: Tag) -> bool: - """Matches `tag` if its class value is in `SEARCH_END_TAG_ATTRS` or the tag is table.""" - for attr in SEARCH_END_TAG_ATTRS: - if attr in tag.get("class", ()): - return True - - return tag.name == "table" - - -def setup(bot: Bot) -> None: - """Load the Doc cog.""" - bot.add_cog(Doc(bot)) diff --git a/bot/exts/info/doc/__init__.py b/bot/exts/info/doc/__init__.py new file mode 100644 index 000000000..38a8975c0 --- /dev/null +++ b/bot/exts/info/doc/__init__.py @@ -0,0 +1,16 @@ +from bot.bot import Bot +from ._redis_cache import DocRedisCache + +MAX_SIGNATURE_AMOUNT = 3 +PRIORITY_PACKAGES = ( + "python", +) +NAMESPACE = "doc" + +doc_cache = DocRedisCache(namespace=NAMESPACE) + + +def setup(bot: Bot) -> None: + """Load the Doc cog.""" + from ._cog import DocCog + bot.add_cog(DocCog(bot)) diff --git a/bot/exts/info/doc/_batch_parser.py b/bot/exts/info/doc/_batch_parser.py new file mode 100644 index 000000000..369bb462c --- /dev/null +++ b/bot/exts/info/doc/_batch_parser.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +import asyncio +import collections +import logging +from collections import defaultdict +from contextlib import suppress +from operator import attrgetter +from typing import Deque, Dict, List, NamedTuple, Optional, Union + +import discord +from bs4 import BeautifulSoup + +import bot +from bot.constants import Channels +from bot.utils import scheduling +from . import _cog, doc_cache +from ._parsing import get_symbol_markdown + +log = logging.getLogger(__name__) + + +class StaleInventoryNotifier: + """Handle sending notifications about stale inventories through `DocItem`s to dev log.""" + + def __init__(self): + self._init_task = bot.instance.loop.create_task( + self._init_channel(), + name="StaleInventoryNotifier channel init" + ) + self._warned_urls = set() + + async def _init_channel(self) -> None: + """Wait for guild and get channel.""" + await bot.instance.wait_until_guild_available() + self._dev_log = bot.instance.get_channel(Channels.dev_log) + + async def send_warning(self, doc_item: _cog.DocItem) -> None: + """Send a warning to dev log if one wasn't already sent for `item`'s url.""" + if doc_item.url not in self._warned_urls: + self._warned_urls.add(doc_item.url) + await self._init_task + embed = discord.Embed( + description=f"Doc item `{doc_item.symbol_id=}` present in loaded documentation inventories " + f"not found on [site]({doc_item.url}), inventories may need to be refreshed." + ) + await self._dev_log.send(embed=embed) + + +class QueueItem(NamedTuple): + """Contains a `DocItem` and the `BeautifulSoup` object needed to parse it.""" + + doc_item: _cog.DocItem + soup: BeautifulSoup + + def __eq__(self, other: Union[QueueItem, _cog.DocItem]): + if isinstance(other, _cog.DocItem): + return self.doc_item == other + return NamedTuple.__eq__(self, other) + + +class ParseResultFuture(asyncio.Future): + """ + Future with metadata for the parser class. + + `user_requested` is set by the parser when a Future is requested by an user and moved to the front, + allowing the futures to only be waited for when clearing if they were user requested. + """ + + def __init__(self): + super().__init__() + self.user_requested = False + + +class BatchParser: + """ + Get the Markdown of all symbols on a page and send them to redis when a symbol is requested. + + DocItems are added through the `add_item` method which adds them to the `_page_doc_items` dict. + `get_markdown` is used to fetch the Markdown; when this is used for the first time on a page, + all of the symbols are queued to be parsed to avoid multiple web requests to the same page. + """ + + def __init__(self): + self._queue: Deque[QueueItem] = collections.deque() + self._page_doc_items: Dict[str, List[_cog.DocItem]] = defaultdict(list) + self._item_futures: Dict[_cog.DocItem, ParseResultFuture] = defaultdict(ParseResultFuture) + self._parse_task = None + + self.stale_inventory_notifier = StaleInventoryNotifier() + + async def get_markdown(self, doc_item: _cog.DocItem) -> Optional[str]: + """ + Get the result Markdown of `doc_item`. + + If no symbols were fetched from `doc_item`s page before, + the HTML has to be fetched and then all items from the page are put into the parse queue. + + Not safe to run while `self.clear` is running. + """ + if doc_item not in self._item_futures and doc_item not in self._queue: + self._item_futures[doc_item].user_requested = True + + async with bot.instance.http_session.get(doc_item.url) as response: + soup = await bot.instance.loop.run_in_executor( + None, + BeautifulSoup, + await response.text(encoding="utf8"), + "lxml", + ) + + self._queue.extendleft(QueueItem(item, soup) for item in self._page_doc_items[doc_item.url]) + log.debug(f"Added items from {doc_item.url} to the parse queue.") + + if self._parse_task is None: + self._parse_task = scheduling.create_task(self._parse_queue(), name="Queue parse") + else: + self._item_futures[doc_item].user_requested = True + with suppress(ValueError): + # If the item is not in the queue then the item is already parsed or is being parsed + self._move_to_front(doc_item) + return await self._item_futures[doc_item] + + async def _parse_queue(self) -> None: + """ + Parse all items from the queue, setting their result Markdown on the futures and sending them to redis. + + The coroutine will run as long as the queue is not empty, resetting `self._parse_task` to None when finished. + """ + log.trace("Starting queue parsing.") + try: + while self._queue: + item, soup = self._queue.pop() + markdown = None + + if (future := self._item_futures[item]).done(): + # Some items are present in the inventories multiple times under different symbol names, + # if we already parsed an equal item, we can just skip it. + continue + + try: + markdown = await bot.instance.loop.run_in_executor(None, get_symbol_markdown, soup, item) + if markdown is not None: + await doc_cache.set(item, markdown) + else: + # Don't wait for this coro as the parsing doesn't depend on anything it does. + scheduling.create_task( + self.stale_inventory_notifier.send_warning(item), name="Stale inventory warning" + ) + except Exception: + log.exception(f"Unexpected error when handling {item}") + future.set_result(markdown) + del self._item_futures[item] + await asyncio.sleep(0.1) + finally: + self._parse_task = None + log.trace("Finished parsing queue.") + + def _move_to_front(self, item: Union[QueueItem, _cog.DocItem]) -> None: + """Move `item` to the front of the parse queue.""" + # The parse queue stores soups along with the doc symbols in QueueItem objects, + # in case we're moving a DocItem we have to get the associated QueueItem first and then move it. + item_index = self._queue.index(item) + queue_item = self._queue[item_index] + del self._queue[item_index] + + self._queue.append(queue_item) + log.trace(f"Moved {item} to the front of the queue.") + + def add_item(self, doc_item: _cog.DocItem) -> None: + """Map a DocItem to its page so that the symbol will be parsed once the page is requested.""" + self._page_doc_items[doc_item.url].append(doc_item) + + async def clear(self) -> None: + """ + Clear all internal symbol data. + + Wait for all user-requested symbols to be parsed before clearing the parser. + """ + for future in filter(attrgetter("user_requested"), self._item_futures.values()): + await future + if self._parse_task is not None: + self._parse_task.cancel() + self._queue.clear() + self._page_doc_items.clear() + self._item_futures.clear() diff --git a/bot/exts/info/doc/_cog.py b/bot/exts/info/doc/_cog.py new file mode 100644 index 000000000..2a8016fb8 --- /dev/null +++ b/bot/exts/info/doc/_cog.py @@ -0,0 +1,442 @@ +from __future__ import annotations + +import asyncio +import logging +import sys +import textwrap +from collections import defaultdict +from contextlib import suppress +from types import SimpleNamespace +from typing import Dict, NamedTuple, Optional, Tuple, Union + +import aiohttp +import discord +from discord.ext import commands + +from bot.bot import Bot +from bot.constants import MODERATION_ROLES, RedirectOutput +from bot.converters import Inventory, PackageName, ValidURL, allowed_strings +from bot.pagination import LinePaginator +from bot.utils.lock import SharedEvent, lock +from bot.utils.messages import send_denial, wait_for_deletion +from bot.utils.scheduling import Scheduler +from . import NAMESPACE, PRIORITY_PACKAGES, _batch_parser, doc_cache +from ._inventory_parser import InventoryDict, fetch_inventory + +log = logging.getLogger(__name__) + +# symbols with a group contained here will get the group prefixed on duplicates +FORCE_PREFIX_GROUPS = ( + "2to3fixer", + "token", + "label", + "pdbcommand", + "term", +) +NOT_FOUND_DELETE_DELAY = RedirectOutput.delete_delay +# Delay to wait before trying to reach a rescheduled inventory again, in minutes +FETCH_RESCHEDULE_DELAY = SimpleNamespace(first=2, repeated=5) + +COMMAND_LOCK_SINGLETON = "inventory refresh" + + +class DocItem(NamedTuple): + """Holds inventory symbol information.""" + + package: str # Name of the package name the symbol is from + group: str # Interpshinx "role" of the symbol, for example `label` or `method` + base_url: str # Absolute path to to which the relative path resolves, same for all items with the same package + relative_url_path: str # Relative path to the page where the symbol is located + symbol_id: str # Fragment id used to locate the symbol on the page + + @property + def url(self) -> str: + """Return the absolute url to the symbol.""" + return self.base_url + self.relative_url_path + + +class DocCog(commands.Cog): + """A set of commands for querying & displaying documentation.""" + + def __init__(self, bot: Bot): + # Contains URLs to documentation home pages. + # Used to calculate inventory diffs on refreshes and to display all currently stored inventories. + self.base_urls = {} + self.bot = bot + self.doc_symbols: Dict[str, DocItem] = {} # Maps symbol names to objects containing their metadata. + self.item_fetcher = _batch_parser.BatchParser() + # Maps a conflicting symbol name to a list of the new, disambiguated names created from conflicts with the name. + self.renamed_symbols = defaultdict(list) + + self.inventory_scheduler = Scheduler(self.__class__.__name__) + + self.refresh_event = asyncio.Event() + self.refresh_event.set() + self.symbol_get_event = SharedEvent() + + self.init_refresh_task = self.bot.loop.create_task( + self.init_refresh_inventory(), + name="Doc inventory init" + ) + + @lock(NAMESPACE, COMMAND_LOCK_SINGLETON, raise_error=True) + async def init_refresh_inventory(self) -> None: + """Refresh documentation inventory on cog initialization.""" + await self.bot.wait_until_guild_available() + await self.refresh_inventories() + + def update_single(self, package_name: str, base_url: str, inventory: InventoryDict) -> None: + """ + Build the inventory for a single package. + + Where: + * `package_name` is the package name to use in logs and when qualifying symbols + * `base_url` is the root documentation URL for the specified package, used to build + absolute paths that link to specific symbols + * `package` is the content of a intersphinx inventory. + """ + self.base_urls[package_name] = base_url + + for group, items in inventory.items(): + for symbol_name, relative_doc_url in items: + + # e.g. get 'class' from 'py:class' + group_name = group.split(":")[1] + symbol_name = self.ensure_unique_symbol_name( + package_name, + group_name, + symbol_name, + ) + + relative_url_path, _, symbol_id = relative_doc_url.partition("#") + # Intern fields that have shared content so we're not storing unique strings for every object + doc_item = DocItem( + package_name, + sys.intern(group_name), + base_url, + sys.intern(relative_url_path), + symbol_id, + ) + self.doc_symbols[symbol_name] = doc_item + self.item_fetcher.add_item(doc_item) + + log.trace(f"Fetched inventory for {package_name}.") + + async def update_or_reschedule_inventory( + self, + api_package_name: str, + base_url: str, + inventory_url: str, + ) -> None: + """ + Update the cog's inventories, or reschedule this method to execute again if the remote inventory is unreachable. + + The first attempt is rescheduled to execute in `FETCH_RESCHEDULE_DELAY.first` minutes, the subsequent attempts + in `FETCH_RESCHEDULE_DELAY.repeated` minutes. + """ + package = await fetch_inventory(inventory_url) + + if not package: + if api_package_name in self.inventory_scheduler: + self.inventory_scheduler.cancel(api_package_name) + delay = FETCH_RESCHEDULE_DELAY.repeated + else: + delay = FETCH_RESCHEDULE_DELAY.first + log.info(f"Failed to fetch inventory; attempting again in {delay} minutes.") + self.inventory_scheduler.schedule_later( + delay*60, + api_package_name, + self.update_or_reschedule_inventory(api_package_name, base_url, inventory_url), + ) + else: + self.update_single(api_package_name, base_url, package) + + def ensure_unique_symbol_name(self, package_name: str, group_name: str, symbol_name: str) -> str: + """ + Ensure `symbol_name` doesn't overwrite an another symbol in `doc_symbols`. + + For conflicts, rename either the current symbol or the existing symbol with which it conflicts. + Store the new name in `renamed_symbols` and return the name to use for the symbol. + + If the existing symbol was renamed or there was no conflict, the returned name is equivalent to `symbol_name`. + """ + if (item := self.doc_symbols.get(symbol_name)) is None: + return symbol_name # There's no conflict so it's fine to simply use the given symbol name. + + def rename(prefix: str, *, rename_extant: bool = False) -> str: + new_name = f"{prefix}.{symbol_name}" + if new_name in self.doc_symbols: + # If there's still a conflict, qualify the name further. + if rename_extant: + new_name = f"{item.package}.{item.group}.{symbol_name}" + else: + new_name = f"{package_name}.{group_name}.{symbol_name}" + + self.renamed_symbols[symbol_name].append(new_name) + + if rename_extant: + # Instead of renaming the current symbol, rename the symbol with which it conflicts. + self.doc_symbols[new_name] = self.doc_symbols[symbol_name] + return symbol_name + else: + return new_name + + # Certain groups are added as prefixes to disambiguate the symbols. + if group_name in FORCE_PREFIX_GROUPS: + return rename(group_name) + + # The existing symbol with which the current symbol conflicts should have a group prefix. + # It currently doesn't have the group prefix because it's only added once there's a conflict. + elif item.group in FORCE_PREFIX_GROUPS: + return rename(item.group, rename_extant=True) + + elif package_name in PRIORITY_PACKAGES: + return rename(item.package, rename_extant=True) + + # If we can't specially handle the symbol through its group or package, + # fall back to prepending its package name to the front. + else: + return rename(package_name) + + async def refresh_inventories(self) -> None: + """Refresh internal documentation inventories.""" + self.refresh_event.clear() + await self.symbol_get_event.wait() + log.debug("Refreshing documentation inventory...") + self.inventory_scheduler.cancel_all() + + self.base_urls.clear() + self.doc_symbols.clear() + self.renamed_symbols.clear() + await self.item_fetcher.clear() + + coros = [ + self.update_or_reschedule_inventory( + package["package"], package["base_url"], package["inventory_url"] + ) for package in await self.bot.api_client.get("bot/documentation-links") + ] + await asyncio.gather(*coros) + log.debug("Finished inventory refresh.") + self.refresh_event.set() + + def get_symbol_item(self, symbol_name: str) -> Tuple[str, Optional[DocItem]]: + """ + Get the `DocItem` and the symbol name used to fetch it from the `doc_symbols` dict. + + If the doc item is not found directly from the passed in name and the name contains a space, + the first word of the name will be attempted to be used to get the item. + """ + doc_item = self.doc_symbols.get(symbol_name) + if doc_item is None and " " in symbol_name: + symbol_name = symbol_name.split(" ", maxsplit=1)[0] + doc_item = self.doc_symbols.get(symbol_name) + + return symbol_name, doc_item + + async def get_symbol_markdown(self, doc_item: DocItem) -> str: + """ + Get the Markdown from the symbol `doc_item` refers to. + + First a redis lookup is attempted, if that fails the `item_fetcher` + is used to fetch the page and parse the HTML from it into Markdown. + """ + markdown = await doc_cache.get(doc_item) + + if markdown is None: + log.debug(f"Redis cache miss with {doc_item}.") + try: + markdown = await self.item_fetcher.get_markdown(doc_item) + + except aiohttp.ClientError as e: + log.warning(f"A network error has occurred when requesting parsing of {doc_item}.", exc_info=e) + return "Unable to parse the requested symbol due to a network error." + + except Exception: + log.exception(f"An unexpected error has occurred when requesting parsing of {doc_item}.") + return "Unable to parse the requested symbol due to an error." + + if markdown is None: + return "Unable to parse the requested symbol." + return markdown + + async def create_symbol_embed(self, symbol_name: str) -> Optional[discord.Embed]: + """ + Attempt to scrape and fetch the data for the given `symbol_name`, and build an embed from its contents. + + If the symbol is known, an Embed with documentation about it is returned. + + First check the DocRedisCache before querying the cog's `BatchParser`. + """ + log.trace(f"Building embed for symbol `{symbol_name}`") + if not self.refresh_event.is_set(): + log.debug("Waiting for inventories to be refreshed before processing item.") + await self.refresh_event.wait() + # Ensure a refresh can't run in case of a context switch until the with block is exited + with self.symbol_get_event: + symbol_name, doc_item = self.get_symbol_item(symbol_name) + if doc_item is None: + log.debug("Symbol does not exist.") + return None + + self.bot.stats.incr(f"doc_fetches.{doc_item.package}") + + # Show all symbols with the same name that were renamed in the footer, + # with a max of 200 chars. + if symbol_name in self.renamed_symbols: + renamed_symbols = ", ".join(self.renamed_symbols[symbol_name]) + footer_text = textwrap.shorten("Similar names: " + renamed_symbols, 200, placeholder=" ...") + else: + footer_text = "" + + embed = discord.Embed( + title=discord.utils.escape_markdown(symbol_name), + url=f"{doc_item.url}#{doc_item.symbol_id}", + description=await self.get_symbol_markdown(doc_item) + ) + embed.set_footer(text=footer_text) + return embed + + @commands.group(name="docs", aliases=("doc", "d"), invoke_without_command=True) + async def docs_group(self, ctx: commands.Context, *, symbol_name: Optional[str]) -> None: + """Look up documentation for Python symbols.""" + await self.get_command(ctx, symbol_name=symbol_name) + + @docs_group.command(name="getdoc", aliases=("g",)) + async def get_command(self, ctx: commands.Context, *, symbol_name: Optional[str]) -> None: + """ + Return a documentation embed for a given symbol. + + If no symbol is given, return a list of all available inventories. + + Examples: + !docs + !docs aiohttp + !docs aiohttp.ClientSession + !docs getdoc aiohttp.ClientSession + """ + if not symbol_name: + inventory_embed = discord.Embed( + title=f"All inventories (`{len(self.base_urls)}` total)", + colour=discord.Colour.blue() + ) + + lines = sorted(f"• [`{name}`]({url})" for name, url in self.base_urls.items()) + if self.base_urls: + await LinePaginator.paginate(lines, ctx, inventory_embed, max_size=400, empty=False) + + else: + inventory_embed.description = "Hmmm, seems like there's nothing here yet." + await ctx.send(embed=inventory_embed) + + else: + symbol = symbol_name.strip("`") + async with ctx.typing(): + doc_embed = await self.create_symbol_embed(symbol) + + if doc_embed is None: + error_message = await send_denial(ctx, "No documentation found for the requested symbol.") + await wait_for_deletion(error_message, (ctx.author.id,), timeout=NOT_FOUND_DELETE_DELAY) + with suppress(discord.NotFound): + await ctx.message.delete() + with suppress(discord.NotFound): + await error_message.delete() + else: + msg = await ctx.send(embed=doc_embed) + await wait_for_deletion(msg, (ctx.author.id,)) + + @docs_group.command(name="setdoc", aliases=("s",)) + @commands.has_any_role(*MODERATION_ROLES) + @lock(NAMESPACE, COMMAND_LOCK_SINGLETON, raise_error=True) + async def set_command( + self, + ctx: commands.Context, + package_name: PackageName, + base_url: ValidURL, + inventory: Inventory, + ) -> None: + """ + Adds a new documentation metadata object to the site's database. + + The database will update the object, should an existing item with the specified `package_name` already exist. + + Example: + !docs setdoc \ + python \ + https://docs.python.org/3/ \ + https://docs.python.org/3/objects.inv + """ + if not base_url.endswith("/"): + raise commands.BadArgument("The base url must end with a slash.") + inventory_url, inventory_dict = inventory + body = { + "package": package_name, + "base_url": base_url, + "inventory_url": inventory_url + } + await self.bot.api_client.post("bot/documentation-links", json=body) + + log.info( + f"User @{ctx.author} ({ctx.author.id}) added a new documentation package:\n" + + "\n".join(f"{key}: {value}" for key, value in body.items()) + ) + + self.update_single(package_name, base_url, inventory_dict) + await ctx.send(f"Added the package `{package_name}` to the database and updated the inventories.") + + @docs_group.command(name="deletedoc", aliases=("removedoc", "rm", "d")) + @commands.has_any_role(*MODERATION_ROLES) + @lock(NAMESPACE, COMMAND_LOCK_SINGLETON, raise_error=True) + async def delete_command(self, ctx: commands.Context, package_name: PackageName) -> None: + """ + Removes the specified package from the database. + + Example: + !docs deletedoc aiohttp + """ + await self.bot.api_client.delete(f"bot/documentation-links/{package_name}") + + async with ctx.typing(): + await self.refresh_inventories() + await doc_cache.delete(package_name) + await ctx.send(f"Successfully deleted `{package_name}` and refreshed the inventories.") + + @docs_group.command(name="refreshdoc", aliases=("rfsh", "r")) + @commands.has_any_role(*MODERATION_ROLES) + @lock(NAMESPACE, COMMAND_LOCK_SINGLETON, raise_error=True) + async def refresh_command(self, ctx: commands.Context) -> None: + """Refresh inventories and show the difference.""" + old_inventories = set(self.base_urls) + with ctx.typing(): + await self.refresh_inventories() + new_inventories = set(self.base_urls) + + if added := ", ".join(new_inventories - old_inventories): + added = "+ " + added + + if removed := ", ".join(old_inventories - new_inventories): + removed = "- " + removed + + embed = discord.Embed( + title="Inventories refreshed", + description=f"```diff\n{added}\n{removed}```" if added or removed else "" + ) + await ctx.send(embed=embed) + + @docs_group.command(name="cleardoccache", aliases=("deletedoccache",)) + @commands.has_any_role(*MODERATION_ROLES) + async def clear_cache_command( + self, + ctx: commands.Context, + package_name: Union[PackageName, allowed_strings("*")] # noqa: F722 + ) -> None: + """Clear the persistent redis cache for `package`.""" + if await doc_cache.delete(package_name): + await ctx.send(f"Successfully cleared the cache for `{package_name}`.") + else: + await ctx.send("No keys matching the package found.") + + def cog_unload(self) -> None: + """Clear scheduled inventories, queued symbols and cleanup task on cog unload.""" + self.inventory_scheduler.cancel_all() + self.init_refresh_task.cancel() + asyncio.create_task(self.item_fetcher.clear(), name="DocCog.item_fetcher unload clear") diff --git a/bot/exts/info/doc/_html.py b/bot/exts/info/doc/_html.py new file mode 100644 index 000000000..94efd81b7 --- /dev/null +++ b/bot/exts/info/doc/_html.py @@ -0,0 +1,136 @@ +import logging +import re +from functools import partial +from typing import Callable, Container, Iterable, List, Union + +from bs4 import BeautifulSoup +from bs4.element import NavigableString, PageElement, SoupStrainer, Tag + +from . import MAX_SIGNATURE_AMOUNT + +log = logging.getLogger(__name__) + +_UNWANTED_SIGNATURE_SYMBOLS_RE = re.compile(r"\[source]|\\\\|¶") +_SEARCH_END_TAG_ATTRS = ( + "data", + "function", + "class", + "exception", + "seealso", + "section", + "rubric", + "sphinxsidebar", +) + + +class Strainer(SoupStrainer): + """Subclass of SoupStrainer to allow matching of both `Tag`s and `NavigableString`s.""" + + def __init__(self, *, include_strings: bool, **kwargs): + self.include_strings = include_strings + passed_text = kwargs.pop("text", None) + if passed_text is not None: + log.warning("`text` is not a supported kwarg in the custom strainer.") + super().__init__(**kwargs) + + Markup = Union[PageElement, List["Markup"]] + + def search(self, markup: Markup) -> Union[PageElement, str]: + """Extend default SoupStrainer behaviour to allow matching both `Tag`s` and `NavigableString`s.""" + if isinstance(markup, str): + # Let everything through the text filter if we're including strings and tags. + if not self.name and not self.attrs and self.include_strings: + return markup + else: + return super().search(markup) + + +def _find_elements_until_tag( + start_element: PageElement, + end_tag_filter: Union[Container[str], Callable[[Tag], bool]], + *, + func: Callable, + include_strings: bool = False, + limit: int = None, +) -> List[Union[Tag, NavigableString]]: + """ + Get all elements up to `limit` or until a tag matching `end_tag_filter` is found. + + `end_tag_filter` can be either a container of string names to check against, + or a filtering callable that's applied to tags. + + When `include_strings` is True, `NavigableString`s from the document will be included in the result along `Tag`s. + + `func` takes in a BeautifulSoup unbound method for finding multiple elements, such as `BeautifulSoup.find_all`. + The method is then iterated over and all elements until the matching tag or the limit are added to the return list. + """ + use_container_filter = not callable(end_tag_filter) + elements = [] + + for element in func(start_element, name=Strainer(include_strings=include_strings), limit=limit): + if isinstance(element, Tag): + if use_container_filter: + if element.name in end_tag_filter: + break + elif end_tag_filter(element): + break + elements.append(element) + + return elements + + +_find_next_children_until_tag = partial(_find_elements_until_tag, func=partial(BeautifulSoup.find_all, recursive=False)) +_find_recursive_children_until_tag = partial(_find_elements_until_tag, func=BeautifulSoup.find_all) +_find_next_siblings_until_tag = partial(_find_elements_until_tag, func=BeautifulSoup.find_next_siblings) +_find_previous_siblings_until_tag = partial(_find_elements_until_tag, func=BeautifulSoup.find_previous_siblings) + + +def _class_filter_factory(class_names: Iterable[str]) -> Callable[[Tag], bool]: + """Create callable that returns True when the passed in tag's class is in `class_names` or when it's a table.""" + def match_tag(tag: Tag) -> bool: + for attr in class_names: + if attr in tag.get("class", ()): + return True + return tag.name == "table" + + return match_tag + + +def get_general_description(start_element: Tag) -> List[Union[Tag, NavigableString]]: + """ + Get page content to a table or a tag with its class in `SEARCH_END_TAG_ATTRS`. + + A headerlink tag is attempted to be found to skip repeating the symbol information in the description. + If it's found it's used as the tag to start the search from instead of the `start_element`. + """ + child_tags = _find_recursive_children_until_tag(start_element, _class_filter_factory(["section"]), limit=100) + header = next(filter(_class_filter_factory(["headerlink"]), child_tags), None) + start_tag = header.parent if header is not None else start_element + return _find_next_siblings_until_tag(start_tag, _class_filter_factory(_SEARCH_END_TAG_ATTRS), include_strings=True) + + +def get_dd_description(symbol: PageElement) -> List[Union[Tag, NavigableString]]: + """Get the contents of the next dd tag, up to a dt or a dl tag.""" + description_tag = symbol.find_next("dd") + return _find_next_children_until_tag(description_tag, ("dt", "dl"), include_strings=True) + + +def get_signatures(start_signature: PageElement) -> List[str]: + """ + Collect up to `_MAX_SIGNATURE_AMOUNT` signatures from dt tags around the `start_signature` dt tag. + + First the signatures under the `start_signature` are included; + if less than 2 are found, tags above the start signature are added to the result if any are present. + """ + signatures = [] + for element in ( + *reversed(_find_previous_siblings_until_tag(start_signature, ("dd",), limit=2)), + start_signature, + *_find_next_siblings_until_tag(start_signature, ("dd",), limit=2), + )[-MAX_SIGNATURE_AMOUNT:]: + signature = _UNWANTED_SIGNATURE_SYMBOLS_RE.sub("", element.text) + + if signature: + signatures.append(signature) + + return signatures diff --git a/bot/exts/info/doc/_inventory_parser.py b/bot/exts/info/doc/_inventory_parser.py new file mode 100644 index 000000000..80d5841a0 --- /dev/null +++ b/bot/exts/info/doc/_inventory_parser.py @@ -0,0 +1,126 @@ +import logging +import re +import zlib +from collections import defaultdict +from typing import AsyncIterator, DefaultDict, List, Optional, Tuple + +import aiohttp + +import bot + +log = logging.getLogger(__name__) + +FAILED_REQUEST_ATTEMPTS = 3 +_V2_LINE_RE = re.compile(r'(?x)(.+?)\s+(\S*:\S*)\s+(-?\d+)\s+?(\S*)\s+(.*)') + +InventoryDict = DefaultDict[str, List[Tuple[str, str]]] + + +class ZlibStreamReader: + """Class used for decoding zlib data of a stream line by line.""" + + READ_CHUNK_SIZE = 16 * 1024 + + def __init__(self, stream: aiohttp.StreamReader) -> None: + self.stream = stream + + async def _read_compressed_chunks(self) -> AsyncIterator[bytes]: + """Read zlib data in `READ_CHUNK_SIZE` sized chunks and decompress.""" + decompressor = zlib.decompressobj() + async for chunk in self.stream.iter_chunked(self.READ_CHUNK_SIZE): + yield decompressor.decompress(chunk) + + yield decompressor.flush() + + async def __aiter__(self) -> AsyncIterator[str]: + """Yield lines of decompressed text.""" + buf = b'' + async for chunk in self._read_compressed_chunks(): + buf += chunk + pos = buf.find(b'\n') + while pos != -1: + yield buf[:pos].decode() + buf = buf[pos + 1:] + pos = buf.find(b'\n') + + +async def _load_v1(stream: aiohttp.StreamReader) -> InventoryDict: + invdata = defaultdict(list) + + async for line in stream: + name, type_, location = line.decode().rstrip().split(maxsplit=2) + # version 1 did not add anchors to the location + if type_ == "mod": + type_ = "py:module" + location += "#module-" + name + else: + type_ = "py:" + type_ + location += "#" + name + invdata[type_].append((name, location)) + return invdata + + +async def _load_v2(stream: aiohttp.StreamReader) -> InventoryDict: + invdata = defaultdict(list) + + async for line in ZlibStreamReader(stream): + m = _V2_LINE_RE.match(line.rstrip()) + name, type_, _prio, location, _dispname = m.groups() # ignore the parsed items we don't need + if location.endswith("$"): + location = location[:-1] + name + + invdata[type_].append((name, location)) + return invdata + + +async def _fetch_inventory(url: str) -> InventoryDict: + """Fetch, parse and return an intersphinx inventory file from an url.""" + timeout = aiohttp.ClientTimeout(sock_connect=5, sock_read=5) + async with bot.instance.http_session.get(url, timeout=timeout, raise_for_status=True) as response: + stream = response.content + + inventory_header = (await stream.readline()).decode().rstrip() + inventory_version = int(inventory_header[-1:]) + await stream.readline() # skip project name + await stream.readline() # skip project version + + if inventory_version == 1: + return await _load_v1(stream) + + elif inventory_version == 2: + if b"zlib" not in await stream.readline(): + raise ValueError(f"Invalid inventory file at url {url}.") + return await _load_v2(stream) + + raise ValueError(f"Invalid inventory file at url {url}.") + + +async def fetch_inventory(url: str) -> Optional[InventoryDict]: + """ + Get an inventory dict from `url`, retrying `FAILED_REQUEST_ATTEMPTS` times on errors. + + `url` should point at a valid sphinx objects.inv inventory file, which will be parsed into the + inventory dict in the format of {"domain:role": [("symbol_name", "relative_url_to_symbol"), ...], ...} + """ + for attempt in range(1, FAILED_REQUEST_ATTEMPTS+1): + try: + inventory = await _fetch_inventory(url) + except aiohttp.ClientConnectorError: + log.warning( + f"Failed to connect to inventory url at {url}; " + f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." + ) + except aiohttp.ClientError: + log.error( + f"Failed to get inventory from {url}; " + f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." + ) + except Exception: + log.exception( + f"An unexpected error has occurred during fetching of {url}; " + f"trying again ({attempt}/{FAILED_REQUEST_ATTEMPTS})." + ) + else: + return inventory + + return None diff --git a/bot/exts/info/doc/_markdown.py b/bot/exts/info/doc/_markdown.py new file mode 100644 index 000000000..1b7d8232b --- /dev/null +++ b/bot/exts/info/doc/_markdown.py @@ -0,0 +1,58 @@ +from urllib.parse import urljoin + +from bs4.element import PageElement +from markdownify import MarkdownConverter + + +class DocMarkdownConverter(MarkdownConverter): + """Subclass markdownify's MarkdownCoverter to provide custom conversion methods.""" + + def __init__(self, *, page_url: str, **options): + super().__init__(**options) + self.page_url = page_url + + def convert_li(self, el: PageElement, text: str, convert_as_inline: bool) -> str: + """Fix markdownify's erroneous indexing in ol tags.""" + parent = el.parent + if parent is not None and parent.name == "ol": + li_tags = parent.find_all("li") + bullet = f"{li_tags.index(el)+1}." + else: + depth = -1 + while el: + if el.name == "ul": + depth += 1 + el = el.parent + bullets = self.options["bullets"] + bullet = bullets[depth % len(bullets)] + return f"{bullet} {text}\n" + + def convert_hn(self, _n: int, el: PageElement, text: str, convert_as_inline: bool) -> str: + """Convert h tags to bold text with ** instead of adding #.""" + if convert_as_inline: + return text + return f"**{text}**\n\n" + + def convert_code(self, el: PageElement, text: str, convert_as_inline: bool) -> str: + """Undo `markdownify`s underscore escaping.""" + return f"`{text}`".replace("\\", "") + + def convert_pre(self, el: PageElement, text: str, convert_as_inline: bool) -> str: + """Wrap any codeblocks in `py` for syntax highlighting.""" + code = "".join(el.strings) + return f"```py\n{code}```" + + def convert_a(self, el: PageElement, text: str, convert_as_inline: bool) -> str: + """Resolve relative URLs to `self.page_url`.""" + el["href"] = urljoin(self.page_url, el["href"]) + return super().convert_a(el, text, convert_as_inline) + + def convert_p(self, el: PageElement, text: str, convert_as_inline: bool) -> str: + """Include only one newline instead of two when the parent is a li tag.""" + if convert_as_inline: + return text + + parent = el.parent + if parent is not None and parent.name == "li": + return f"{text}\n" + return super().convert_p(el, text, convert_as_inline) diff --git a/bot/exts/info/doc/_parsing.py b/bot/exts/info/doc/_parsing.py new file mode 100644 index 000000000..bf840b96f --- /dev/null +++ b/bot/exts/info/doc/_parsing.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +import logging +import re +import string +import textwrap +from collections import namedtuple +from typing import Collection, Iterable, Iterator, List, Optional, TYPE_CHECKING, Union + +from bs4 import BeautifulSoup +from bs4.element import NavigableString, Tag + +from bot.utils.helpers import find_nth_occurrence +from . import MAX_SIGNATURE_AMOUNT +from ._html import get_dd_description, get_general_description, get_signatures +from ._markdown import DocMarkdownConverter +if TYPE_CHECKING: + from ._cog import DocItem + +log = logging.getLogger(__name__) + +_WHITESPACE_AFTER_NEWLINES_RE = re.compile(r"(?<=\n\n)(\s+)") +_PARAMETERS_RE = re.compile(r"\((.+)\)") + +_NO_SIGNATURE_GROUPS = { + "attribute", + "envvar", + "setting", + "tempaltefilter", + "templatetag", + "term", +} +_EMBED_CODE_BLOCK_LINE_LENGTH = 61 +# _MAX_SIGNATURE_AMOUNT code block wrapped lines with py syntax highlight +_MAX_SIGNATURES_LENGTH = (_EMBED_CODE_BLOCK_LINE_LENGTH + 8) * MAX_SIGNATURE_AMOUNT +# Maximum embed description length - signatures on top +_MAX_DESCRIPTION_LENGTH = 2048 - _MAX_SIGNATURES_LENGTH +_TRUNCATE_STRIP_CHARACTERS = "!?:;." + string.whitespace + +BracketPair = namedtuple("BracketPair", ["opening_bracket", "closing_bracket"]) +_BRACKET_PAIRS = { + "{": BracketPair("{", "}"), + "(": BracketPair("(", ")"), + "[": BracketPair("[", "]"), + "<": BracketPair("<", ">"), +} + + +def _split_parameters(parameters_string: str) -> Iterator[str]: + """ + Split parameters of a signature into individual parameter strings on commas. + + Long string literals are not accounted for. + """ + last_split = 0 + depth = 0 + current_search: Optional[BracketPair] = None + + enumerated_string = enumerate(parameters_string) + for index, character in enumerated_string: + if character in {"'", '"'}: + # Skip everything inside of strings, regardless of the depth. + quote_character = character # The closing quote must equal the opening quote. + preceding_backslashes = 0 + for _, character in enumerated_string: + # If an odd number of backslashes precedes the quote, it was escaped. + if character == quote_character and not preceding_backslashes % 2: + break + if character == "\\": + preceding_backslashes += 1 + else: + preceding_backslashes = 0 + + elif current_search is None: + if (current_search := _BRACKET_PAIRS.get(character)) is not None: + depth = 1 + elif character == ",": + yield parameters_string[last_split:index] + last_split = index + 1 + + else: + if character == current_search.opening_bracket: + depth += 1 + + elif character == current_search.closing_bracket: + depth -= 1 + if depth == 0: + current_search = None + + yield parameters_string[last_split:] + + +def _truncate_signatures(signatures: Collection[str]) -> Union[List[str], Collection[str]]: + """ + Truncate passed signatures to not exceed `_MAX_SIGNATURES_LENGTH`. + + If the signatures need to be truncated, parameters are collapsed until they fit withing the limit. + Individual signatures can consist of max 1, 2, ..., `_MAX_SIGNATURE_AMOUNT` lines of text, + inversely proportional to the amount of signatures. + A maximum of `_MAX_SIGNATURE_AMOUNT` signatures is assumed to be passed. + """ + if sum(len(signature) for signature in signatures) <= _MAX_SIGNATURES_LENGTH: + # Total length of signatures is under the length limit; no truncation needed. + return signatures + + max_signature_length = _EMBED_CODE_BLOCK_LINE_LENGTH * (MAX_SIGNATURE_AMOUNT + 1 - len(signatures)) + formatted_signatures = [] + for signature in signatures: + signature = signature.strip() + if len(signature) > max_signature_length: + if (parameters_match := _PARAMETERS_RE.search(signature)) is None: + # The signature has no parameters or the regex failed; perform a simple truncation of the text. + formatted_signatures.append(textwrap.shorten(signature, max_signature_length, placeholder="...")) + continue + + truncated_signature = [] + parameters_string = parameters_match[1] + running_length = len(signature) - len(parameters_string) + for parameter in _split_parameters(parameters_string): + # Check if including this parameter would still be within the maximum length. + if (len(parameter) + running_length) <= max_signature_length - 5: # account for comma and placeholder + truncated_signature.append(parameter) + running_length += len(parameter) + 1 + else: + # There's no more room for this parameter. Truncate the parameter list and put it in the signature. + truncated_signature.append(" ...") + formatted_signatures.append(signature.replace(parameters_string, ",".join(truncated_signature))) + break + else: + # The current signature is under the length limit; no truncation needed. + formatted_signatures.append(signature) + + return formatted_signatures + + +def _get_truncated_description( + elements: Iterable[Union[Tag, NavigableString]], + markdown_converter: DocMarkdownConverter, + max_length: int, + max_lines: int, +) -> str: + """ + Truncate the Markdown from `elements` to be at most `max_length` characters when rendered or `max_lines` newlines. + + `max_length` limits the length of the rendered characters in the string, + with the real string length limited to `_MAX_DESCRIPTION_LENGTH` to accommodate discord length limits. + """ + result = "" + markdown_element_ends = [] # Stores indices into `result` which point to the end boundary of each Markdown element. + rendered_length = 0 + + tag_end_index = 0 + for element in elements: + is_tag = isinstance(element, Tag) + element_length = len(element.text) if is_tag else len(element) + + if rendered_length + element_length < max_length: + if is_tag: + element_markdown = markdown_converter.process_tag(element, convert_as_inline=False) + else: + element_markdown = markdown_converter.process_text(element) + + rendered_length += element_length + tag_end_index += len(element_markdown) + + if not element_markdown.isspace(): + markdown_element_ends.append(tag_end_index) + result += element_markdown + else: + break + + if not markdown_element_ends: + return "" + + # Determine the "hard" truncation index. Account for the ellipsis placeholder for the max length. + newline_truncate_index = find_nth_occurrence(result, "\n", max_lines) + if newline_truncate_index is not None and newline_truncate_index < _MAX_DESCRIPTION_LENGTH - 3: + # Truncate based on maximum lines if there are more than the maximum number of lines. + truncate_index = newline_truncate_index + else: + # There are less than the maximum number of lines; truncate based on the max char length. + truncate_index = _MAX_DESCRIPTION_LENGTH - 3 + + # Nothing needs to be truncated if the last element ends before the truncation index. + if truncate_index >= markdown_element_ends[-1]: + return result + + # Determine the actual truncation index. + possible_truncation_indices = [cut for cut in markdown_element_ends if cut < truncate_index] + if not possible_truncation_indices: + # In case there is no Markdown element ending before the truncation index, try to find a good cutoff point. + force_truncated = result[:truncate_index] + # If there is an incomplete codeblock, cut it out. + if force_truncated.count("```") % 2: + force_truncated = force_truncated[:force_truncated.rfind("```")] + # Search for substrings to truncate at, with decreasing desirability. + for string_ in ("\n\n", "\n", ". ", ", ", ",", " "): + cutoff = force_truncated.rfind(string_) + + if cutoff != -1: + truncated_result = force_truncated[:cutoff] + break + else: + truncated_result = force_truncated + + else: + # Truncate at the last Markdown element that comes before the truncation index. + markdown_truncate_index = possible_truncation_indices[-1] + truncated_result = result[:markdown_truncate_index] + + return truncated_result.strip(_TRUNCATE_STRIP_CHARACTERS) + "..." + + +def _create_markdown(signatures: Optional[List[str]], description: Iterable[Tag], url: str) -> str: + """ + Create a Markdown string with the signatures at the top, and the converted html description below them. + + The signatures are wrapped in python codeblocks, separated from the description by a newline. + The result Markdown string is max 750 rendered characters for the description with signatures at the start. + """ + description = _get_truncated_description( + description, + markdown_converter=DocMarkdownConverter(bullets="•", page_url=url), + max_length=750, + max_lines=13 + ) + description = _WHITESPACE_AFTER_NEWLINES_RE.sub("", description) + if signatures is not None: + signature = "".join(f"```py\n{signature}```" for signature in _truncate_signatures(signatures)) + return f"{signature}\n{description}" + else: + return description + + +def get_symbol_markdown(soup: BeautifulSoup, symbol_data: DocItem) -> Optional[str]: + """ + Return parsed Markdown of the passed item using the passed in soup, truncated to fit within a discord message. + + The method of parsing and what information gets included depends on the symbol's group. + """ + symbol_heading = soup.find(id=symbol_data.symbol_id) + if symbol_heading is None: + return None + signature = None + # Modules, doc pages and labels don't point to description list tags but to tags like divs, + # no special parsing can be done so we only try to include what's under them. + if symbol_heading.name != "dt": + description = get_general_description(symbol_heading) + + elif symbol_data.group in _NO_SIGNATURE_GROUPS: + description = get_dd_description(symbol_heading) + + else: + signature = get_signatures(symbol_heading) + description = get_dd_description(symbol_heading) + return _create_markdown(signature, description, symbol_data.url).replace("¶", "").strip() diff --git a/bot/exts/info/doc/_redis_cache.py b/bot/exts/info/doc/_redis_cache.py new file mode 100644 index 000000000..ad764816f --- /dev/null +++ b/bot/exts/info/doc/_redis_cache.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import datetime +from typing import Optional, TYPE_CHECKING + +from async_rediscache.types.base import RedisObject, namespace_lock +if TYPE_CHECKING: + from ._cog import DocItem + +WEEK_SECONDS = datetime.timedelta(weeks=1).total_seconds() + + +class DocRedisCache(RedisObject): + """Interface for redis functionality needed by the Doc cog.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._set_expires = set() + + @namespace_lock + async def set(self, item: DocItem, value: str) -> None: + """ + Set the Markdown `value` for the symbol `item`. + + All keys from a single page are stored together, expiring a week after the first set. + """ + url_key = remove_suffix(item.relative_url_path, ".html") + redis_key = f"{self.namespace}:{item.package}:{url_key}" + needs_expire = False + + with await self._get_pool_connection() as connection: + if redis_key not in self._set_expires: + # An expire is only set if the key didn't exist before. + # If this is the first time setting values for this key check if it exists and add it to + # `_set_expires` to prevent redundant checks for subsequent uses with items from the same page. + self._set_expires.add(redis_key) + needs_expire = not await connection.exists(redis_key) + + await connection.hset(redis_key, item.symbol_id, value) + if needs_expire: + await connection.expire(redis_key, WEEK_SECONDS) + + @namespace_lock + async def get(self, item: DocItem) -> Optional[str]: + """Return the Markdown content of the symbol `item` if it exists.""" + url_key = remove_suffix(item.relative_url_path, ".html") + + with await self._get_pool_connection() as connection: + return await connection.hget(f"{self.namespace}:{item.package}:{url_key}", item.symbol_id, encoding="utf8") + + @namespace_lock + async def delete(self, package: str) -> bool: + """Remove all values for `package`; return True if at least one key was deleted, False otherwise.""" + with await self._get_pool_connection() as connection: + package_keys = [ + package_key async for package_key in connection.iscan(match=f"{self.namespace}:{package}:*") + ] + if package_keys: + await connection.delete(*package_keys) + return True + return False + + +def remove_suffix(string: str, suffix: str) -> str: + """Remove `suffix` from end of `string`.""" + # TODO replace usages with str.removesuffix on 3.9 + if string.endswith(suffix): + return string[:-len(suffix)] + else: + return string diff --git a/bot/exts/info/source.py b/bot/exts/info/source.py index 49e74f204..78a6f68ac 100644 --- a/bot/exts/info/source.py +++ b/bot/exts/info/source.py @@ -68,7 +68,8 @@ class BotSource(commands.Cog): Raise BadArgument if `source_item` is a dynamically-created object (e.g. via internal eval). """ if isinstance(source_item, commands.Command): - src = source_item.callback.__code__ + source_item = inspect.unwrap(source_item.callback) + src = source_item.__code__ filename = src.co_filename elif isinstance(source_item, str): tags_cog = self.bot.get_cog("Tags") diff --git a/bot/utils/function.py b/bot/utils/function.py index 3ab32fe3c..9bc44e753 100644 --- a/bot/utils/function.py +++ b/bot/utils/function.py @@ -1,14 +1,23 @@ """Utilities for interaction with functions.""" +import functools import inspect +import logging +import types import typing as t +log = logging.getLogger(__name__) + Argument = t.Union[int, str] BoundArgs = t.OrderedDict[str, t.Any] Decorator = t.Callable[[t.Callable], t.Callable] ArgValGetter = t.Callable[[BoundArgs], t.Any] +class GlobalNameConflictError(Exception): + """Raised when there's a conflict between the globals used to resolve annotations of wrapped and its wrapper.""" + + def get_arg_value(name_or_pos: Argument, arguments: BoundArgs) -> t.Any: """ Return a value from `arguments` based on a name or position. @@ -73,3 +82,66 @@ def get_bound_args(func: t.Callable, args: t.Tuple, kwargs: t.Dict[str, t.Any]) bound_args.apply_defaults() return bound_args.arguments + + +def update_wrapper_globals( + wrapper: types.FunctionType, + wrapped: types.FunctionType, + *, + ignored_conflict_names: t.Set[str] = frozenset(), +) -> types.FunctionType: + """ + Update globals of `wrapper` with the globals from `wrapped`. + + For forwardrefs in command annotations discordpy uses the __global__ attribute of the function + to resolve their values, with decorators that replace the function this breaks because they have + their own globals. + + This function creates a new function functionally identical to `wrapper`, which has the globals replaced with + a merge of `wrapped`s globals and the `wrapper`s globals. + + An exception will be raised in case `wrapper` and `wrapped` share a global name that is used by + `wrapped`'s typehints and is not in `ignored_conflict_names`, + as this can cause incorrect objects being used by discordpy's converters. + """ + annotation_global_names = ( + ann.split(".", maxsplit=1)[0] for ann in wrapped.__annotations__.values() if isinstance(ann, str) + ) + # Conflicting globals from both functions' modules that are also used in the wrapper and in wrapped's annotations. + shared_globals = set(wrapper.__code__.co_names) & set(annotation_global_names) + shared_globals &= set(wrapped.__globals__) & set(wrapper.__globals__) - ignored_conflict_names + if shared_globals: + raise GlobalNameConflictError( + f"wrapper and the wrapped function share the following " + f"global names used by annotations: {', '.join(shared_globals)}. Resolve the conflicts or add " + f"the name to the `ignored_conflict_names` set to suppress this error if this is intentional." + ) + + new_globals = wrapper.__globals__.copy() + new_globals.update((k, v) for k, v in wrapped.__globals__.items() if k not in wrapper.__code__.co_names) + return types.FunctionType( + code=wrapper.__code__, + globals=new_globals, + name=wrapper.__name__, + argdefs=wrapper.__defaults__, + closure=wrapper.__closure__, + ) + + +def command_wraps( + wrapped: types.FunctionType, + assigned: t.Sequence[str] = functools.WRAPPER_ASSIGNMENTS, + updated: t.Sequence[str] = functools.WRAPPER_UPDATES, + *, + ignored_conflict_names: t.Set[str] = frozenset(), +) -> t.Callable[[types.FunctionType], types.FunctionType]: + """Update the decorated function to look like `wrapped` and update globals for discordpy forwardref evaluation.""" + def decorator(wrapper: types.FunctionType) -> types.FunctionType: + return functools.update_wrapper( + update_wrapper_globals(wrapper, wrapped, ignored_conflict_names=ignored_conflict_names), + wrapped, + assigned, + updated, + ) + + return decorator diff --git a/bot/utils/lock.py b/bot/utils/lock.py index e44776340..ec6f92cd4 100644 --- a/bot/utils/lock.py +++ b/bot/utils/lock.py @@ -1,13 +1,15 @@ import asyncio import inspect import logging +import types from collections import defaultdict -from functools import partial, wraps +from functools import partial from typing import Any, Awaitable, Callable, Hashable, Union from weakref import WeakValueDictionary from bot.errors import LockedResourceError from bot.utils import function +from bot.utils.function import command_wraps log = logging.getLogger(__name__) __lock_dicts = defaultdict(WeakValueDictionary) @@ -17,6 +19,35 @@ _IdCallable = Callable[[function.BoundArgs], _IdCallableReturn] ResourceId = Union[Hashable, _IdCallable] +class SharedEvent: + """ + Context manager managing an internal event exposed through the wait coro. + + While any code is executing in this context manager, the underlying event will not be set; + when all of the holders finish the event will be set. + """ + + def __init__(self): + self._active_count = 0 + self._event = asyncio.Event() + self._event.set() + + def __enter__(self): + """Increment the count of the active holders and clear the internal event.""" + self._active_count += 1 + self._event.clear() + + def __exit__(self, _exc_type, _exc_val, _exc_tb): # noqa: ANN001 + """Decrement the count of the active holders; if 0 is reached set the internal event.""" + self._active_count -= 1 + if not self._active_count: + self._event.set() + + async def wait(self) -> None: + """Wait for all active holders to exit.""" + await self._event.wait() + + def lock( namespace: Hashable, resource_id: ResourceId, @@ -41,10 +72,10 @@ def lock( If decorating a command, this decorator must go before (below) the `command` decorator. """ - def decorator(func: Callable) -> Callable: + def decorator(func: types.FunctionType) -> types.FunctionType: name = func.__name__ - @wraps(func) + @command_wraps(func) async def wrapper(*args, **kwargs) -> Any: log.trace(f"{name}: mutually exclusive decorator called") diff --git a/bot/utils/messages.py b/bot/utils/messages.py index 077dd9569..0bcaed43d 100644 --- a/bot/utils/messages.py +++ b/bot/utils/messages.py @@ -141,14 +141,14 @@ def sub_clyde(username: Optional[str]) -> Optional[str]: return username # Empty string or None -async def send_denial(ctx: Context, reason: str) -> None: +async def send_denial(ctx: Context, reason: str) -> discord.Message: """Send an embed denying the user with the given reason.""" embed = discord.Embed() embed.colour = discord.Colour.red() embed.title = random.choice(NEGATIVE_REPLIES) embed.description = reason - await ctx.send(embed=embed) + return await ctx.send(embed=embed) def format_user(user: discord.abc.User) -> str: diff --git a/tests/bot/exts/info/doc/__init__.py b/tests/bot/exts/info/doc/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/bot/exts/info/doc/__init__.py diff --git a/tests/bot/exts/info/doc/test_parsing.py b/tests/bot/exts/info/doc/test_parsing.py new file mode 100644 index 000000000..1663d8491 --- /dev/null +++ b/tests/bot/exts/info/doc/test_parsing.py @@ -0,0 +1,66 @@ +from unittest import TestCase + +from bot.exts.info.doc import _parsing as parsing + + +class SignatureSplitter(TestCase): + + def test_basic_split(self): + test_cases = ( + ("0,0,0", ["0", "0", "0"]), + ("0,a=0,a=0", ["0", "a=0", "a=0"]), + ) + self._run_tests(test_cases) + + def test_commas_ignored_in_brackets(self): + test_cases = ( + ("0,[0,0],0,[0,0],0", ["0", "[0,0]", "0", "[0,0]", "0"]), + ("(0,),0,(0,(0,),0),0", ["(0,)", "0", "(0,(0,),0)", "0"]), + ) + self._run_tests(test_cases) + + def test_mixed_brackets(self): + tests_cases = ( + ("[0,{0},0],0,{0:0},0", ["[0,{0},0]", "0", "{0:0}", "0"]), + ("([0],0,0),0,(0,0),0", ["([0],0,0)", "0", "(0,0)", "0"]), + ("([(0,),(0,)],0),0", ["([(0,),(0,)],0)", "0"]), + ) + self._run_tests(tests_cases) + + def test_string_contents_ignored(self): + test_cases = ( + ("'0,0',0,',',0", ["'0,0'", "0", "','", "0"]), + ("0,[']',0],0", ["0", "[']',0]", "0"]), + ("{0,0,'}}',0,'{'},0", ["{0,0,'}}',0,'{'}", "0"]), + ) + self._run_tests(test_cases) + + def test_mixed_quotes(self): + test_cases = ( + ("\"0',0',\",'0,0',0", ["\"0',0',\"", "'0,0'", "0"]), + ("\",',\",'\",',0", ["\",',\"", "'\",'", "0"]), + ) + self._run_tests(test_cases) + + def test_quote_escaped(self): + test_cases = ( + (r"'\',','\\',0", [r"'\','", r"'\\'", "0"]), + (r"'0\',0\\\'\\',0", [r"'0\',0\\\'\\'", "0"]), + ) + self._run_tests(test_cases) + + def test_real_signatures(self): + test_cases = ( + ("start, stop[, step]", ["start", " stop[, step]"]), + ("object=b'', encoding='utf-8', errors='strict'", ["object=b''", " encoding='utf-8'", " errors='strict'"]), + ( + "typename, field_names, *, rename=False, defaults=None, module=None", + ["typename", " field_names", " *", " rename=False", " defaults=None", " module=None"] + ), + ) + self._run_tests(test_cases) + + def _run_tests(self, test_cases): + for input_string, expected_output in test_cases: + with self.subTest(input_string=input_string): + self.assertEqual(list(parsing._split_parameters(input_string)), expected_output) diff --git a/tests/bot/test_converters.py b/tests/bot/test_converters.py index c42111f3f..4af84dde5 100644 --- a/tests/bot/test_converters.py +++ b/tests/bot/test_converters.py @@ -10,9 +10,9 @@ from bot.converters import ( Duration, HushDurationConverter, ISODateTime, + PackageName, TagContentConverter, TagNameConverter, - ValidPythonIdentifier, ) @@ -78,24 +78,23 @@ class ConverterTests(unittest.IsolatedAsyncioTestCase): with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): await TagNameConverter.convert(self.context, invalid_name) - async def test_valid_python_identifier_for_valid(self): - """ValidPythonIdentifier returns valid identifiers unchanged.""" - test_values = ('foo', 'lemon') + async def test_package_name_for_valid(self): + """PackageName returns valid package names unchanged.""" + test_values = ('foo', 'le_mon', 'num83r') for name in test_values: with self.subTest(identifier=name): - conversion = await ValidPythonIdentifier.convert(self.context, name) + conversion = await PackageName.convert(self.context, name) self.assertEqual(name, conversion) - async def test_valid_python_identifier_for_invalid(self): - """ValidPythonIdentifier raises the proper exception for invalid identifiers.""" - test_values = ('nested.stuff', '#####') + async def test_package_name_for_invalid(self): + """PackageName raises the proper exception for invalid package names.""" + test_values = ('text_with_a_dot.', 'UpperCaseName', 'dashed-name') for name in test_values: with self.subTest(identifier=name): - exception_message = f'`{name}` is not a valid Python identifier' - with self.assertRaisesRegex(BadArgument, re.escape(exception_message)): - await ValidPythonIdentifier.convert(self.context, name) + with self.assertRaises(BadArgument): + await PackageName.convert(self.context, name) async def test_duration_converter_for_valid(self): """Duration returns the correct `datetime` for valid duration strings.""" |