diff options
| -rw-r--r-- | Pipfile | 3 | ||||
| -rw-r--r-- | Pipfile.lock | 156 | ||||
| -rw-r--r-- | bot/__main__.py | 2 | ||||
| -rw-r--r-- | bot/cogs/doc.py | 490 | ||||
| -rw-r--r-- | bot/cogs/utils.py | 83 | ||||
| -rw-r--r-- | bot/converters.py | 56 | ||||
| -rw-r--r-- | config-default.yml | 1 |
7 files changed, 787 insertions, 4 deletions
@@ -12,6 +12,9 @@ aiodns = "*" logmatic-python = "*" aiohttp = "<2.3.0,>=2.0.0" websockets = ">=4.0,<5.0" +sphinx = "*" +markdownify = "*" +lxml = "*" pyyaml = "*" yarl = "==1.1.1" fuzzywuzzy = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 64652ae3b..9f4c3a1d4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1529809913b543e7fbbf6091c33369f96567ab1fc8d7d637df574d4ac9d8b050" + "sha256": "2475ab039e5ce97636b9a255dd7e21b67e7b66d832f341b4065876f7535b2d88" }, "pipfile-spec": 6, "requires": { @@ -51,6 +51,13 @@ "index": "pypi", "version": "==2.2.5" }, + "alabaster": { + "hashes": [ + "sha256:2eef172f44e8d301d25aff8068fddd65f767a3f04b5f15b0f4922f113aa1c732", + "sha256:37cdcb9e9954ed60912ebc1ca12a9d12178c26637abdf124e3cde2341c257fe0" + ], + "version": "==0.7.10" + }, "async-timeout": { "hashes": [ "sha256:474d4bc64cee20603e225eb1ece15e248962958b45a3648a9f5cc29e827a610c", @@ -58,6 +65,21 @@ ], "version": "==3.0.0" }, + "babel": { + "hashes": [ + "sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669", + "sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23" + ], + "version": "==2.6.0" + }, + "beautifulsoup4": { + "hashes": [ + "sha256:11a9a27b7d3bddc6d86f59fb76afb70e921a25ac2d6cc55b40d072bd68435a76", + "sha256:7015e76bf32f1f574636c4288399a6de66ce08fb7b2457f628a8d70c0fbabb11", + "sha256:808b6ac932dccb0a4126558f7dfdcf41710dd44a4ef497a0bb59a77f9f078e89" + ], + "version": "==4.6.0" + }, "certifi": { "hashes": [ "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7", @@ -76,6 +98,14 @@ "egg": "discord.py[voice]", "file": "https://github.com/Rapptz/discord.py/archive/rewrite.zip" }, + "docutils": { + "hashes": [ + "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", + "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", + "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" + ], + "version": "==0.14" + }, "dulwich": { "hashes": [ "sha256:c51e10c260543240e0806052af046e1a78b98cbe1ac1ef3880a78d2269e09da4" @@ -98,6 +128,20 @@ ], "version": "==2.6" }, + "imagesize": { + "hashes": [ + "sha256:3620cc0cadba3f7475f9940d22431fc4d407269f1be59ec9b8edcca26440cf18", + "sha256:5b326e4678b6925158ccc66a9fa3122b6106d7c876ee32d7de6ce59385b96315" + ], + "version": "==1.0.0" + }, + "jinja2": { + "hashes": [ + "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", + "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" + ], + "version": "==2.10" + }, "logmatic-python": { "hashes": [ "sha256:0c15ac9f5faa6a60059b28910db642c3dc7722948c3cc940923f8c9039604342" @@ -105,6 +149,53 @@ "index": "pypi", "version": "==0.1.7" }, + "lxml": { + "hashes": [ + "sha256:01c45df6d90497c20aa2a07789a41941f9a1029faa30bf725fc7f6d515b1afe9", + "sha256:0c9fef4f8d444e337df96c54544aeb85b7215b2ed7483bb6c35de97ac99f1bcd", + "sha256:0e3cd94c95d30ba9ca3cff40e9b2a14e1a10a4fd8131105b86c6b61648f57e4b", + "sha256:0e7996e9b46b4d8b4ac1c329a00e2d10edcd8380b95d2a676fccabf4c1dd0512", + "sha256:1858b1933d483ec5727549d3fe166eeb54229fbd6a9d3d7ea26d2c8a28048058", + "sha256:1b164bba1320b14905dcff77da10d5ce9c411ac4acc4fb4ed9a2a4d10fae38c9", + "sha256:1b46f37927fa6cd1f3fe34b54f1a23bd5bea1d905657289e08e1297069a1a597", + "sha256:231047b05907315ae9a9b6925751f9fd2c479cf7b100fff62485a25e382ca0d4", + "sha256:28f0c6652c1b130f1e576b60532f84b19379485eb8da6185c29bd8c9c9bc97bf", + "sha256:34d49d0f72dd82b9530322c48b70ac78cca0911275da741c3b1d2f3603c5f295", + "sha256:3682a17fbf72d56d7e46db2e80ca23850b79c28cfe75dcd9b82f58808f730909", + "sha256:3cf2830b9a6ad7f6e965fa53a768d4d2372a7856f20ffa6ce43d2fe9c0d34b19", + "sha256:5b653c9379ce29ce271fbe1010c5396670f018e78b643e21beefbb3dc6d291de", + "sha256:65a272821d5d8194358d6b46f3ca727fa56a6b63981606eac737c86d27309cdd", + "sha256:691f2cd97cf026c611df1ea5055755eec7f878f2d4f4330dc8686583de6fc5fd", + "sha256:6b6379495d3baacf7ed755ac68547c8dff6ce5d37bf370f0b7678888dc1283f9", + "sha256:75322a531504d4f383264391d89993a42e286da8821ddc5ac315e57305cb84f0", + "sha256:7f457cbda964257f443bac861d3a36732dcba8183149e7818ee2fb7c86901b94", + "sha256:7ff1fc76d8804e0f870c343a72007ff587090c218b0f92d8ee784ac2b6eaf5b9", + "sha256:8523fbde9c2216f3f2b950cb01ebe52e785eaa8a07ffeb456dd3576ca1b4fb9b", + "sha256:8f37627f16e026523fca326f1b5c9a43534862fede6c3e99c2ba6a776d75c1ab", + "sha256:a7182ea298cc3555ea56ffbb0748fe0d5e0d81451e2bc16d7f4645cd01b1ca70", + "sha256:abbd2fb4a5a04c11b5e04eb146659a0cf67bb237dd3d7ca3b9994d3a9f826e55", + "sha256:accc9f6b77bed0a6f267b4fae120f6008a951193d548cdbe9b61fc98a08b1cf8", + "sha256:bd88c8ce0d1504fdfd96a35911dd4f3edfb2e560d7cfdb5a3d09aa571ae5fbae", + "sha256:c557ad647facb3c0027a9d0af58853f905e85a0a2f04dcb73f8e665272fcdc3a", + "sha256:defabb7fbb99f9f7b3e0b24b286a46855caef4776495211b066e9e6592d12b04", + "sha256:e2629cdbcad82b83922a3488937632a4983ecc0fed3e5cfbf430d069382eeb9b" + ], + "index": "pypi", + "version": "==4.2.1" + }, + "markdownify": { + "hashes": [ + "sha256:28ce67d1888e4908faaab7b04d2193cda70ea4f902f156a21d0aaea55e63e0a1" + ], + "index": "pypi", + "version": "==0.4.1" + }, + "markupsafe": { + "hashes": [ + "sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665" + ], + "version": "==1.0" + }, "mpmath": { "hashes": [ "sha256:04d14803b6875fe6d69e6dccea87d5ae5599802e4b1df7997bddd2024001050c" @@ -130,6 +221,13 @@ "index": "pypi", "version": "==4.3.1" }, + "packaging": { + "hashes": [ + "sha256:e9215d2d2535d3ae866c3d6efc77d5b24a0192cce0ff20e42896cc0664f889c0", + "sha256:f019b770dd64e585a99714f1fd5e01c7a8f11b45635aa953fd41c689a657375b" + ], + "version": "==17.1" + }, "pika": { "hashes": [ "sha256:63131aaeec48a6c8f1db1fe657e1e74cf384c3927eb7d1725e31edae4220dea4", @@ -212,6 +310,25 @@ ], "version": "==2.3.0" }, + "pygments": { + "hashes": [ + "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d", + "sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc" + ], + "version": "==2.2.0" + }, + "pyparsing": { + "hashes": [ + "sha256:0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04", + "sha256:281683241b25fe9b80ec9d66017485f6deff1af5cde372469134b56ca8447a07", + "sha256:8f1e18d3fd36c6795bb7e02a39fd05c611ffc2596c1e0d995d34d67630426c18", + "sha256:9e8143a3e15c13713506886badd96ca4b579a87fbdf49e550dbfc057d6cb218e", + "sha256:b8b3117ed9bdf45e14dcc89345ce638ec7e0e29b2b579fa1ecf32ce45ebac8a5", + "sha256:e4d45427c6e20a59bf4f88c639dcc03ce30d193112047f94012102f235853a58", + "sha256:fee43f17a9c4087e7ed1605bd6df994c6173c1e977d7ade7b651292fab2bd010" + ], + "version": "==2.2.0" + }, "python-dateutil": { "hashes": [ "sha256:1adb80e7a782c12e52ef9a8182bebeb73f1d7e24e374397af06fb4956c8dc5c0", @@ -234,6 +351,13 @@ "index": "pypi", "version": "==0.12.0" }, + "pytz": { + "hashes": [ + "sha256:65ae0c8101309c45772196b21b74c46b2e5d11b6275c45d251b150d5da334555", + "sha256:c06425302f2cf668f1bba7a0a03f3c1d34d4ebeef2c72003da308b3947c7f749" + ], + "version": "==2018.4" + }, "pyyaml": { "hashes": [ "sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8", @@ -254,6 +378,13 @@ "index": "pypi", "version": "==3.12" }, + "requests": { + "hashes": [ + "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", + "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e" + ], + "version": "==2.18.4" + }, "shortuuid": { "hashes": [ "sha256:d08fd398f40f8baf87e15eef8355e92fa541bca4eb8465fefab7ee22f92711b9" @@ -267,6 +398,28 @@ ], "version": "==1.11.0" }, + "snowballstemmer": { + "hashes": [ + "sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128", + "sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89" + ], + "version": "==1.2.1" + }, + "sphinx": { + "hashes": [ + "sha256:85f7e32c8ef07f4ba5aeca728e0f7717bef0789fba8458b8d9c5c294cad134f3", + "sha256:d45480a229edf70d84ca9fae3784162b1bc75ee47e480ffe04a4b7f21a95d76d" + ], + "index": "pypi", + "version": "==1.7.5" + }, + "sphinxcontrib-websupport": { + "hashes": [ + "sha256:7a85961326aa3a400cd4ad3c816d70ed6f7c740acd7ce5d78cd0a67825072eb9", + "sha256:f4932e95869599b89bf4f80fc3989132d83c9faa5bf633e7b5e0c25dffb75da2" + ], + "version": "==1.0.1" + }, "sympy": { "hashes": [ "sha256:ac5b57691bc43919dcc21167660a57cc51797c28a4301a6144eff07b751216a4" @@ -485,7 +638,6 @@ "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e" ], - "index": "pypi", "version": "==2.18.4" }, "safety": { diff --git a/bot/__main__.py b/bot/__main__.py index 50027bf1b..b1e9c61fa 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -55,12 +55,14 @@ else: log.info("`CLICKUP_KEY` not set in the environment, not loading the ClickUp cog.") bot.load_extension("bot.cogs.deployment") +bot.load_extension("bot.cogs.doc") bot.load_extension("bot.cogs.eval") bot.load_extension("bot.cogs.fun") bot.load_extension("bot.cogs.hiphopify") bot.load_extension("bot.cogs.snakes") bot.load_extension("bot.cogs.tags") bot.load_extension("bot.cogs.verification") +bot.load_extension("bot.cogs.utils") bot.run(BotConfig.token) diff --git a/bot/cogs/doc.py b/bot/cogs/doc.py new file mode 100644 index 000000000..d84de0ba3 --- /dev/null +++ b/bot/cogs/doc.py @@ -0,0 +1,490 @@ +import asyncio +import functools +import logging +import random +import re +from collections import OrderedDict +from typing import Dict, List, Optional, Tuple + +import discord +from bs4 import BeautifulSoup +from discord.ext import commands +from markdownify import MarkdownConverter +from requests import ConnectionError +from sphinx.ext import intersphinx + +from bot.constants import ERROR_REPLIES, Keys, Roles, URLs +from bot.converters import ValidPythonIdentifier, ValidURL +from bot.decorators import with_role + + +log = logging.getLogger(__name__) +logging.getLogger('urllib3').setLevel(logging.WARNING) + + +UNWANTED_SIGNATURE_SYMBOLS = ('[source]', '¶') +WHITESPACE_AFTER_NEWLINES_RE = re.compile(r"(?<=\n\n)(\s+)") + + +def async_cache(max_size=128, arg_offset=0): + """ + LRU cache implementation for coroutines. + + :param max_size: + Specifies the maximum size the cache should have. + Once it exceeds the maximum size, keys are deleted in FIFO order. + :param arg_offset: + The offset that should be applied to the coroutine's arguments + when creating the cache key. Defaults to `0`. + """ + + # Assign the cache to the function itself so we can clear it from outside. + async_cache.cache = OrderedDict() + + def decorator(function): + @functools.wraps(function) + async def wrapper(*args): + key = ':'.join(args[arg_offset:]) + + value = async_cache.cache.get(key) + if value is None: + if len(async_cache.cache) > max_size: + async_cache.cache.popitem(last=False) + + async_cache.cache[key] = await function(*args) + return async_cache.cache[key] + return wrapper + return decorator + + +class DocMarkdownConverter(MarkdownConverter): + def convert_code(self, el, text): + """Undo `markdownify`s underscore escaping.""" + + return f"`{text}`".replace('\\', '') + + def convert_pre(self, el, text): + """Wrap any codeblocks in `py` for syntax highlighting.""" + + code = ''.join(el.strings) + return f"```py\n{code}```" + + +def markdownify(html): + return DocMarkdownConverter(bullets='•').convert(html) + + +class DummyObject(object): + """ + A dummy object which supports assigning anything, + which the builtin `object()` does not support normally. + """ + + +class SphinxConfiguration: + """Dummy configuration for use with intersphinx.""" + + config = DummyObject() + config.intersphinx_timeout = 3 + config.tls_verify = True + + +class InventoryURL(commands.Converter): + """ + Represents an Intersphinx inventory URL. + + This converter checks whether intersphinx + accepts the given inventory URL, and raises + `BadArgument` if that is not the case. + Otherwise, it simply passes through the given URL. + """ + + @staticmethod + async def convert(ctx, url: str): + try: + intersphinx.fetch_inventory(SphinxConfiguration(), '', url) + except AttributeError: + raise commands.BadArgument(f"Failed to fetch Intersphinx inventory from URL `{url}`.") + except ConnectionError: + if url.startswith('https'): + raise commands.BadArgument( + f"Cannot establish a connection to `{url}`. Does it support HTTPS?" + ) + raise commands.BadArgument(f"Cannot connect to host with URL `{url}`.") + except ValueError: + raise commands.BadArgument( + f"Failed to read Intersphinx inventory from URL `{url}`. " + "Are you sure that it's a valid inventory file?" + ) + return url + + +class Doc: + def __init__(self, bot): + self.base_urls = {} + self.bot = bot + self.inventories = {} + self.headers = {"X-API-KEY": Keys.site_api} + + async def on_ready(self): + await self.refresh_inventory() + + async def update_single( + self, package_name: str, base_url: str, inventory_url: str, config: SphinxConfiguration + ): + """ + Rebuild the inventory for a single package. + + :param package_name: The package name to use, appears in the log. + :param base_url: The root documentation URL for the specified package. + Used to build absolute paths that link to specific symbols. + :param inventory_url: The absolute URL to the intersphinx inventory. + Fetched by running `intersphinx.fetch_inventory` in an + executor on the bot's event loop. + :param config: A `SphinxConfiguration` instance to mock the regular sphinx + project layout. Required for use with intersphinx. + """ + + self.base_urls[package_name] = base_url + + fetch_func = functools.partial(intersphinx.fetch_inventory, config, '', inventory_url) + for _, value in (await self.bot.loop.run_in_executor(None, fetch_func)).items(): + # Each value has a bunch of information in the form + # `(package_name, version, relative_url, ???)`, and we only + # need the relative documentation URL. + for symbol, (_, _, relative_doc_url, _) in value.items(): + absolute_doc_url = base_url + relative_doc_url + self.inventories[symbol] = absolute_doc_url + + log.trace(f"Fetched inventory for {package_name}.") + + async def refresh_inventory(self): + log.debug("Refreshing documentation inventory...") + + # Clear the old base URLS and inventories to ensure + # that we start from a fresh local dataset. + # Also, reset the cache used for fetching documentation. + self.base_urls.clear() + self.inventories.clear() + async_cache.cache = OrderedDict() + + # Since Intersphinx is intended to be used with Sphinx, + # we need to mock its configuration. + config = SphinxConfiguration() + + # Run all coroutines concurrently - since each of them performs a HTTP + # request, this speeds up fetching the inventory data heavily. + coros = [ + self.update_single( + package["package"], package["base_url"], package["inventory_url"], config + ) for package in await self.get_all_packages() + ] + await asyncio.gather(*coros) + + async def get_symbol_html(self, symbol: str) -> Optional[Tuple[str, str]]: + """ + Given a Python symbol, return its signature and description. + + :param symbol: The symbol for which HTML data should be returned. + :return: + A tuple in the form (str, str), or `None`. + The first tuple element is the signature of the given + symbol as a markup-free string, and the second tuple + element is the description of the given symbol with HTML + markup included. If the given symbol could not be found, + returns `None`. + """ + + url = self.inventories.get(symbol) + if url is None: + return None + + async with self.bot.http_session.get(url) as response: + html = await response.text(encoding='utf-8') + + # Find the signature header and parse the relevant parts. + symbol_id = url.split('#')[-1] + soup = BeautifulSoup(html, 'lxml') + symbol_heading = soup.find(id=symbol_id) + signature_buffer = [] + + # Traverse the tags of the signature header and ignore any + # unwanted symbols from it. Add all of it to a temporary buffer. + for tag in symbol_heading.strings: + if tag not in UNWANTED_SIGNATURE_SYMBOLS: + signature_buffer.append(tag.replace('\\', '')) + + signature = ''.join(signature_buffer) + description = str(symbol_heading.next_sibling.next_sibling).replace('¶', '') + + return signature, description + + @async_cache(arg_offset=1) + async def get_symbol_embed(self, symbol: str) -> Optional[discord.Embed]: + """ + Using `get_symbol_html`, attempt to scrape and + fetch the data for the given `symbol`, and build + a formatted embed out of its contents. + + :param symbol: The symbol for which the embed should be returned + :return: + If the symbol is known, an Embed with documentation about it. + Otherwise, `None`. + """ + + scraped_html = await self.get_symbol_html(symbol) + if scraped_html is None: + return None + + signature = scraped_html[0] + permalink = self.inventories[symbol] + description = markdownify(scraped_html[1]) + + # Truncate the description of the embed to the last occurrence + # of a double newline (interpreted as a paragraph) before index 1000. + if len(description) > 1000: + shortened = description[:1000] + last_paragraph_end = shortened.rfind('\n\n') + description = description[:last_paragraph_end] + f"... [read more]({permalink})" + + description = WHITESPACE_AFTER_NEWLINES_RE.sub('', description) + + if not signature: + # It's some "meta-page", for example: + # https://docs.djangoproject.com/en/dev/ref/views/#module-django.views + return discord.Embed( + title=f'`{symbol}`', + url=permalink, + description="This appears to be a generic page not tied to a specific symbol." + ) + + return discord.Embed( + title=f'`{symbol}`', + url=permalink, + description=f"```py\n{signature}```{description}" + ) + + async def get_all_packages(self) -> List[Dict[str, str]]: + """ + Performs HTTP GET to get all packages from the website. + + :return: + A list of packages, in the following format: + [ + { + "package": "example-package", + "base_url": "https://example.readthedocs.io", + "inventory_url": "https://example.readthedocs.io/objects.inv" + }, + ... + ] + `package` specifies the package name, for example 'aiohttp'. + `base_url` specifies the documentation root URL, used to build absolute links. + `inventory_url` specifies the location of the Intersphinx inventory. + """ + + async with self.bot.http_session.get(URLs.site_docs_api, headers=self.headers) as resp: + return await resp.json() + + async def get_package(self, package_name: str) -> Optional[Dict[str, str]]: + """ + Performs HTTP GET to get the specified package from the documentation database. + + :param package_name: The package name for which information should be returned. + :return: + Either a dictionary with information in the following format: + { + "package": "example-package", + "base_url": "https://example.readthedocs.io", + "inventory_url": "https://example.readthedocs.io/objects.inv" + } + or `None` if the site didn't returned no results for the given name. + """ + + params = {"package": package_name} + + async with self.bot.http_session.get(URLs.site_docs_api, + headers=self.headers, + params=params) as resp: + package_data = await resp.json() + if not package_data: + return None + return package_data[0] + + async def set_package(self, name: str, base_url: str, inventory_url: str) -> Dict[str, bool]: + """ + Performs HTTP POST to add a new package to the website's documentation database. + + :param name: The name of the package, for example `aiohttp`. + :param base_url: The documentation root URL, used to build absolute links. + :param inventory_url: The absolute URl to the intersphinx inventory of the package. + + :return: The JSON response of the server, which is always: + { + "success": True + } + """ + + package_json = { + 'package': name, + 'base_url': base_url, + 'inventory_url': inventory_url + } + + async with self.bot.http_session.post(URLs.site_docs_api, + headers=self.headers, + json=package_json) as resp: + return await resp.json() + + async def delete_package(self, name: str) -> bool: + """ + Performs HTTP DELETE to delete the specified package from the documentation database. + + :param name: The package to delete. + + :return: `True` if successful, `False` if the package is unknown. + """ + + package_json = {'package': name} + + async with self.bot.http_session.delete(URLs.site_docs_api, + headers=self.headers, + json=package_json) as resp: + changes = await resp.json() + return changes["deleted"] == 1 # Did the package delete successfully? + + @commands.command(name='docs.get()', aliases=['docs.get']) + async def get_command(self, ctx, symbol: commands.clean_content = None): + """ + Return a documentation embed for a given symbol. + If no symbol is given, return a list of all available inventories. + + :param ctx: Discord message context + :param symbol: The symbol for which documentation should be returned, + or nothing to get a list of all inventories + + Examples: + bot.docs.get('aiohttp') + bot.docs['aiohttp'] + """ + + if symbol is None: + all_inventories = "\n".join( + f"• [`{name}`]({url})" for name, url in self.base_urls.items() + ) + inventory_embed = discord.Embed( + title="All inventories", + description=all_inventories or "*Seems like there's nothing here yet.*", + colour=discord.Colour.blue() + ) + await ctx.send(embed=inventory_embed) + + else: + # Fetching documentation for a symbol (at least for the first time, since + # caching is used) takes quite some time, so let's send typing to indicate + # that we got the command, but are still working on it. + async with ctx.typing(): + doc_embed = await self.get_symbol_embed(symbol) + + if doc_embed is None: + error_embed = discord.Embed( + description=f"Sorry, I could not find any documentation for `{symbol}`.", + colour=discord.Colour.red() + ) + await ctx.send(embed=error_embed) + else: + await ctx.send(embed=doc_embed) + + @with_role(Roles.admin, Roles.owner, Roles.moderator) + @commands.command(name='docs.set()', aliases=['docs.set']) + async def set_command( + self, ctx, package_name: ValidPythonIdentifier, + base_url: ValidURL, inventory_url: InventoryURL + ): + """ + Adds a new documentation metadata object to the site's database. + The database will update the object, should an existing item + with the specified `package_name` already exist. + + :param ctx: Discord message context + :param package_name: The package name, for example `aiohttp`. + :param base_url: The package documentation's root URL, used to build absolute links. + :param inventory_url: The intersphinx inventory URL. + + Example: + bot.docs.set( + 'discord', + 'https://discordpy.readthedocs.io/en/rewrite/', + 'https://discordpy.readthedocs.io/en/rewrite/objects.inv' + ) + """ + + await self.set_package(package_name, base_url, inventory_url) + log.info( + f"User @{ctx.author.name}#{ctx.author.discriminator} ({ctx.author.id}) " + "added a new documentation package:\n" + f"Package name: {package_name}\n" + f"Base url: {base_url}\n" + f"Inventory URL: {inventory_url}" + ) + + # Rebuilding the inventory can take some time, so lets send out a + # typing event to show that the Bot is still working. + async with ctx.typing(): + await self.refresh_inventory() + await ctx.send(f"Added package `{package_name}` to database and refreshed inventory.") + + @with_role(Roles.admin, Roles.owner, Roles.moderator) + @commands.command(name='docs.delete()', aliases=['docs.delete', 'docs.remove()', 'docs.remove']) + async def delete_command(self, ctx, package_name: ValidPythonIdentifier): + """ + Removes the specified package from the database. + + :param ctx: Discord message context + :param package_name: The package name, for example `aiohttp`. + + Examples: + bot.tags.delete('aiohttp') + bot.tags['aiohttp'] = None + """ + + success = await self.delete_package(package_name) + if success: + + async with ctx.typing(): + # Rebuild the inventory to ensure that everything + # that was from this package is properly deleted. + await self.refresh_inventory() + await ctx.send(f"Successfully deleted `{package_name}` and refreshed inventory.") + + else: + await ctx.send( + f"Can't find any package named `{package_name}` in the database. " + "View all known packages by using `docs.get()`." + ) + + @get_command.error + @delete_command.error + @set_command.error + async def general_command_error(self, ctx, error: commands.CommandError): + """ + Handle the `BadArgument` error caused by + the commands when argument validation fails. + + :param ctx: Discord message context of the message creating the error + :param error: The error raised, usually `BadArgument` + """ + + if isinstance(error, commands.BadArgument): + embed = discord.Embed( + title=random.choice(ERROR_REPLIES), + description=f"Error: {error}", + colour=discord.Colour.red() + ) + await ctx.send(embed=embed) + else: + log.exception(f"Unhandled error: {error}") + + +def setup(bot): + bot.add_cog(Doc(bot)) diff --git a/bot/cogs/utils.py b/bot/cogs/utils.py new file mode 100644 index 000000000..3d3d0de36 --- /dev/null +++ b/bot/cogs/utils.py @@ -0,0 +1,83 @@ +import logging +from email.parser import HeaderParser +from io import StringIO + + +from discord import Colour, Embed +from discord.ext.commands import AutoShardedBot, Context, command + +from bot.constants import Roles +from bot.decorators import with_role + +log = logging.getLogger(__name__) + + +class Utils: + """ + A selection of utilities which don't have a clear category. + """ + + def __init__(self, bot: AutoShardedBot): + self.bot = bot + + self.base_pep_url = "http://www.python.org/dev/peps/pep-" + self.base_github_pep_url = "https://raw.githubusercontent.com/python/peps/master/pep-" + + @command(name="pep()", aliases=["pep", "get_pep"]) + @with_role(Roles.verified) + async def pep_search(self, ctx: Context, pep_number: str): + """ + Fetches information about a PEP and sends it to the channel. + """ + + # Attempt to fetch the PEP from Github. + pep_url = f"{self.base_github_pep_url}{pep_number:04}.txt" + log.trace(f"Requesting PEP {pep_number} with {pep_url}") + response = await self.bot.http_session.get(pep_url) + + if response.status == 200: + log.trace("PEP found") + + pep_content = await response.text() + + # Taken from https://github.com/python/peps/blob/master/pep0/pep.py#L179 + pep_header = HeaderParser().parse(StringIO(pep_content)) + + # Assemble the embed + pep_embed = Embed( + title=f"**PEP {pep_number} - {pep_header['Title']}**", + description=f"[Link]({self.base_pep_url}{pep_number:04})", + ) + + pep_embed.set_thumbnail(url="https://www.python.org/static/opengraph-icon-200x200.png") + + # Add the interesting information + if "Status" in pep_header: + pep_embed.add_field(name="Status", value=pep_header["Status"]) + if "Python-Version" in pep_header: + pep_embed.add_field(name="Python-Version", value=pep_header["Python-Version"]) + if "Created" in pep_header: + pep_embed.add_field(name="Created", value=pep_header["Created"]) + if "Type" in pep_header: + pep_embed.add_field(name="Type", value=pep_header["Type"]) + + elif response.status == 404: + log.trace("PEP was not found") + not_found = f"PEP {pep_number} does not exist." + pep_embed = Embed(title="PEP not found", description=not_found) + pep_embed.colour = Colour.red() + + else: + log.trace(f"The user requested PEP {pep_number}, but the response had an unexpected status code: " + f"{response.status}.\n{response.text}") + + error_message = "Unexpected HTTP error during PEP search. Please let us know." + pep_embed = Embed(title="Unexpected error", description=error_message) + pep_embed.colour = Colour.red() + + await ctx.message.channel.send(embed=pep_embed) + + +def setup(bot): + bot.add_cog(Utils(bot)) + log.info("Utils cog loaded") diff --git a/bot/converters.py b/bot/converters.py index dc78e5e08..5637ab8b2 100644 --- a/bot/converters.py +++ b/bot/converters.py @@ -1,9 +1,10 @@ import random import socket +from ssl import CertificateError import discord -from aiohttp import AsyncResolver, ClientSession, TCPConnector -from discord.ext.commands import Converter +from aiohttp import AsyncResolver, ClientConnectorError, ClientSession, TCPConnector +from discord.ext.commands import BadArgument, Converter from fuzzywuzzy import fuzz from bot.constants import DEBUG_MODE, Keys, URLs @@ -105,3 +106,54 @@ class Snake(Converter): await cls.build_list() names = [snake['scientific'] for snake in cls.snakes] return random.choice(names) + + +class ValidPythonIdentifier(Converter): + """ + A converter that checks whether the given string is a valid Python identifier. + + This is used to have package names + that correspond to how you would use + the package in your code, e.g. + `import package`. Raises `BadArgument` + if the argument is not a valid Python + identifier, and simply passes through + the given argument otherwise. + """ + + @staticmethod + async def convert(ctx, argument: str): + if not argument.isidentifier(): + raise BadArgument(f"`{argument}` is not a valid Python identifier") + return argument + + +class ValidURL(Converter): + """ + Represents a valid webpage URL. + + This converter checks whether the given + URL can be reached and requesting it returns + a status code of 200. If not, `BadArgument` + is raised. Otherwise, it simply passes through the given URL. + """ + + @staticmethod + async def convert(ctx, url: str): + try: + async with ctx.bot.http_session.get(url) as resp: + if resp.status != 200: + raise BadArgument( + f"HTTP GET on `{url}` returned status `{resp.status_code}`, expected 200" + ) + except CertificateError: + if url.startswith('https'): + raise BadArgument( + f"Got a `CertificateError` for URL `{url}`. Does it support HTTPS?" + ) + raise BadArgument(f"Got a `CertificateError` for URL `{url}`.") + except ValueError: + raise BadArgument(f"`{url}` doesn't look like a valid hostname to me.") + except ClientConnectorError: + raise BadArgument(f"Cannot connect to host with URL `{url}`.") + return url diff --git a/config-default.yml b/config-default.yml index eda77c2d0..e10150617 100644 --- a/config-default.yml +++ b/config-default.yml @@ -72,6 +72,7 @@ urls: status: !ENV 'STATUS_URL' site: 'pythondiscord.com' site_hiphopify_api: 'https://api.pythondiscord.com/bot/hiphopify' + site_docs_api: 'https://api.pythondiscord.com/bot/docs' site_tags_api: 'https://api.pythondiscord.com/bot/tags' site_user_api: 'https://api.pythondiscord.com/bot/users' site_quiz_api: 'https://api.pythondiscord.com/bot/snake_quiz' |