Add documentation lookup support. (#53)

* Allow users to query documentation link.

* Show symbol signature and excerpt from docs.

* Use `•` character for showing list items.

* Trim whitespace after two newlines.

* Allow users to query documentation link.

* Show symbol signature and excerpt from docs.

* Use `•` character for showing list items.

* Trim whitespace after two newlines.

* Add docstrings, simplify command.

* Use a cache for getting doc embeds.

* Suppress `urllib3` log messages below WARNING.

* Use shared `http_session` on the bot.

* Comment `fetch_initial_inventory_data`, unpack URL directly.

* Address various review comments.

- use `str.format` for generating the stdlib documentation URLs
- move `convert_code` comment to a docstring
- fix up python syntax parsing by not using a `*`
- send red error embeds when documentation could not be found

* Use `lxml` for HTML parsing.

* Pin yarl to 1.1.1

* Use the website's API as package documentation metadata database.

* Fetch inventories concurrently.

* Use red error embed & error title on error.

* Add examples to command docstrings.

* Use typing notifications instead of edits.

* Flush cache on inventory refresh.

* Address @martmists' review comments.

* Use `BadArgument` instead of custom exception class.

* Move universal converters to `bot/converters.py`.

6 files changed, 703 insertions, 4 deletions

diff --git a/Pipfile b/Pipfile
index 71c122dc6..e6d1366c6 100644
--- a/Pipfile
+++ b/Pipfile
@@ -12,6 +12,9 @@ aiodns = "*"
 logmatic-python = "*"
 aiohttp = "<2.3.0,>=2.0.0"
 websockets = ">=4.0,<5.0"
+sphinx = "*"
+markdownify = "*"
+lxml = "*"
 pyyaml = "*"
 yarl = "==1.1.1"
 fuzzywuzzy = "*"
diff --git a/Pipfile.lock b/Pipfile.lock
index 64652ae3b..9f4c3a1d4 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "1529809913b543e7fbbf6091c33369f96567ab1fc8d7d637df574d4ac9d8b050"
+            "sha256": "2475ab039e5ce97636b9a255dd7e21b67e7b66d832f341b4065876f7535b2d88"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -51,6 +51,13 @@
             "index": "pypi",
             "version": "==2.2.5"
         },
+        "alabaster": {
+            "hashes": [
+                "sha256:2eef172f44e8d301d25aff8068fddd65f767a3f04b5f15b0f4922f113aa1c732",
+                "sha256:37cdcb9e9954ed60912ebc1ca12a9d12178c26637abdf124e3cde2341c257fe0"
+            ],
+            "version": "==0.7.10"
+        },
         "async-timeout": {
             "hashes": [
                 "sha256:474d4bc64cee20603e225eb1ece15e248962958b45a3648a9f5cc29e827a610c",
@@ -58,6 +65,21 @@
             ],
             "version": "==3.0.0"
         },
+        "babel": {
+            "hashes": [
+                "sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669",
+                "sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23"
+            ],
+            "version": "==2.6.0"
+        },
+        "beautifulsoup4": {
+            "hashes": [
+                "sha256:11a9a27b7d3bddc6d86f59fb76afb70e921a25ac2d6cc55b40d072bd68435a76",
+                "sha256:7015e76bf32f1f574636c4288399a6de66ce08fb7b2457f628a8d70c0fbabb11",
+                "sha256:808b6ac932dccb0a4126558f7dfdcf41710dd44a4ef497a0bb59a77f9f078e89"
+            ],
+            "version": "==4.6.0"
+        },
         "certifi": {
             "hashes": [
                 "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7",
@@ -76,6 +98,14 @@
             "egg": "discord.py[voice]",
             "file": "https://github.com/Rapptz/discord.py/archive/rewrite.zip"
         },
+        "docutils": {
+            "hashes": [
+                "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6",
+                "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274",
+                "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6"
+            ],
+            "version": "==0.14"
+        },
         "dulwich": {
             "hashes": [
                 "sha256:c51e10c260543240e0806052af046e1a78b98cbe1ac1ef3880a78d2269e09da4"
@@ -98,6 +128,20 @@
             ],
             "version": "==2.6"
         },
+        "imagesize": {
+            "hashes": [
+                "sha256:3620cc0cadba3f7475f9940d22431fc4d407269f1be59ec9b8edcca26440cf18",
+                "sha256:5b326e4678b6925158ccc66a9fa3122b6106d7c876ee32d7de6ce59385b96315"
+            ],
+            "version": "==1.0.0"
+        },
+        "jinja2": {
+            "hashes": [
+                "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd",
+                "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4"
+            ],
+            "version": "==2.10"
+        },
         "logmatic-python": {
             "hashes": [
                 "sha256:0c15ac9f5faa6a60059b28910db642c3dc7722948c3cc940923f8c9039604342"
@@ -105,6 +149,53 @@
             "index": "pypi",
             "version": "==0.1.7"
         },
+        "lxml": {
+            "hashes": [
+                "sha256:01c45df6d90497c20aa2a07789a41941f9a1029faa30bf725fc7f6d515b1afe9",
+                "sha256:0c9fef4f8d444e337df96c54544aeb85b7215b2ed7483bb6c35de97ac99f1bcd",
+                "sha256:0e3cd94c95d30ba9ca3cff40e9b2a14e1a10a4fd8131105b86c6b61648f57e4b",
+                "sha256:0e7996e9b46b4d8b4ac1c329a00e2d10edcd8380b95d2a676fccabf4c1dd0512",
+                "sha256:1858b1933d483ec5727549d3fe166eeb54229fbd6a9d3d7ea26d2c8a28048058",
+                "sha256:1b164bba1320b14905dcff77da10d5ce9c411ac4acc4fb4ed9a2a4d10fae38c9",
+                "sha256:1b46f37927fa6cd1f3fe34b54f1a23bd5bea1d905657289e08e1297069a1a597",
+                "sha256:231047b05907315ae9a9b6925751f9fd2c479cf7b100fff62485a25e382ca0d4",
+                "sha256:28f0c6652c1b130f1e576b60532f84b19379485eb8da6185c29bd8c9c9bc97bf",
+                "sha256:34d49d0f72dd82b9530322c48b70ac78cca0911275da741c3b1d2f3603c5f295",
+                "sha256:3682a17fbf72d56d7e46db2e80ca23850b79c28cfe75dcd9b82f58808f730909",
+                "sha256:3cf2830b9a6ad7f6e965fa53a768d4d2372a7856f20ffa6ce43d2fe9c0d34b19",
+                "sha256:5b653c9379ce29ce271fbe1010c5396670f018e78b643e21beefbb3dc6d291de",
+                "sha256:65a272821d5d8194358d6b46f3ca727fa56a6b63981606eac737c86d27309cdd",
+                "sha256:691f2cd97cf026c611df1ea5055755eec7f878f2d4f4330dc8686583de6fc5fd",
+                "sha256:6b6379495d3baacf7ed755ac68547c8dff6ce5d37bf370f0b7678888dc1283f9",
+                "sha256:75322a531504d4f383264391d89993a42e286da8821ddc5ac315e57305cb84f0",
+                "sha256:7f457cbda964257f443bac861d3a36732dcba8183149e7818ee2fb7c86901b94",
+                "sha256:7ff1fc76d8804e0f870c343a72007ff587090c218b0f92d8ee784ac2b6eaf5b9",
+                "sha256:8523fbde9c2216f3f2b950cb01ebe52e785eaa8a07ffeb456dd3576ca1b4fb9b",
+                "sha256:8f37627f16e026523fca326f1b5c9a43534862fede6c3e99c2ba6a776d75c1ab",
+                "sha256:a7182ea298cc3555ea56ffbb0748fe0d5e0d81451e2bc16d7f4645cd01b1ca70",
+                "sha256:abbd2fb4a5a04c11b5e04eb146659a0cf67bb237dd3d7ca3b9994d3a9f826e55",
+                "sha256:accc9f6b77bed0a6f267b4fae120f6008a951193d548cdbe9b61fc98a08b1cf8",
+                "sha256:bd88c8ce0d1504fdfd96a35911dd4f3edfb2e560d7cfdb5a3d09aa571ae5fbae",
+                "sha256:c557ad647facb3c0027a9d0af58853f905e85a0a2f04dcb73f8e665272fcdc3a",
+                "sha256:defabb7fbb99f9f7b3e0b24b286a46855caef4776495211b066e9e6592d12b04",
+                "sha256:e2629cdbcad82b83922a3488937632a4983ecc0fed3e5cfbf430d069382eeb9b"
+            ],
+            "index": "pypi",
+            "version": "==4.2.1"
+        },
+        "markdownify": {
+            "hashes": [
+                "sha256:28ce67d1888e4908faaab7b04d2193cda70ea4f902f156a21d0aaea55e63e0a1"
+            ],
+            "index": "pypi",
+            "version": "==0.4.1"
+        },
+        "markupsafe": {
+            "hashes": [
+                "sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665"
+            ],
+            "version": "==1.0"
+        },
         "mpmath": {
             "hashes": [
                 "sha256:04d14803b6875fe6d69e6dccea87d5ae5599802e4b1df7997bddd2024001050c"
@@ -130,6 +221,13 @@
             "index": "pypi",
             "version": "==4.3.1"
         },
+        "packaging": {
+            "hashes": [
+                "sha256:e9215d2d2535d3ae866c3d6efc77d5b24a0192cce0ff20e42896cc0664f889c0",
+                "sha256:f019b770dd64e585a99714f1fd5e01c7a8f11b45635aa953fd41c689a657375b"
+            ],
+            "version": "==17.1"
+        },
         "pika": {
             "hashes": [
                 "sha256:63131aaeec48a6c8f1db1fe657e1e74cf384c3927eb7d1725e31edae4220dea4",
@@ -212,6 +310,25 @@
             ],
             "version": "==2.3.0"
         },
+        "pygments": {
+            "hashes": [
+                "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d",
+                "sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc"
+            ],
+            "version": "==2.2.0"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04",
+                "sha256:281683241b25fe9b80ec9d66017485f6deff1af5cde372469134b56ca8447a07",
+                "sha256:8f1e18d3fd36c6795bb7e02a39fd05c611ffc2596c1e0d995d34d67630426c18",
+                "sha256:9e8143a3e15c13713506886badd96ca4b579a87fbdf49e550dbfc057d6cb218e",
+                "sha256:b8b3117ed9bdf45e14dcc89345ce638ec7e0e29b2b579fa1ecf32ce45ebac8a5",
+                "sha256:e4d45427c6e20a59bf4f88c639dcc03ce30d193112047f94012102f235853a58",
+                "sha256:fee43f17a9c4087e7ed1605bd6df994c6173c1e977d7ade7b651292fab2bd010"
+            ],
+            "version": "==2.2.0"
+        },
         "python-dateutil": {
             "hashes": [
                 "sha256:1adb80e7a782c12e52ef9a8182bebeb73f1d7e24e374397af06fb4956c8dc5c0",
@@ -234,6 +351,13 @@
             "index": "pypi",
             "version": "==0.12.0"
         },
+        "pytz": {
+            "hashes": [
+                "sha256:65ae0c8101309c45772196b21b74c46b2e5d11b6275c45d251b150d5da334555",
+                "sha256:c06425302f2cf668f1bba7a0a03f3c1d34d4ebeef2c72003da308b3947c7f749"
+            ],
+            "version": "==2018.4"
+        },
         "pyyaml": {
             "hashes": [
                 "sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8",
@@ -254,6 +378,13 @@
             "index": "pypi",
             "version": "==3.12"
         },
+        "requests": {
+            "hashes": [
+                "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b",
+                "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e"
+            ],
+            "version": "==2.18.4"
+        },
         "shortuuid": {
             "hashes": [
                 "sha256:d08fd398f40f8baf87e15eef8355e92fa541bca4eb8465fefab7ee22f92711b9"
@@ -267,6 +398,28 @@
             ],
             "version": "==1.11.0"
         },
+        "snowballstemmer": {
+            "hashes": [
+                "sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128",
+                "sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89"
+            ],
+            "version": "==1.2.1"
+        },
+        "sphinx": {
+            "hashes": [
+                "sha256:85f7e32c8ef07f4ba5aeca728e0f7717bef0789fba8458b8d9c5c294cad134f3",
+                "sha256:d45480a229edf70d84ca9fae3784162b1bc75ee47e480ffe04a4b7f21a95d76d"
+            ],
+            "index": "pypi",
+            "version": "==1.7.5"
+        },
+        "sphinxcontrib-websupport": {
+            "hashes": [
+                "sha256:7a85961326aa3a400cd4ad3c816d70ed6f7c740acd7ce5d78cd0a67825072eb9",
+                "sha256:f4932e95869599b89bf4f80fc3989132d83c9faa5bf633e7b5e0c25dffb75da2"
+            ],
+            "version": "==1.0.1"
+        },
         "sympy": {
             "hashes": [
                 "sha256:ac5b57691bc43919dcc21167660a57cc51797c28a4301a6144eff07b751216a4"
@@ -485,7 +638,6 @@
                 "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b",
                 "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e"
             ],
-            "index": "pypi",
             "version": "==2.18.4"
         },
         "safety": {
diff --git a/bot/__main__.py b/bot/__main__.py
index 12c446b55..b1e9c61fa 100644
--- a/bot/__main__.py
+++ b/bot/__main__.py
@@ -55,6 +55,7 @@ else:
     log.info("`CLICKUP_KEY` not set in the environment, not loading the ClickUp cog.")
 
 bot.load_extension("bot.cogs.deployment")
+bot.load_extension("bot.cogs.doc")
 bot.load_extension("bot.cogs.eval")
 bot.load_extension("bot.cogs.fun")
 bot.load_extension("bot.cogs.hiphopify")
diff --git a/bot/cogs/doc.py b/bot/cogs/doc.py
new file mode 100644
index 000000000..d84de0ba3
--- /dev/null
+++ b/bot/cogs/doc.py
@@ -0,0 +1,490 @@
+import asyncio
+import functools
+import logging
+import random
+import re
+from collections import OrderedDict
+from typing import Dict, List, Optional, Tuple
+
+import discord
+from bs4 import BeautifulSoup
+from discord.ext import commands
+from markdownify import MarkdownConverter
+from requests import ConnectionError
+from sphinx.ext import intersphinx
+
+from bot.constants import ERROR_REPLIES, Keys, Roles, URLs
+from bot.converters import ValidPythonIdentifier, ValidURL
+from bot.decorators import with_role
+
+
+log = logging.getLogger(__name__)
+logging.getLogger('urllib3').setLevel(logging.WARNING)
+
+
+UNWANTED_SIGNATURE_SYMBOLS = ('[source]', '¶')
+WHITESPACE_AFTER_NEWLINES_RE = re.compile(r"(?<=\n\n)(\s+)")
+
+
+def async_cache(max_size=128, arg_offset=0):
+    """
+    LRU cache implementation for coroutines.
+
+    :param max_size:
+    Specifies the maximum size the cache should have.
+    Once it exceeds the maximum size, keys are deleted in FIFO order.
+    :param arg_offset:
+    The offset that should be applied to the coroutine's arguments
+    when creating the cache key. Defaults to `0`.
+    """
+
+    # Assign the cache to the function itself so we can clear it from outside.
+    async_cache.cache = OrderedDict()
+
+    def decorator(function):
+        @functools.wraps(function)
+        async def wrapper(*args):
+            key = ':'.join(args[arg_offset:])
+
+            value = async_cache.cache.get(key)
+            if value is None:
+                if len(async_cache.cache) > max_size:
+                    async_cache.cache.popitem(last=False)
+
+                async_cache.cache[key] = await function(*args)
+            return async_cache.cache[key]
+        return wrapper
+    return decorator
+
+
+class DocMarkdownConverter(MarkdownConverter):
+    def convert_code(self, el, text):
+        """Undo `markdownify`s underscore escaping."""
+
+        return f"`{text}`".replace('\\', '')
+
+    def convert_pre(self, el, text):
+        """Wrap any codeblocks in `py` for syntax highlighting."""
+
+        code = ''.join(el.strings)
+        return f"```py\n{code}```"
+
+
+def markdownify(html):
+    return DocMarkdownConverter(bullets='•').convert(html)
+
+
+class DummyObject(object):
+    """
+    A dummy object which supports assigning anything,
+    which the builtin `object()` does not support normally.
+    """
+
+
+class SphinxConfiguration:
+    """Dummy configuration for use with intersphinx."""
+
+    config = DummyObject()
+    config.intersphinx_timeout = 3
+    config.tls_verify = True
+
+
+class InventoryURL(commands.Converter):
+    """
+    Represents an Intersphinx inventory URL.
+
+    This converter checks whether intersphinx
+    accepts the given inventory URL, and raises
+    `BadArgument` if that is not the case.
+    Otherwise, it simply passes through the given URL.
+    """
+
+    @staticmethod
+    async def convert(ctx, url: str):
+        try:
+            intersphinx.fetch_inventory(SphinxConfiguration(), '', url)
+        except AttributeError:
+            raise commands.BadArgument(f"Failed to fetch Intersphinx inventory from URL `{url}`.")
+        except ConnectionError:
+            if url.startswith('https'):
+                raise commands.BadArgument(
+                    f"Cannot establish a connection to `{url}`. Does it support HTTPS?"
+                )
+            raise commands.BadArgument(f"Cannot connect to host with URL `{url}`.")
+        except ValueError:
+            raise commands.BadArgument(
+                f"Failed to read Intersphinx inventory from URL `{url}`. "
+                "Are you sure that it's a valid inventory file?"
+            )
+        return url
+
+
+class Doc:
+    def __init__(self, bot):
+        self.base_urls = {}
+        self.bot = bot
+        self.inventories = {}
+        self.headers = {"X-API-KEY": Keys.site_api}
+
+    async def on_ready(self):
+        await self.refresh_inventory()
+
+    async def update_single(
+        self, package_name: str, base_url: str, inventory_url: str, config: SphinxConfiguration
+    ):
+        """
+        Rebuild the inventory for a single package.
+
+        :param package_name: The package name to use, appears in the log.
+        :param base_url: The root documentation URL for the specified package.
+                         Used to build absolute paths that link to specific symbols.
+        :param inventory_url: The absolute URL to the intersphinx inventory.
+                              Fetched by running `intersphinx.fetch_inventory` in an
+                              executor on the bot's event loop.
+        :param config: A `SphinxConfiguration` instance to mock the regular sphinx
+                       project layout. Required for use with intersphinx.
+        """
+
+        self.base_urls[package_name] = base_url
+
+        fetch_func = functools.partial(intersphinx.fetch_inventory, config, '', inventory_url)
+        for _, value in (await self.bot.loop.run_in_executor(None, fetch_func)).items():
+            # Each value has a bunch of information in the form
+            # `(package_name, version, relative_url, ???)`, and we only
+            # need the relative documentation URL.
+            for symbol, (_, _, relative_doc_url, _) in value.items():
+                absolute_doc_url = base_url + relative_doc_url
+                self.inventories[symbol] = absolute_doc_url
+
+        log.trace(f"Fetched inventory for {package_name}.")
+
+    async def refresh_inventory(self):
+        log.debug("Refreshing documentation inventory...")
+
+        # Clear the old base URLS and inventories to ensure
+        # that we start from a fresh local dataset.
+        # Also, reset the cache used for fetching documentation.
+        self.base_urls.clear()
+        self.inventories.clear()
+        async_cache.cache = OrderedDict()
+
+        # Since Intersphinx is intended to be used with Sphinx,
+        # we need to mock its configuration.
+        config = SphinxConfiguration()
+
+        # Run all coroutines concurrently - since each of them performs a HTTP
+        # request, this speeds up fetching the inventory data heavily.
+        coros = [
+            self.update_single(
+                package["package"], package["base_url"], package["inventory_url"], config
+            ) for package in await self.get_all_packages()
+        ]
+        await asyncio.gather(*coros)
+
+    async def get_symbol_html(self, symbol: str) -> Optional[Tuple[str, str]]:
+        """
+        Given a Python symbol, return its signature and description.
+
+        :param symbol: The symbol for which HTML data should be returned.
+        :return:
+        A tuple in the form (str, str), or `None`.
+        The first tuple element is the signature of the given
+        symbol as a markup-free string, and the second tuple
+        element is the description of the given symbol with HTML
+        markup included. If the given symbol could not be found,
+        returns `None`.
+        """
+
+        url = self.inventories.get(symbol)
+        if url is None:
+            return None
+
+        async with self.bot.http_session.get(url) as response:
+            html = await response.text(encoding='utf-8')
+
+        # Find the signature header and parse the relevant parts.
+        symbol_id = url.split('#')[-1]
+        soup = BeautifulSoup(html, 'lxml')
+        symbol_heading = soup.find(id=symbol_id)
+        signature_buffer = []
+
+        # Traverse the tags of the signature header and ignore any
+        # unwanted symbols from it. Add all of it to a temporary buffer.
+        for tag in symbol_heading.strings:
+            if tag not in UNWANTED_SIGNATURE_SYMBOLS:
+                signature_buffer.append(tag.replace('\\', ''))
+
+        signature = ''.join(signature_buffer)
+        description = str(symbol_heading.next_sibling.next_sibling).replace('¶', '')
+
+        return signature, description
+
+    @async_cache(arg_offset=1)
+    async def get_symbol_embed(self, symbol: str) -> Optional[discord.Embed]:
+        """
+        Using `get_symbol_html`, attempt to scrape and
+        fetch the data for the given `symbol`, and build
+        a formatted embed out of its contents.
+
+        :param symbol: The symbol for which the embed should be returned
+        :return:
+        If the symbol is known, an Embed with documentation about it.
+        Otherwise, `None`.
+        """
+
+        scraped_html = await self.get_symbol_html(symbol)
+        if scraped_html is None:
+            return None
+
+        signature = scraped_html[0]
+        permalink = self.inventories[symbol]
+        description = markdownify(scraped_html[1])
+
+        # Truncate the description of the embed to the last occurrence
+        # of a double newline (interpreted as a paragraph) before index 1000.
+        if len(description) > 1000:
+            shortened = description[:1000]
+            last_paragraph_end = shortened.rfind('\n\n')
+            description = description[:last_paragraph_end] + f"... [read more]({permalink})"
+
+        description = WHITESPACE_AFTER_NEWLINES_RE.sub('', description)
+
+        if not signature:
+            # It's some "meta-page", for example:
+            # https://docs.djangoproject.com/en/dev/ref/views/#module-django.views
+            return discord.Embed(
+                title=f'`{symbol}`',
+                url=permalink,
+                description="This appears to be a generic page not tied to a specific symbol."
+            )
+
+        return discord.Embed(
+            title=f'`{symbol}`',
+            url=permalink,
+            description=f"```py\n{signature}```{description}"
+        )
+
+    async def get_all_packages(self) -> List[Dict[str, str]]:
+        """
+        Performs HTTP GET to get all packages from the website.
+
+        :return:
+        A list of packages, in the following format:
+        [
+            {
+                "package": "example-package",
+                "base_url": "https://example.readthedocs.io",
+                "inventory_url": "https://example.readthedocs.io/objects.inv"
+            },
+            ...
+        ]
+        `package` specifies the package name, for example 'aiohttp'.
+        `base_url` specifies the documentation root URL, used to build absolute links.
+        `inventory_url` specifies the location of the Intersphinx inventory.
+        """
+
+        async with self.bot.http_session.get(URLs.site_docs_api, headers=self.headers) as resp:
+            return await resp.json()
+
+    async def get_package(self, package_name: str) -> Optional[Dict[str, str]]:
+        """
+        Performs HTTP GET to get the specified package from the documentation database.
+
+        :param package_name: The package name for which information should be returned.
+        :return:
+        Either a dictionary with information in the following format:
+        {
+            "package": "example-package",
+            "base_url": "https://example.readthedocs.io",
+            "inventory_url": "https://example.readthedocs.io/objects.inv"
+        }
+        or `None` if the site didn't returned no results for the given name.
+        """
+
+        params = {"package": package_name}
+
+        async with self.bot.http_session.get(URLs.site_docs_api,
+                                             headers=self.headers,
+                                             params=params) as resp:
+            package_data = await resp.json()
+            if not package_data:
+                return None
+            return package_data[0]
+
+    async def set_package(self, name: str, base_url: str, inventory_url: str) -> Dict[str, bool]:
+        """
+        Performs HTTP POST to add a new package to the website's documentation database.
+
+        :param name: The name of the package, for example `aiohttp`.
+        :param base_url: The documentation root URL, used to build absolute links.
+        :param inventory_url: The absolute URl to the intersphinx inventory of the package.
+
+        :return: The JSON response of the server, which is always:
+        {
+            "success": True
+        }
+        """
+
+        package_json = {
+            'package': name,
+            'base_url': base_url,
+            'inventory_url': inventory_url
+        }
+
+        async with self.bot.http_session.post(URLs.site_docs_api,
+                                              headers=self.headers,
+                                              json=package_json) as resp:
+            return await resp.json()
+
+    async def delete_package(self, name: str) -> bool:
+        """
+        Performs HTTP DELETE to delete the specified package from the documentation database.
+
+        :param name: The package to delete.
+
+        :return: `True` if successful, `False` if the package is unknown.
+        """
+
+        package_json = {'package': name}
+
+        async with self.bot.http_session.delete(URLs.site_docs_api,
+                                                headers=self.headers,
+                                                json=package_json) as resp:
+            changes = await resp.json()
+            return changes["deleted"] == 1  # Did the package delete successfully?
+
+    @commands.command(name='docs.get()', aliases=['docs.get'])
+    async def get_command(self, ctx, symbol: commands.clean_content = None):
+        """
+        Return a documentation embed for a given symbol.
+        If no symbol is given, return a list of all available inventories.
+
+        :param ctx: Discord message context
+        :param symbol: The symbol for which documentation should be returned,
+                       or nothing to get a list of all inventories
+
+        Examples:
+            bot.docs.get('aiohttp')
+            bot.docs['aiohttp']
+        """
+
+        if symbol is None:
+            all_inventories = "\n".join(
+                f"• [`{name}`]({url})" for name, url in self.base_urls.items()
+            )
+            inventory_embed = discord.Embed(
+                title="All inventories",
+                description=all_inventories or "*Seems like there's nothing here yet.*",
+                colour=discord.Colour.blue()
+            )
+            await ctx.send(embed=inventory_embed)
+
+        else:
+            # Fetching documentation for a symbol (at least for the first time, since
+            # caching is used) takes quite some time, so let's send typing to indicate
+            # that we got the command, but are still working on it.
+            async with ctx.typing():
+                doc_embed = await self.get_symbol_embed(symbol)
+
+            if doc_embed is None:
+                error_embed = discord.Embed(
+                    description=f"Sorry, I could not find any documentation for `{symbol}`.",
+                    colour=discord.Colour.red()
+                )
+                await ctx.send(embed=error_embed)
+            else:
+                await ctx.send(embed=doc_embed)
+
+    @with_role(Roles.admin, Roles.owner, Roles.moderator)
+    @commands.command(name='docs.set()', aliases=['docs.set'])
+    async def set_command(
+        self, ctx, package_name: ValidPythonIdentifier,
+        base_url: ValidURL, inventory_url: InventoryURL
+    ):
+        """
+        Adds a new documentation metadata object to the site's database.
+        The database will update the object, should an existing item
+        with the specified `package_name` already exist.
+
+        :param ctx: Discord message context
+        :param package_name: The package name, for example `aiohttp`.
+        :param base_url: The package documentation's root URL, used to build absolute links.
+        :param inventory_url: The intersphinx inventory URL.
+
+        Example:
+            bot.docs.set(
+                'discord',
+                'https://discordpy.readthedocs.io/en/rewrite/',
+                'https://discordpy.readthedocs.io/en/rewrite/objects.inv'
+            )
+        """
+
+        await self.set_package(package_name, base_url, inventory_url)
+        log.info(
+            f"User @{ctx.author.name}#{ctx.author.discriminator} ({ctx.author.id}) "
+            "added a new documentation package:\n"
+            f"Package name: {package_name}\n"
+            f"Base url: {base_url}\n"
+            f"Inventory URL: {inventory_url}"
+        )
+
+        # Rebuilding the inventory can take some time, so lets send out a
+        # typing event to show that the Bot is still working.
+        async with ctx.typing():
+            await self.refresh_inventory()
+        await ctx.send(f"Added package `{package_name}` to database and refreshed inventory.")
+
+    @with_role(Roles.admin, Roles.owner, Roles.moderator)
+    @commands.command(name='docs.delete()', aliases=['docs.delete', 'docs.remove()', 'docs.remove'])
+    async def delete_command(self, ctx, package_name: ValidPythonIdentifier):
+        """
+        Removes the specified package from the database.
+
+        :param ctx: Discord message context
+        :param package_name: The package name, for example `aiohttp`.
+
+        Examples:
+            bot.tags.delete('aiohttp')
+            bot.tags['aiohttp'] = None
+        """
+
+        success = await self.delete_package(package_name)
+        if success:
+
+            async with ctx.typing():
+                # Rebuild the inventory to ensure that everything
+                # that was from this package is properly deleted.
+                await self.refresh_inventory()
+            await ctx.send(f"Successfully deleted `{package_name}` and refreshed inventory.")
+
+        else:
+            await ctx.send(
+                f"Can't find any package named `{package_name}` in the database. "
+                "View all known packages by using `docs.get()`."
+            )
+
+    @get_command.error
+    @delete_command.error
+    @set_command.error
+    async def general_command_error(self, ctx, error: commands.CommandError):
+        """
+        Handle the `BadArgument` error caused by
+        the commands when argument validation fails.
+
+        :param ctx: Discord message context of the message creating the error
+        :param error: The error raised, usually `BadArgument`
+        """
+
+        if isinstance(error, commands.BadArgument):
+            embed = discord.Embed(
+                title=random.choice(ERROR_REPLIES),
+                description=f"Error: {error}",
+                colour=discord.Colour.red()
+            )
+            await ctx.send(embed=embed)
+        else:
+            log.exception(f"Unhandled error: {error}")
+
+
+def setup(bot):
+    bot.add_cog(Doc(bot))
diff --git a/bot/converters.py b/bot/converters.py
index dc78e5e08..5637ab8b2 100644
--- a/bot/converters.py
+++ b/bot/converters.py
@@ -1,9 +1,10 @@
 import random
 import socket
+from ssl import CertificateError
 
 import discord
-from aiohttp import AsyncResolver, ClientSession, TCPConnector
-from discord.ext.commands import Converter
+from aiohttp import AsyncResolver, ClientConnectorError, ClientSession, TCPConnector
+from discord.ext.commands import BadArgument, Converter
 from fuzzywuzzy import fuzz
 
 from bot.constants import DEBUG_MODE, Keys, URLs
@@ -105,3 +106,54 @@ class Snake(Converter):
         await cls.build_list()
         names = [snake['scientific'] for snake in cls.snakes]
         return random.choice(names)
+
+
+class ValidPythonIdentifier(Converter):
+    """
+    A converter that checks whether the given string is a valid Python identifier.
+
+    This is used to have package names
+    that correspond to how you would use
+    the package in your code, e.g.
+    `import package`. Raises `BadArgument`
+    if the argument is not a valid Python
+    identifier, and simply passes through
+    the given argument otherwise.
+    """
+
+    @staticmethod
+    async def convert(ctx, argument: str):
+        if not argument.isidentifier():
+            raise BadArgument(f"`{argument}` is not a valid Python identifier")
+        return argument
+
+
+class ValidURL(Converter):
+    """
+    Represents a valid webpage URL.
+
+    This converter checks whether the given
+    URL can be reached and requesting it returns
+    a status code of 200. If not, `BadArgument`
+    is raised. Otherwise, it simply passes through the given URL.
+    """
+
+    @staticmethod
+    async def convert(ctx, url: str):
+        try:
+            async with ctx.bot.http_session.get(url) as resp:
+                if resp.status != 200:
+                    raise BadArgument(
+                        f"HTTP GET on `{url}` returned status `{resp.status_code}`, expected 200"
+                    )
+        except CertificateError:
+            if url.startswith('https'):
+                raise BadArgument(
+                    f"Got a `CertificateError` for URL `{url}`. Does it support HTTPS?"
+                )
+            raise BadArgument(f"Got a `CertificateError` for URL `{url}`.")
+        except ValueError:
+            raise BadArgument(f"`{url}` doesn't look like a valid hostname to me.")
+        except ClientConnectorError:
+            raise BadArgument(f"Cannot connect to host with URL `{url}`.")
+        return url
diff --git a/config-default.yml b/config-default.yml
index eda77c2d0..e10150617 100644
--- a/config-default.yml
+++ b/config-default.yml
@@ -72,6 +72,7 @@ urls:
     status: !ENV 'STATUS_URL'
     site: 'pythondiscord.com'
     site_hiphopify_api: 'https://api.pythondiscord.com/bot/hiphopify'
+    site_docs_api: 'https://api.pythondiscord.com/bot/docs'
     site_tags_api: 'https://api.pythondiscord.com/bot/tags'
     site_user_api: 'https://api.pythondiscord.com/bot/users'
     site_quiz_api: 'https://api.pythondiscord.com/bot/snake_quiz'