Add Sphinx-MultiVersion

Adds the sphinx-multiversion package to be used for generating docs
for all versions of the project, not just the latest. This includes
all the necessary configuration to make it work cleanly.

Signed-off-by: Hassan Abouelela <[email protected]>

7 files changed, 222 insertions, 18 deletions

diff --git a/docs/README.md b/docs/README.md
index 2146ce5b..16c9e8cc 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -3,6 +3,8 @@ Meta information about this project's documentation.
 
 Table of contents:
 - [Building the docs](#Building)
+- [Docs Layout](#Layout)
+- [Building all versions](#Versions)
 - [Writing docstrings](#Docstrings)
 - [Writing a changelog](#Changelog)
 
@@ -15,11 +17,26 @@ poetry run task docs
 
 The output will be in the [`/docs/build`](.) directory.
 
-Additionally, there are two helper tasks: `apidoc` and `builddoc`.
-`apidoc` is responsible for calling autodoc and generating docs from docstrings.
-`builddoc` generates the HTML site, and should be called after apidoc.
 
-Neither of these two tasks needs to be manually called, as the `docs` task calls both.
+## Versions
+The project supports building all different versions at once using [sphinx-multiversion][multiversion]
+after version `v7.1.0`. You can run the following command to achieve that:
+
+```shell
+poetry run sphinx_multiversion -v docs docs/build -n -j auto -n
+```
+
+This will build all tags, as well as the main branch. To build branches besides the main one
+(such as the one you are currently working on), set the `BUILD_DOCS_FOR_HEAD` environment variable
+to True.
+
+When using multi-version, keep the following in mind:
+1. This command will not fail on warnings, unlike the docs task. Make sure that passes first
+   before using this one.
+2. Make sure to clear the build directory before running this script to avoid conflicts.
+
+
+[multiversion]: https://holzhaus.github.io/sphinx-multiversion/master/index.html
 
 
 ## Docstrings
diff --git a/docs/_templates/base.html b/docs/_templates/base.html
new file mode 100644
index 00000000..541dbd0b
--- /dev/null
+++ b/docs/_templates/base.html
@@ -0,0 +1,12 @@
+{% extends "furo/base.html" %}
+
+{# Make sure the project name uses the correct version #}
+{% if versions %}
+    {% if current_version == latest_version %}
+        {% set docstitle = "Latest (" + current_version.version + ")" %}
+    {% else %}
+        {% set docstitle = current_version.name %}
+    {% endif %}
+
+    {% set docstitle = project + " " + docstitle %}
+{% endif %}
diff --git a/docs/_templates/sidebar/navigation.html b/docs/_templates/sidebar/navigation.html
new file mode 100644
index 00000000..02239887
--- /dev/null
+++ b/docs/_templates/sidebar/navigation.html
@@ -0,0 +1,46 @@
+<div class="sidebar-tree">
+    {{ furo_navigation_tree }}
+
+    {# Include a version navigation menu in the side bar #}
+    {% if versions %}
+    <ul>
+        <li class="toctree-l1 has-children {{ "current-page" if pagename == "versions" }}">
+            {# The following block is taken from furo's generated sidebar dropdown #}
+            <a class="reference internal" href="{{ pathto("versions") }}">Versions</a>
+            <input {{ "checked" if pagename == "versions" }} class="toctree-checkbox" id="toctree-checkbox-versions" name="toctree-checkbox-versions" role="switch" type="checkbox">
+            <label for="toctree-checkbox-versions">
+                <div class="visually-hidden">Toggle child pages in navigation</div>
+                <i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i>
+            </label>
+            {# End copied block #}
+
+            <ul>
+                {% for version in versions | reverse %}
+                    <li class="toctree-l2 {{ "current-page" if version == current_version }}">
+                        <a class="version_link reference internal" href="{{ version.url }}">{{ version.name }}</a>
+                    </li>
+                {% endfor %}
+
+                <script>
+                    // Make sure we keep any hyperlinked resources when switching version
+                    function updateHash() {
+                        for (let tag of document.getElementsByClassName("version_link")) {
+                            // Extract the original URL
+                            let destination = tag.getAttribute("href");
+                            if (destination.indexOf("#") !== -1) {
+                                destination = destination.slice(0, destination.indexOf("#"));
+                            }
+
+                            // Update the url with the current hash
+                            tag.setAttribute("href", destination + document.location.hash);
+                        }
+                    }
+
+                    updateHash();
+                    addEventListener("hashchange", _ => { updateHash() });
+                </script>
+            </ul>
+        </li>
+    </ul>
+    {% endif %}
+</div>
diff --git a/docs/conf.py b/docs/conf.py
index 1cea4026..12431235 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -3,10 +3,10 @@
 
 import functools
 import os.path
+import shutil
 import sys
 from pathlib import Path
 
-import git
 import releases
 import tomli
 from sphinx.application import Sphinx
@@ -22,10 +22,9 @@ project = "Bot Core"
 copyright = "2021, Python Discord"
 author = "Python Discord"
 
-PROJECT_ROOT = Path(__file__).parent.parent
 REPO_LINK = "https://github.com/python-discord/bot-core"
-SOURCE_FILE_LINK = f"{REPO_LINK}/blob/{git.Repo(PROJECT_ROOT).commit().hexsha}"
 
+PROJECT_ROOT = Path(__file__).parent.parent
 sys.path.insert(0, str(PROJECT_ROOT.absolute()))
 
 # The full version, including alpha/beta/rc tags
@@ -50,10 +49,11 @@ extensions = [
     "releases",
     "sphinx.ext.linkcode",
     "sphinx.ext.githubpages",
+    "sphinx_multiversion",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ["_templates"]
+templates_path = ["_templates", "pages"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -78,6 +78,13 @@ html_theme_options = {
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
+
+# Html files under pages/ are rendered separately and added to the final build
+html_additional_pages = {
+    file.removesuffix(".html"): file
+    for file in utils.get_recursive_file_uris(Path("pages"), "*.html")
+}
+
 html_title = f"{project} v{version}"
 html_short_title = project
 
@@ -88,7 +95,7 @@ static = Path("_static")
 html_css_files = utils.get_recursive_file_uris(static, "*.css")
 html_js_files = utils.get_recursive_file_uris(static, "*.js")
 
-utils.cleanup()
+utils.build_api_doc()
 
 
 def skip(*args) -> bool:
@@ -103,9 +110,26 @@ def skip(*args) -> bool:
     return would_skip
 
 
+def post_build(_: Sphinx, exception: Exception) -> None:
+    """Clean up and process files after the build has finished."""
+    if exception:
+        # Don't accidentally supress exceptions
+        raise exception from None
+
+    build_folder = PROJECT_ROOT / "docs" / "build"
+    main_build = build_folder / "main"
+
+    if main_build.exists() and not (build_folder / "index.html").exists():
+        # We don't have an index in the root folder, add a redirect
+        shutil.copy((main_build / "index_redirect.html"), (build_folder / "index.html"))
+        shutil.copytree((main_build / "_static"), (build_folder / "_static"), dirs_exist_ok=True)
+        (build_folder / ".nojekyll").touch(exist_ok=True)
+
+
 def setup(app: Sphinx) -> None:
     """Add extra hook-based autodoc configuration."""
     app.connect("autodoc-skip-member", skip)
+    app.connect("build-finished", post_build)
     app.add_role("literal-url", utils.emphasized_url)
 
     # Add a `breaking` role to the changelog
@@ -154,9 +178,19 @@ intersphinx_mapping = {
 
 
 # -- Options for the linkcode extension --------------------------------------
-linkcode_resolve = functools.partial(utils.linkcode_resolve, SOURCE_FILE_LINK)
+linkcode_resolve = functools.partial(utils.linkcode_resolve, REPO_LINK)
 
 
 # -- Options for releases extension ------------------------------------------
 releases_github_path = REPO_LINK.removeprefix("https://github.com/")
 releases_release_uri = f"{REPO_LINK}/releases/tag/v%s"
+
+
+# -- Options for the multiversion extension ----------------------------------
+# Only include local refs, filter out older versions, and don't build branches other than main
+# unless `BUILD_DOCS_FOR_HEAD` env variable is True.
+smv_remote_whitelist = None
+smv_latest_version = "main"
+if os.getenv("BUILD_DOCS_FOR_HEAD", "False").lower() == "false":
+    smv_branch_whitelist = "main"
+smv_tag_whitelist = r"v(?!([0-6]\.)|(7\.[0-1]\.0))"  # Don't include any versions prior to v7.1.1
diff --git a/docs/pages/index_redirect.html b/docs/pages/index_redirect.html
new file mode 100644
index 00000000..3745c62c
--- /dev/null
+++ b/docs/pages/index_redirect.html
@@ -0,0 +1,19 @@
+{% extends "base.html" %}
+{% set pagename = master_doc %}
+
+{# Redirect users to the actual docs page #}
+{% block site_meta %}
+    {# Make sure this is loaded as early as possible #}
+    <script>window.location.replace("./main/index.html")</script>
+    {{ super() }}
+{% endblock %}
+
+{# Show some text in-case the redirection fails #}
+{% block body %}
+    {{ super() }}
+    <div style="display: flex; text-align: center; justify-content: center; align-items: center; height: 100%;">
+        <h2><a href="./main/index.html">
+            Please click here if you were not redirected to the latest build.
+        </a></h2>
+    </div>
+{% endblock %}
diff --git a/docs/pages/versions.html b/docs/pages/versions.html
new file mode 100644
index 00000000..f4564dbf
--- /dev/null
+++ b/docs/pages/versions.html
@@ -0,0 +1,23 @@
+{% extends "page.html" %}
+{% set title = "Versions" %}
+
+{% block content -%}
+    {% if versions %}
+        <h1>Versions</h1>
+        <dl>
+            <dt>Documentation is available for the following versions:</dt>
+            <dd><ul>
+                {# List all avaialble versions #}
+                {% for version in versions | reverse %}
+                    <li>
+                        <a class="reference internal" href="{{ version.url }}">{{ version.name }}</a>
+                        {{ "(current)" if version == current_version }}
+                        {{ "- latest" if version == latest_version }}
+                    </li>
+                {% endfor %}
+            </ul></dd>
+        </dl>
+    {% else %}
+        <h1>No version information available!</h1>
+    {% endif %}
+{%- endblock %}
diff --git a/docs/utils.py b/docs/utils.py
index aaa656c3..bb8074ba 100644
--- a/docs/utils.py
+++ b/docs/utils.py
@@ -1,19 +1,31 @@
 """Utilities used in generating docs."""
 
 import ast
-import importlib
+import importlib.util
 import inspect
+import os
+import subprocess
 import typing
 from pathlib import Path
 
 import docutils.nodes
 import docutils.parsers.rst.states
+import git
 import releases
+import sphinx.util.logging
 
-PROJECT_ROOT = Path(__file__).parent.parent
+logger = sphinx.util.logging.getLogger(__name__)
 
 
-def linkcode_resolve(source_url: str, domain: str, info: dict[str, str]) -> typing.Optional[str]:
+def get_build_root() -> Path:
+    """Get the project root folder for the current build."""
+    root = Path.cwd()
+    if root.name == "docs":
+        root = root.parent
+    return root
+
+
+def linkcode_resolve(repo_link: str, domain: str, info: dict[str, str]) -> typing.Optional[str]:
     """
     Function called by linkcode to get the URL for a given resource.
 
@@ -25,7 +37,25 @@ def linkcode_resolve(source_url: str, domain: str, info: dict[str, str]) -> typi
 
     symbol_name = info["fullname"]
 
-    module = importlib.import_module(info["module"])
+    build_root = get_build_root()
+
+    # Import the package to find files
+    origin = build_root / info["module"].replace(".", "/")
+    search_locations = []
+
+    if origin.is_dir():
+        search_locations.append(origin.absolute().as_posix())
+        origin = origin / "__init__.py"
+    else:
+        origin = Path(origin.absolute().as_posix() + ".py")
+        if not origin.exists():
+            raise Exception(f"Could not find `{info['module']}` as a package or file.")
+
+    # We can't use a normal import (importlib.import_module), because the module can conflict with another copy
+    # in multiversion builds. We load the module from the file location instead
+    spec = importlib.util.spec_from_file_location(info["module"], origin, submodule_search_locations=search_locations)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
 
     symbol = [module]
     for name in symbol_name.split("."):
@@ -62,9 +92,15 @@ def linkcode_resolve(source_url: str, domain: str, info: dict[str, str]) -> typi
         start += offset
         end += offset
 
-    file = Path(inspect.getfile(module)).relative_to(PROJECT_ROOT).as_posix()
+    file = Path(inspect.getfile(module)).relative_to(build_root).as_posix()
+
+    try:
+        sha = git.Repo(build_root).commit().hexsha
+    except git.InvalidGitRepositoryError:
+        # We are building a historical version, no git data available
+        sha = build_root.name
 
-    url = f"{source_url}/{file}#L{start}"
+    url = f"{repo_link}/blob/{sha}/{file}#L{start}"
     if end != start:
         url += f"-L{end}"
 
@@ -75,7 +111,7 @@ def cleanup() -> None:
     """Remove unneeded autogenerated doc files, and clean up others."""
     included = __get_included()
 
-    for file in (PROJECT_ROOT / "docs" / "output").iterdir():
+    for file in (get_build_root() / "docs" / "output").iterdir():
         if file.name in ("botcore.rst", "botcore.exts.rst", "botcore.utils.rst") and file.name in included:
             content = file.read_text(encoding="utf-8").splitlines(keepends=True)
 
@@ -96,7 +132,6 @@ def cleanup() -> None:
 
         else:
             # These are files that have not been explicitly included in the docs via __all__
-            print("Deleted file", file.name)
             file.unlink()
             continue
 
@@ -105,6 +140,24 @@ def cleanup() -> None:
         file.write_text(content, encoding="utf-8")
 
 
+def build_api_doc() -> None:
+    """Generate auto-module directives using apidoc."""
+    cmd = os.getenv("APIDOC_COMMAND") or "sphinx-apidoc -o docs/output botcore -feM"
+    cmd = cmd.split()
+
+    build_root = get_build_root()
+    output_folder = build_root / cmd[cmd.index("-o") + 1]
+
+    if output_folder.exists():
+        logger.info(f"Skipping api-doc for {output_folder.as_posix()} as it already exists.")
+        return
+
+    result = subprocess.run(cmd, cwd=build_root, stdout=subprocess.PIPE, check=True, env=os.environ)
+    logger.debug("api-doc Output:\n" + result.stdout.decode(encoding="utf-8") + "\n")
+
+    cleanup()
+
+
 def __get_included() -> set[str]:
     """Get a list of files that should be included in the final build."""