From ed4b2fbea5c664a1950df66a534b3233efdc5c4e Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Wed, 11 May 2022 02:42:43 +0400 Subject: Restore Releases Changelog Add the releases-based changelog system back, and migrate the old changelog entries. Signed-off-by: Hassan Abouelela --- docs/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'docs/README.md') diff --git a/docs/README.md b/docs/README.md index fa719292..2146ce5b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -38,11 +38,11 @@ Refer to the [sphinx documentation][docstring-sections] for more information. Each change requires an entry in the [Changelog](./changelog.rst). Refer to the [Releases][releases] documentation for more information on the exact format and content of entries -You can use [this site][releases] to get the PR number you'll use for your entry. +You can use [this site][next] to get the PR number that'll be assigned for your entry. -[next]: https://ichard26.github.io/next-pr-number/?owner=python-discord&name=bot-core [releases]: https://releases.readthedocs.io/en/latest/concepts.html +[next]: https://ichard26.github.io/next-pr-number/?owner=python-discord&name=bot-core ## Static Builds We deploy our docs to netlify to power static previews on PRs. -- cgit v1.2.3 From 4c9cad2552ebeb96f747467017ef3155595a9d1c Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Wed, 11 May 2022 04:08:42 +0400 Subject: Add Sphinx-MultiVersion Adds the sphinx-multiversion package to be used for generating docs for all versions of the project, not just the latest. This includes all the necessary configuration to make it work cleanly. Signed-off-by: Hassan Abouelela --- docs/README.md | 25 ++++++++++-- docs/_templates/base.html | 12 ++++++ docs/_templates/sidebar/navigation.html | 46 ++++++++++++++++++++++ docs/conf.py | 46 +++++++++++++++++++--- docs/pages/index_redirect.html | 19 +++++++++ docs/pages/versions.html | 23 +++++++++++ docs/utils.py | 69 +++++++++++++++++++++++++++++---- poetry.lock | 17 +++++++- pyproject.toml | 7 ++-- 9 files changed, 241 insertions(+), 23 deletions(-) create mode 100644 docs/_templates/base.html create mode 100644 docs/_templates/sidebar/navigation.html create mode 100644 docs/pages/index_redirect.html create mode 100644 docs/pages/versions.html (limited to 'docs/README.md') diff --git a/docs/README.md b/docs/README.md index 2146ce5b..16c9e8cc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -3,6 +3,8 @@ Meta information about this project's documentation. Table of contents: - [Building the docs](#Building) +- [Docs Layout](#Layout) +- [Building all versions](#Versions) - [Writing docstrings](#Docstrings) - [Writing a changelog](#Changelog) @@ -15,11 +17,26 @@ poetry run task docs The output will be in the [`/docs/build`](.) directory. -Additionally, there are two helper tasks: `apidoc` and `builddoc`. -`apidoc` is responsible for calling autodoc and generating docs from docstrings. -`builddoc` generates the HTML site, and should be called after apidoc. -Neither of these two tasks needs to be manually called, as the `docs` task calls both. +## Versions +The project supports building all different versions at once using [sphinx-multiversion][multiversion] +after version `v7.1.0`. You can run the following command to achieve that: + +```shell +poetry run sphinx_multiversion -v docs docs/build -n -j auto -n +``` + +This will build all tags, as well as the main branch. To build branches besides the main one +(such as the one you are currently working on), set the `BUILD_DOCS_FOR_HEAD` environment variable +to True. + +When using multi-version, keep the following in mind: +1. This command will not fail on warnings, unlike the docs task. Make sure that passes first + before using this one. +2. Make sure to clear the build directory before running this script to avoid conflicts. + + +[multiversion]: https://holzhaus.github.io/sphinx-multiversion/master/index.html ## Docstrings diff --git a/docs/_templates/base.html b/docs/_templates/base.html new file mode 100644 index 00000000..541dbd0b --- /dev/null +++ b/docs/_templates/base.html @@ -0,0 +1,12 @@ +{% extends "furo/base.html" %} + +{# Make sure the project name uses the correct version #} +{% if versions %} + {% if current_version == latest_version %} + {% set docstitle = "Latest (" + current_version.version + ")" %} + {% else %} + {% set docstitle = current_version.name %} + {% endif %} + + {% set docstitle = project + " " + docstitle %} +{% endif %} diff --git a/docs/_templates/sidebar/navigation.html b/docs/_templates/sidebar/navigation.html new file mode 100644 index 00000000..02239887 --- /dev/null +++ b/docs/_templates/sidebar/navigation.html @@ -0,0 +1,46 @@ +
+ {{ furo_navigation_tree }} + + {# Include a version navigation menu in the side bar #} + {% if versions %} +
    +
  • + {# The following block is taken from furo's generated sidebar dropdown #} + Versions + + + {# End copied block #} + +
      + {% for version in versions | reverse %} +
    • + {{ version.name }} +
      • + {% endfor %} + + +
    +
    • +
+ {% endif %} +
diff --git a/docs/conf.py b/docs/conf.py index 1cea4026..12431235 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -3,10 +3,10 @@ import functools import os.path +import shutil import sys from pathlib import Path -import git import releases import tomli from sphinx.application import Sphinx @@ -22,10 +22,9 @@ project = "Bot Core" copyright = "2021, Python Discord" author = "Python Discord" -PROJECT_ROOT = Path(__file__).parent.parent REPO_LINK = "https://github.com/python-discord/bot-core" -SOURCE_FILE_LINK = f"{REPO_LINK}/blob/{git.Repo(PROJECT_ROOT).commit().hexsha}" +PROJECT_ROOT = Path(__file__).parent.parent sys.path.insert(0, str(PROJECT_ROOT.absolute())) # The full version, including alpha/beta/rc tags @@ -50,10 +49,11 @@ extensions = [ "releases", "sphinx.ext.linkcode", "sphinx.ext.githubpages", + "sphinx_multiversion", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] +templates_path = ["_templates", "pages"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -78,6 +78,13 @@ html_theme_options = { # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] + +# Html files under pages/ are rendered separately and added to the final build +html_additional_pages = { + file.removesuffix(".html"): file + for file in utils.get_recursive_file_uris(Path("pages"), "*.html") +} + html_title = f"{project} v{version}" html_short_title = project @@ -88,7 +95,7 @@ static = Path("_static") html_css_files = utils.get_recursive_file_uris(static, "*.css") html_js_files = utils.get_recursive_file_uris(static, "*.js") -utils.cleanup() +utils.build_api_doc() def skip(*args) -> bool: @@ -103,9 +110,26 @@ def skip(*args) -> bool: return would_skip +def post_build(_: Sphinx, exception: Exception) -> None: + """Clean up and process files after the build has finished.""" + if exception: + # Don't accidentally supress exceptions + raise exception from None + + build_folder = PROJECT_ROOT / "docs" / "build" + main_build = build_folder / "main" + + if main_build.exists() and not (build_folder / "index.html").exists(): + # We don't have an index in the root folder, add a redirect + shutil.copy((main_build / "index_redirect.html"), (build_folder / "index.html")) + shutil.copytree((main_build / "_static"), (build_folder / "_static"), dirs_exist_ok=True) + (build_folder / ".nojekyll").touch(exist_ok=True) + + def setup(app: Sphinx) -> None: """Add extra hook-based autodoc configuration.""" app.connect("autodoc-skip-member", skip) + app.connect("build-finished", post_build) app.add_role("literal-url", utils.emphasized_url) # Add a `breaking` role to the changelog @@ -154,9 +178,19 @@ intersphinx_mapping = { # -- Options for the linkcode extension -------------------------------------- -linkcode_resolve = functools.partial(utils.linkcode_resolve, SOURCE_FILE_LINK) +linkcode_resolve = functools.partial(utils.linkcode_resolve, REPO_LINK) # -- Options for releases extension ------------------------------------------ releases_github_path = REPO_LINK.removeprefix("https://github.com/") releases_release_uri = f"{REPO_LINK}/releases/tag/v%s" + + +# -- Options for the multiversion extension ---------------------------------- +# Only include local refs, filter out older versions, and don't build branches other than main +# unless `BUILD_DOCS_FOR_HEAD` env variable is True. +smv_remote_whitelist = None +smv_latest_version = "main" +if os.getenv("BUILD_DOCS_FOR_HEAD", "False").lower() == "false": + smv_branch_whitelist = "main" +smv_tag_whitelist = r"v(?!([0-6]\.)|(7\.[0-1]\.0))" # Don't include any versions prior to v7.1.1 diff --git a/docs/pages/index_redirect.html b/docs/pages/index_redirect.html new file mode 100644 index 00000000..3745c62c --- /dev/null +++ b/docs/pages/index_redirect.html @@ -0,0 +1,19 @@ +{% extends "base.html" %} +{% set pagename = master_doc %} + +{# Redirect users to the actual docs page #} +{% block site_meta %} + {# Make sure this is loaded as early as possible #} + + {{ super() }} +{% endblock %} + +{# Show some text in-case the redirection fails #} +{% block body %} + {{ super() }} +
+

+ Please click here if you were not redirected to the latest build. +

+
+{% endblock %} diff --git a/docs/pages/versions.html b/docs/pages/versions.html new file mode 100644 index 00000000..f4564dbf --- /dev/null +++ b/docs/pages/versions.html @@ -0,0 +1,23 @@ +{% extends "page.html" %} +{% set title = "Versions" %} + +{% block content -%} + {% if versions %} +

Versions

+
+
Documentation is available for the following versions:
+
    + {# List all avaialble versions #} + {% for version in versions | reverse %} +
  • + {{ version.name }} + {{ "(current)" if version == current_version }} + {{ "- latest" if version == latest_version }} +
    • + {% endfor %} +
+
+ {% else %} +

No version information available!

+ {% endif %} +{%- endblock %} diff --git a/docs/utils.py b/docs/utils.py index aaa656c3..bb8074ba 100644 --- a/docs/utils.py +++ b/docs/utils.py @@ -1,19 +1,31 @@ """Utilities used in generating docs.""" import ast -import importlib +import importlib.util import inspect +import os +import subprocess import typing from pathlib import Path import docutils.nodes import docutils.parsers.rst.states +import git import releases +import sphinx.util.logging -PROJECT_ROOT = Path(__file__).parent.parent +logger = sphinx.util.logging.getLogger(__name__) -def linkcode_resolve(source_url: str, domain: str, info: dict[str, str]) -> typing.Optional[str]: +def get_build_root() -> Path: + """Get the project root folder for the current build.""" + root = Path.cwd() + if root.name == "docs": + root = root.parent + return root + + +def linkcode_resolve(repo_link: str, domain: str, info: dict[str, str]) -> typing.Optional[str]: """ Function called by linkcode to get the URL for a given resource. @@ -25,7 +37,25 @@ def linkcode_resolve(source_url: str, domain: str, info: dict[str, str]) -> typi symbol_name = info["fullname"] - module = importlib.import_module(info["module"]) + build_root = get_build_root() + + # Import the package to find files + origin = build_root / info["module"].replace(".", "/") + search_locations = [] + + if origin.is_dir(): + search_locations.append(origin.absolute().as_posix()) + origin = origin / "__init__.py" + else: + origin = Path(origin.absolute().as_posix() + ".py") + if not origin.exists(): + raise Exception(f"Could not find `{info['module']}` as a package or file.") + + # We can't use a normal import (importlib.import_module), because the module can conflict with another copy + # in multiversion builds. We load the module from the file location instead + spec = importlib.util.spec_from_file_location(info["module"], origin, submodule_search_locations=search_locations) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) symbol = [module] for name in symbol_name.split("."): @@ -62,9 +92,15 @@ def linkcode_resolve(source_url: str, domain: str, info: dict[str, str]) -> typi start += offset end += offset - file = Path(inspect.getfile(module)).relative_to(PROJECT_ROOT).as_posix() + file = Path(inspect.getfile(module)).relative_to(build_root).as_posix() + + try: + sha = git.Repo(build_root).commit().hexsha + except git.InvalidGitRepositoryError: + # We are building a historical version, no git data available + sha = build_root.name - url = f"{source_url}/{file}#L{start}" + url = f"{repo_link}/blob/{sha}/{file}#L{start}" if end != start: url += f"-L{end}" @@ -75,7 +111,7 @@ def cleanup() -> None: """Remove unneeded autogenerated doc files, and clean up others.""" included = __get_included() - for file in (PROJECT_ROOT / "docs" / "output").iterdir(): + for file in (get_build_root() / "docs" / "output").iterdir(): if file.name in ("botcore.rst", "botcore.exts.rst", "botcore.utils.rst") and file.name in included: content = file.read_text(encoding="utf-8").splitlines(keepends=True) @@ -96,7 +132,6 @@ def cleanup() -> None: else: # These are files that have not been explicitly included in the docs via __all__ - print("Deleted file", file.name) file.unlink() continue @@ -105,6 +140,24 @@ def cleanup() -> None: file.write_text(content, encoding="utf-8") +def build_api_doc() -> None: + """Generate auto-module directives using apidoc.""" + cmd = os.getenv("APIDOC_COMMAND") or "sphinx-apidoc -o docs/output botcore -feM" + cmd = cmd.split() + + build_root = get_build_root() + output_folder = build_root / cmd[cmd.index("-o") + 1] + + if output_folder.exists(): + logger.info(f"Skipping api-doc for {output_folder.as_posix()} as it already exists.") + return + + result = subprocess.run(cmd, cwd=build_root, stdout=subprocess.PIPE, check=True, env=os.environ) + logger.debug("api-doc Output:\n" + result.stdout.decode(encoding="utf-8") + "\n") + + cleanup() + + def __get_included() -> set[str]: """Get a list of files that should be included in the final build.""" diff --git a/poetry.lock b/poetry.lock index af24501a..18c91773 100644 --- a/poetry.lock +++ b/poetry.lock @@ -892,6 +892,17 @@ Sphinx = ">=4.5" testing = ["covdefaults (>=2.2)", "coverage (>=6.3)", "diff-cover (>=6.4)", "nptyping (>=2)", "pytest (>=7.1)", "pytest-cov (>=3)", "sphobjinv (>=2)", "typing-extensions (>=4.1)"] type_comments = ["typed-ast (>=1.5.2)"] +[[package]] +name = "sphinx-multiversion" +version = "0.2.4" +description = "Add support for multiple versions to sphinx" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +sphinx = ">=2.1" + [[package]] name = "sphinxcontrib-applehelp" version = "1.0.2" @@ -1078,7 +1089,7 @@ async-rediscache = ["async-rediscache"] [metadata] lock-version = "1.1" python-versions = "3.9.*" -content-hash = "b4b7d5cf3933b4237051c02050f3dadefa58ad39e5cebc7434f097b7315ea179" +content-hash = "c59b21365534625a127a8bfaffd41a704714c3241817f4d71ad86b68602cf930" [metadata.files] aiohttp = [ @@ -1771,6 +1782,10 @@ sphinx-autodoc-typehints = [ {file = "sphinx_autodoc_typehints-1.18.1-py3-none-any.whl", hash = "sha256:f8f5bb7c13a9a71537dc2be2eb3b9e28a9711e2454df63587005eacf6fbac453"}, {file = "sphinx_autodoc_typehints-1.18.1.tar.gz", hash = "sha256:07631c5f0c6641e5ba27143494aefc657e029bed3982138d659250e617f6f929"}, ] +sphinx-multiversion = [ + {file = "sphinx-multiversion-0.2.4.tar.gz", hash = "sha256:5cd1ca9ecb5eed63cb8d6ce5e9c438ca13af4fa98e7eb6f376be541dd4990bcb"}, + {file = "sphinx_multiversion-0.2.4-py3-none-any.whl", hash = "sha256:dec29f2a5890ad68157a790112edc0eb63140e70f9df0a363743c6258fbeb478"}, +] sphinxcontrib-applehelp = [ {file = "sphinxcontrib-applehelp-1.0.2.tar.gz", hash = "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58"}, {file = "sphinxcontrib_applehelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a"}, diff --git a/pyproject.toml b/pyproject.toml index 4ab1af72..4290417f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "bot-core" -version = "7.1.0" +version = "7.1.1" description = "Bot-Core provides the core functionality and utilities for the bots of the Python Discord community." authors = ["Python Discord "] license = "MIT" @@ -44,13 +44,12 @@ GitPython = "3.1.27" sphinx-autodoc-typehints = "1.18.1" furo = "2022.4.7" releases = "1.6.3" +sphinx-multiversion = "0.2.4" [tool.taskipy.tasks] lint = "pre-commit run --all-files" precommit = "pre-commit install" -apidoc = "sphinx-apidoc -o docs/output botcore -feM" -builddoc = "sphinx-build -nW -j auto -b html docs docs/build" -docs = "task apidoc && task builddoc" +docs = "sphinx-build -nW -j auto -b html docs docs/build" test = "pytest -n 8 --ff" retest = "pytest -n 8 --lf" -- cgit v1.2.3 From 8f2067b0f2f9cd6001a9565f8da4590f5b414f1e Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sun, 29 May 2022 16:27:22 +0400 Subject: Document The Docs Folder Add a brief explanation about each item in the docs folder, to help contributors navigate and understand what each part does. Signed-off-by: Hassan Abouelela --- docs/README.md | 19 +++++++++++++++++++ docs/changelog.rst | 1 + 2 files changed, 20 insertions(+) (limited to 'docs/README.md') diff --git a/docs/README.md b/docs/README.md index 16c9e8cc..89124d5e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,6 +18,25 @@ poetry run task docs The output will be in the [`/docs/build`](.) directory. +## Layout +The docs folder has a few moving components, here's a brief description of each: + +- `_static`: This folder includes extra resources that are copied blindly by sphinx into the result + making it perfect for resources such as custom CSS or JS. +- `_templates` & `pages`: Both are considered HTML templates, and passed to Sphinx as `templates_path`. + The difference between them is that `_templates` is only used to provide templates and overrides that + are used by other pages, while `pages` are full-blown independent pages that will be included in the + result. Files in `pages` are passed to Sphinx as `html_additional_pages`, and will maintain the same + structure as the folder. That is to say, a file such as `pages/a/b.html` will create `https://bot-core/a/b.html`. +- `changelog.rst`: This contains a list of all the project's changes. Please refer to [Changelog](#Changelog) + below for more info. +- `index.rst`: The main content for the project's homepage. +- `conf.py`: Configuration for Sphinx. This includes things such as the project name, version, + plugins and their configuration. +- `utils.py`: Helpful function used by `conf.py` to properly create the docs. +- `netliy_build.py`: Script which downloads the build output in netlify. Refer to [Static Netlify Build](#static-builds) + + ## Versions The project supports building all different versions at once using [sphinx-multiversion][multiversion] after version `v7.1.0`. You can run the following command to achieve that: diff --git a/docs/changelog.rst b/docs/changelog.rst index 3cee3a7a..a4c60ef3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog ========= +- :support:`79` Add `sphinx-multiversion `_ to make available older doc versions. - :support:`79` Restore on-site changelog. -- cgit v1.2.3