Update how we auto-generate docs

2 files changed, 132 insertions, 103 deletions

diff --git a/docs/conf.py b/docs/conf.py
index 142ceb41..e2801e2c 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -1,16 +1,14 @@
 # Configuration file for the Sphinx documentation builder.
 # https://www.sphinx-doc.org/en/master/usage/configuration.html
 
-import ast
-import importlib
-import inspect
+import functools
 import sys
-import typing
 from pathlib import Path
 
 import git
 import tomli
-from sphinx.application import Sphinx
+
+from docs import utils
 
 # -- Project information -----------------------------------------------------
 
@@ -90,43 +88,6 @@ html_js_files = [
 ]
 
 
-# -- Autodoc cleanup ---------------------------------------------------------
-# Clean up the output generated by autodoc to produce a nicer documentation tree
-# This is kept in a function to avoid polluting the namespace
-def __cleanup() -> None:
-    for file in (PROJECT_ROOT / "docs" / "output").iterdir():
-        if file.name == "modules.rst":
-            # We only have one module, so this is redundant
-            # Remove it and flatten out the tree
-            file.unlink()
-            continue
-
-        elif file.name in ("botcore.rst", "botcore.exts.rst"):
-            content = file.read_text(encoding="utf-8").splitlines(keepends=True)
-
-            # Rename the extension to be less wordy
-            # Example: botcore.exts -> Botcore Exts
-            title = content[0].split()[0].strip().replace("botcore.", "").replace(".", " ").title()
-            title = f"{title}\n{'=' * len(title)}\n\n"
-            content[0:2] = title
-
-            file.write_text("".join(content), encoding="utf-8")
-
-        else:
-            # Clean up the submodule name so it's just the name without the top level module name
-            # example: `botcore.regex module` -> `regex`
-            lines = file.read_text(encoding="utf-8").splitlines(keepends=True)
-            lines[0] = lines[0].replace("module", "").strip().split(".")[-1] + "\n"
-            file.write_text("".join(lines))
-
-        # Take the opportunity to configure autodoc
-        content = file.read_text(encoding="utf-8").replace("undoc-members", "special-members")
-        file.write_text(content, encoding="utf-8")
-
-
-__cleanup()
-
-
 def skip(*args) -> bool:
     """Things that should be skipped by the autodoc generation."""
     name = args[2]
@@ -139,11 +100,6 @@ def skip(*args) -> bool:
     return would_skip
 
 
-def setup(app: Sphinx) -> None:
-    """Add extra hook-based autodoc configuration."""
-    app.connect("autodoc-skip-member", skip)
-
-
 # -- Extension configuration -------------------------------------------------
 
 # -- Options for todo extension ----------------------------------------------
@@ -175,60 +131,16 @@ intersphinx_mapping = {
 }
 
 
+# -- Options for the autodoc extension ---------------------------------------
+utils.cleanup()
+autodoc_default_options = {
+    "members": True,
+    "special-members": True,
+    "show-inheritance": True,
+    "imported-members": False,
+    "exclude-members": "__weakref__"
+}
+
+
 # -- Options for the linkcode extension --------------------------------------
-def linkcode_resolve(domain: str, info: dict[str, str]) -> typing.Optional[str]:
-    """
-    Function called by linkcode to get the URL for a given resource.
-
-    See for more details:
-    https://www.sphinx-doc.org/en/master/usage/extensions/linkcode.html#confval-linkcode_resolve
-    """
-    if domain != "py":
-        raise Exception("Unknown domain passed to linkcode function.")
-
-    symbol_name = info["fullname"]
-
-    module = importlib.import_module(info["module"])
-
-    symbol = [module]
-    for name in symbol_name.split("."):
-        symbol.append(getattr(symbol[-1], name))
-        symbol_name = name
-
-    try:
-        lines, start = inspect.getsourcelines(symbol[-1])
-        end = start + len(lines)
-    except TypeError:
-        # Find variables by parsing the ast
-        source = ast.parse(inspect.getsource(symbol[-2]))
-        while isinstance(source.body[0], ast.ClassDef):
-            source = source.body[0]
-
-        for ast_obj in source.body:
-            if isinstance(ast_obj, ast.Assign):
-                names = []
-                for target in ast_obj.targets:
-                    if isinstance(target, ast.Tuple):
-                        names.extend([name.id for name in target.elts])
-                    else:
-                        names.append(target.id)
-
-                if symbol_name in names:
-                    start, end = ast_obj.lineno, ast_obj.end_lineno
-                    break
-        else:
-            raise Exception(f"Could not find symbol `{symbol_name}` in {module.__name__}.")
-
-        _, offset = inspect.getsourcelines(symbol[-2])
-        if offset != 0:
-            offset -= 1
-        start += offset
-        end += offset
-
-    file = Path(inspect.getfile(module)).relative_to(PROJECT_ROOT).as_posix()
-
-    url = f"{SOURCE_FILE_LINK}/{file}#L{start}"
-    if end != start:
-        url += f"-L{end}"
-
-    return url
+linkcode_resolve = functools.partial(utils.linkcode_resolve, SOURCE_FILE_LINK)
diff --git a/docs/utils.py b/docs/utils.py
new file mode 100644
index 00000000..8bc69ccd
--- /dev/null
+++ b/docs/utils.py
@@ -0,0 +1,117 @@
+"""Utilities used in generating docs."""
+
+import ast
+import importlib
+import inspect
+import typing
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).parent.parent
+
+
+def linkcode_resolve(source_url: str, domain: str, info: dict[str, str]) -> typing.Optional[str]:
+    """
+    Function called by linkcode to get the URL for a given resource.
+
+    See for more details:
+    https://www.sphinx-doc.org/en/master/usage/extensions/linkcode.html#confval-linkcode_resolve
+    """
+    if domain != "py":
+        raise Exception("Unknown domain passed to linkcode function.")
+
+    symbol_name = info["fullname"]
+
+    module = importlib.import_module(info["module"])
+
+    symbol = [module]
+    for name in symbol_name.split("."):
+        symbol.append(getattr(symbol[-1], name))
+        symbol_name = name
+
+    try:
+        lines, start = inspect.getsourcelines(symbol[-1])
+        end = start + len(lines)
+    except TypeError:
+        # Find variables by parsing the ast
+        source = ast.parse(inspect.getsource(symbol[-2]))
+        while isinstance(source.body[0], ast.ClassDef):
+            source = source.body[0]
+
+        for ast_obj in source.body:
+            if isinstance(ast_obj, ast.Assign):
+                names = []
+                for target in ast_obj.targets:
+                    if isinstance(target, ast.Tuple):
+                        names.extend([name.id for name in target.elts])
+                    else:
+                        names.append(target.id)
+
+                if symbol_name in names:
+                    start, end = ast_obj.lineno, ast_obj.end_lineno
+                    break
+        else:
+            raise Exception(f"Could not find symbol `{symbol_name}` in {module.__name__}.")
+
+        _, offset = inspect.getsourcelines(symbol[-2])
+        if offset != 0:
+            offset -= 1
+        start += offset
+        end += offset
+
+    file = Path(inspect.getfile(module)).relative_to(PROJECT_ROOT).as_posix()
+
+    url = f"{source_url}/{file}#L{start}"
+    if end != start:
+        url += f"-L{end}"
+
+    return url
+
+
+def cleanup() -> None:
+    """Remove unneeded autogenerated doc files, and clean up others."""
+    included = __get_included()
+
+    for file in (PROJECT_ROOT / "docs" / "output").iterdir():
+        if file.name in ("botcore.rst", "botcore.exts.rst") and file.name in included:
+            content = file.read_text(encoding="utf-8").splitlines(keepends=True)
+
+            # Rename the extension to be less wordy
+            # Example: botcore.exts -> Botcore Exts
+            title = content[0].split()[0].strip().replace("botcore.", "").replace(".", " ").title()
+            title = f"{title}\n{'=' * len(title)}\n\n"
+            content[0:2] = title
+
+            file.write_text("".join(content), encoding="utf-8")
+
+        elif file.name in included:
+            # Clean up the submodule name so it's just the name without the top level module name
+            # example: `botcore.regex module` -> `regex`
+            lines = file.read_text(encoding="utf-8").splitlines(keepends=True)
+            lines[0] = lines[0].replace("module", "").strip().split(".")[-1] + "\n"
+            file.write_text("".join(lines))
+
+        else:
+            # These are files that have not been explicitly included in the docs via __all__
+            print("Deleted file", file.name)
+            file.unlink()
+            continue
+
+        # Take the opportunity to configure autodoc
+        content = file.read_text(encoding="utf-8").replace("undoc-members", "special-members")
+        file.write_text(content, encoding="utf-8")
+
+
+def __get_included() -> set[str]:
+    """Get a list of files that should be included in the final build."""
+
+    def get_all_from_module(module_name: str) -> set[str]:
+        module = importlib.import_module(module_name)
+        _modules = {module.__name__ + ".rst"}
+
+        if hasattr(module, "__all__"):
+            for sub_module in module.__all__:
+                _modules.update(get_all_from_module(sub_module))
+
+        return _modules
+
+    return get_all_from_module("botcore")