diff options
| -rw-r--r-- | bot/exts/info/doc/_batch_parser.py | 32 | ||||
| -rw-r--r-- | bot/exts/info/doc/_cog.py | 108 | ||||
| -rw-r--r-- | bot/exts/info/doc/_parsing.py | 2 |
3 files changed, 71 insertions, 71 deletions
diff --git a/bot/exts/info/doc/_batch_parser.py b/bot/exts/info/doc/_batch_parser.py index 072545e66..2edf05ff0 100644 --- a/bot/exts/info/doc/_batch_parser.py +++ b/bot/exts/info/doc/_batch_parser.py @@ -33,27 +33,27 @@ class StaleInventoryNotifier: await bot.instance.wait_until_guild_available() self._dev_log = bot.instance.get_channel(Channels.dev_log) - async def send_warning(self, item: _cog.DocItem) -> None: + async def send_warning(self, doc_item: _cog.DocItem) -> None: """Send a warning to dev log is one wasn't already sent for `item`'s url.""" - if item.url not in self._warned_urls: - self._warned_urls.add(item.url) + if doc_item.url not in self._warned_urls: + self._warned_urls.add(doc_item.url) await self._init_task embed = discord.Embed( - description=f"Doc item `{item.symbol_id=}` present in loaded documentation inventories " - f"not found on [site]({item.url}), inventories may need to be refreshed." + description=f"Doc item `{doc_item.symbol_id=}` present in loaded documentation inventories " + f"not found on [site]({doc_item.url}), inventories may need to be refreshed." ) await self._dev_log.send(embed=embed) class QueueItem(NamedTuple): - """Contains a symbol and the BeautifulSoup object needed to parse it.""" + """Contains a doc_item and the BeautifulSoup object needed to parse it.""" - symbol: _cog.DocItem + doc_item: _cog.DocItem soup: BeautifulSoup def __eq__(self, other: Union[QueueItem, _cog.DocItem]): if isinstance(other, _cog.DocItem): - return self.symbol == other + return self.doc_item == other return NamedTuple.__eq__(self, other) @@ -83,14 +83,14 @@ class BatchParser: """ Get the Markdown of all symbols on a page and send them to redis when a symbol is requested. - DocItems are added through the `add_item` method which adds them to the `_page_symbols` dict. + DocItems are added through the `add_item` method which adds them to the `_page_doc_items` dict. `get_markdown` is used to fetch the Markdown; when this is used for the first time on a page, all of the symbols are queued to be parsed to avoid multiple web requests to the same page. """ def __init__(self): self._queue: List[QueueItem] = [] - self._page_symbols: Dict[str, List[_cog.DocItem]] = defaultdict(list) + self._page_doc_items: Dict[str, List[_cog.DocItem]] = defaultdict(list) self._item_futures: Dict[_cog.DocItem, ParseResultFuture] = {} self._parse_task = None @@ -109,14 +109,14 @@ class BatchParser: Not safe to run while `self.clear` is running. """ if doc_item not in self._item_futures: - self._item_futures.update((symbol, ParseResultFuture()) for symbol in self._page_symbols[doc_item.url]) + self._item_futures.update((item, ParseResultFuture()) for item in self._page_doc_items[doc_item.url]) self._item_futures[doc_item].user_requested = True async with bot.instance.http_session.get(doc_item.url) as response: soup = BeautifulSoup(await response.text(encoding="utf8"), "lxml") - self._queue.extend(QueueItem(symbol, soup) for symbol in self._page_symbols[doc_item.url]) - log.debug(f"Added symbols from {doc_item.url} to parse queue.") + self._queue.extend(QueueItem(item, soup) for item in self._page_doc_items[doc_item.url]) + log.debug(f"Added items from {doc_item.url} to parse queue.") if self._parse_task is None: self._parse_task = asyncio.create_task(self._parse_queue()) @@ -139,7 +139,7 @@ class BatchParser: item, soup = self._queue.pop() try: if (future := self._item_futures[item]).done(): - # Some items are present in the inventories multiple times under different symbols, + # Some items are present in the inventories multiple times under different symbol names, # if we already parsed an equal item, we can just skip it. continue @@ -173,7 +173,7 @@ class BatchParser: def add_item(self, doc_item: _cog.DocItem) -> None: """Map a DocItem to its page so that the symbol will be parsed once the page is requested.""" - self._page_symbols[doc_item.url].append(doc_item) + self._page_doc_items[doc_item.url].append(doc_item) async def clear(self) -> None: """ @@ -186,7 +186,7 @@ class BatchParser: if self._parse_task is not None: self._parse_task.cancel() self._queue.clear() - self._page_symbols.clear() + self._page_doc_items.clear() self._item_futures.clear() async def _cleanup_futures(self) -> None: diff --git a/bot/exts/info/doc/_cog.py b/bot/exts/info/doc/_cog.py index 16baa6320..0ff775ac7 100644 --- a/bot/exts/info/doc/_cog.py +++ b/bot/exts/info/doc/_cog.py @@ -92,31 +92,31 @@ class DocCog(commands.Cog): self.base_urls[api_package_name] = base_url for group, items in package.items(): - for symbol, relative_doc_url in items: + for symbol_name, relative_doc_url in items: # e.g. get 'class' from 'py:class' group_name = group.split(":")[1] - if (original_symbol := self.doc_symbols.get(symbol)) is not None: + if (original_item := self.doc_symbols.get(symbol_name)) is not None: replaced_symbol_name = self.ensure_unique_symbol_name( api_package_name, group_name, - original_symbol, - symbol, + original_item, + symbol_name, ) if replaced_symbol_name is not None: - symbol = replaced_symbol_name + symbol_name = replaced_symbol_name relative_url_path, _, symbol_id = relative_doc_url.partition("#") # Intern fields that have shared content so we're not storing unique strings for every object - symbol_item = DocItem( + doc_item = DocItem( api_package_name, sys.intern(group_name), base_url, sys.intern(relative_url_path), - symbol_id + symbol_id, ) - self.doc_symbols[symbol] = symbol_item - self.item_fetcher.add_item(symbol_item) + self.doc_symbols[symbol_name] = doc_item + self.item_fetcher.add_item(doc_item) log.trace(f"Fetched inventory for {api_package_name}.") @@ -124,7 +124,7 @@ class DocCog(commands.Cog): self, api_package_name: str, base_url: str, - inventory_url: str + inventory_url: str, ) -> None: """ Update the cog's inventory, or reschedule this method to execute again if the remote inventory unreachable. @@ -144,7 +144,7 @@ class DocCog(commands.Cog): self.inventory_scheduler.schedule_later( delay*60, api_package_name, - self.update_or_reschedule_inventory(api_package_name, base_url, inventory_url) + self.update_or_reschedule_inventory(api_package_name, base_url, inventory_url), ) else: self.update_single(api_package_name, base_url, package) @@ -154,7 +154,7 @@ class DocCog(commands.Cog): package_name: str, group_name: str, original_item: DocItem, - symbol_name: str + symbol_name: str, ) -> Optional[str]: """ Ensure `symbol_name` doesn't overwrite an another symbol in `doc_symbols`. @@ -166,42 +166,42 @@ class DocCog(commands.Cog): """ # Certain groups are added as prefixes to disambiguate the symbols. if group_name in FORCE_PREFIX_GROUPS: - new_symbol = f"{group_name}.{symbol_name}" - if new_symbol in self.doc_symbols: + new_symbol_name = f"{group_name}.{symbol_name}" + if new_symbol_name in self.doc_symbols: # If there's still a conflict, prefix with package name. - new_symbol = f"{package_name}.{new_symbol}" - self.renamed_symbols[symbol_name].append(new_symbol) - return new_symbol + new_symbol_name = f"{package_name}.{new_symbol_name}" + self.renamed_symbols[symbol_name].append(new_symbol_name) + return new_symbol_name # The existing symbol with which the current symbol conflicts should have a group prefix. # It currently doesn't have the group prefix because it's only added once there's a conflict. elif (original_symbol_group := original_item.group) in FORCE_PREFIX_GROUPS: - overridden_symbol = f"{original_symbol_group}.{symbol_name}" - if overridden_symbol in self.doc_symbols: + overridden_symbol_name = f"{original_symbol_group}.{symbol_name}" + if overridden_symbol_name in self.doc_symbols: # If there's still a conflict, prefix with package name. - overridden_symbol = f"{original_item.package}.{overridden_symbol}" + overridden_symbol_name = f"{original_item.package}.{overridden_symbol_name}" - self.doc_symbols[overridden_symbol] = original_item - self.renamed_symbols[symbol_name].append(overridden_symbol) + self.doc_symbols[overridden_symbol_name] = original_item + self.renamed_symbols[symbol_name].append(overridden_symbol_name) elif package_name in PRIORITY_PACKAGES: - overridden_symbol = f"{original_item.package}.{symbol_name}" - if overridden_symbol in self.doc_symbols: + overridden_symbol_name = f"{original_item.package}.{symbol_name}" + if overridden_symbol_name in self.doc_symbols: # If there's still a conflict, add the symbol's group in the middle. - overridden_symbol = f"{original_item.package}.{original_item.group}.{symbol_name}" + overridden_symbol_name = f"{original_item.package}.{original_item.group}.{symbol_name}" - self.doc_symbols[overridden_symbol] = original_item - self.renamed_symbols[symbol_name].append(overridden_symbol) + self.doc_symbols[overridden_symbol_name] = original_item + self.renamed_symbols[symbol_name].append(overridden_symbol_name) # If we can't specially handle the symbol through its group or package, # fall back to prepending its package name to the front. else: - new_symbol = f"{package_name}.{symbol_name}" - if new_symbol in self.doc_symbols: + new_symbol_name = f"{package_name}.{symbol_name}" + if new_symbol_name in self.doc_symbols: # If there's still a conflict, add the symbol's group in the middle. - new_symbol = f"{package_name}.{group_name}.{symbol_name}" - self.renamed_symbols[symbol_name].append(new_symbol) - return new_symbol + new_symbol_name = f"{package_name}.{group_name}.{symbol_name}" + self.renamed_symbols[symbol_name].append(new_symbol_name) + return new_symbol_name async def refresh_inventory(self) -> None: """Refresh internal documentation inventory.""" @@ -229,7 +229,7 @@ class DocCog(commands.Cog): log.debug("Finished inventory refresh.") self.refresh_event.set() - async def get_symbol_embed(self, symbol: str) -> Optional[discord.Embed]: + async def get_symbol_embed(self, symbol_name: str) -> Optional[discord.Embed]: """ Attempt to scrape and fetch the data for the given `symbol`, and build an embed from its contents. @@ -237,47 +237,47 @@ class DocCog(commands.Cog): First check the DocRedisCache before querying the cog's `BatchParser`. """ - log.trace(f"Building embed for symbol `{symbol}`") + log.trace(f"Building embed for symbol `{symbol_name}`") if not self.refresh_event.is_set(): log.debug("Waiting for inventories to be refreshed before processing item.") await self.refresh_event.wait() - symbol_info = self.doc_symbols.get(symbol) - if symbol_info is None: - if symbol.count(" "): + doc_item = self.doc_symbols.get(symbol_name) + if doc_item is None: + if symbol_name.count(" "): # If an invalid symbol contains a space, check if the command was invoked # in the format !d <symbol> <message> - symbol = symbol.split(" ", maxsplit=1)[0] - symbol_info = self.doc_symbols.get(symbol) - if symbol_info is None: + symbol_name = symbol_name.split(" ", maxsplit=1)[0] + doc_item = self.doc_symbols.get(symbol_name) + if doc_item is None: log.debug("Symbol does not exist.") return None else: log.debug("Symbol does not exist.") return None - self.bot.stats.incr(f"doc_fetches.{symbol_info.package}") + self.bot.stats.incr(f"doc_fetches.{doc_item.package}") with self.symbol_get_event: - markdown = await doc_cache.get(symbol_info) + markdown = await doc_cache.get(doc_item) if markdown is None: - log.debug(f"Redis cache miss for symbol `{symbol}`.") - markdown = await self.item_fetcher.get_markdown(symbol_info) + log.debug(f"Redis cache miss for symbol `{symbol_name}`.") + markdown = await self.item_fetcher.get_markdown(doc_item) if markdown is not None: - await doc_cache.set(symbol_info, markdown) + await doc_cache.set(doc_item, markdown) else: markdown = "Unable to parse the requested symbol." embed = discord.Embed( - title=discord.utils.escape_markdown(symbol), - url=f"{symbol_info.url}#{symbol_info.symbol_id}", + title=discord.utils.escape_markdown(symbol_name), + url=f"{doc_item.url}#{doc_item.symbol_id}", description=markdown ) # Show all symbols with the same name that were renamed in the footer, # with a max of 100 chars. - if symbol in self.renamed_symbols: - renamed_symbols = ', '.join(self.renamed_symbols[symbol]) + if symbol_name in self.renamed_symbols: + renamed_symbols = ', '.join(self.renamed_symbols[symbol_name]) footer_text = textwrap.shorten("Moved: " + renamed_symbols, 100, placeholder=' ...') else: footer_text = "" @@ -285,12 +285,12 @@ class DocCog(commands.Cog): return embed @commands.group(name='docs', aliases=('doc', 'd'), invoke_without_command=True) - async def docs_group(self, ctx: commands.Context, *, symbol: Optional[str]) -> None: + async def docs_group(self, ctx: commands.Context, *, symbol_name: Optional[str]) -> None: """Look up documentation for Python symbols.""" - await self.get_command(ctx, symbol=symbol) + await self.get_command(ctx, symbol_name=symbol_name) @docs_group.command(name='getdoc', aliases=('g',)) - async def get_command(self, ctx: commands.Context, *, symbol: Optional[str]) -> None: + async def get_command(self, ctx: commands.Context, *, symbol_name: Optional[str]) -> None: """ Return a documentation embed for a given symbol. @@ -302,7 +302,7 @@ class DocCog(commands.Cog): !docs aiohttp.ClientSession !docs getdoc aiohttp.ClientSession """ - if not symbol: + if not symbol_name: inventory_embed = discord.Embed( title=f"All inventories (`{len(self.base_urls)}` total)", colour=discord.Colour.blue() @@ -317,7 +317,7 @@ class DocCog(commands.Cog): await ctx.send(embed=inventory_embed) else: - symbol = symbol.strip("`") + symbol = symbol_name.strip("`") # Fetching documentation for a symbol (at least for the first time, since # caching is used) takes quite some time, so let's send typing to indicate # that we got the command, but are still working on it. diff --git a/bot/exts/info/doc/_parsing.py b/bot/exts/info/doc/_parsing.py index 0251b0105..8e1b4d7a1 100644 --- a/bot/exts/info/doc/_parsing.py +++ b/bot/exts/info/doc/_parsing.py @@ -232,7 +232,7 @@ def _create_markdown(signatures: Optional[List[str]], description: Iterable[Tag] def get_symbol_markdown(soup: BeautifulSoup, symbol_data: DocItem) -> Optional[str]: """ - Return parsed markdown of the passed symbol using the passed in soup, truncated to fit within a discord message. + Return parsed markdown of the passed item using the passed in soup, truncated to fit within a discord message. The method of parsing and what information gets included depends on the symbol's group. """ |