aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorGravatar Numerlor <[email protected]>2021-02-03 11:49:53 +0100
committerGravatar Numerlor <[email protected]>2021-02-03 12:00:37 +0100
commit1df9459c2c9c33f3e6dbcbd1b76415610aade192 (patch)
treef0bd52de0c9b4d0525ee719be0bf6b37d60342a0
parentImport module directly to avoid circular dependency and use of type (diff)
Use more consistent naming for doc items and symbols
Trailing commas were also added where missing
-rw-r--r--bot/exts/info/doc/_batch_parser.py32
-rw-r--r--bot/exts/info/doc/_cog.py108
-rw-r--r--bot/exts/info/doc/_parsing.py2
3 files changed, 71 insertions, 71 deletions
diff --git a/bot/exts/info/doc/_batch_parser.py b/bot/exts/info/doc/_batch_parser.py
index 072545e66..2edf05ff0 100644
--- a/bot/exts/info/doc/_batch_parser.py
+++ b/bot/exts/info/doc/_batch_parser.py
@@ -33,27 +33,27 @@ class StaleInventoryNotifier:
await bot.instance.wait_until_guild_available()
self._dev_log = bot.instance.get_channel(Channels.dev_log)
- async def send_warning(self, item: _cog.DocItem) -> None:
+ async def send_warning(self, doc_item: _cog.DocItem) -> None:
"""Send a warning to dev log is one wasn't already sent for `item`'s url."""
- if item.url not in self._warned_urls:
- self._warned_urls.add(item.url)
+ if doc_item.url not in self._warned_urls:
+ self._warned_urls.add(doc_item.url)
await self._init_task
embed = discord.Embed(
- description=f"Doc item `{item.symbol_id=}` present in loaded documentation inventories "
- f"not found on [site]({item.url}), inventories may need to be refreshed."
+ description=f"Doc item `{doc_item.symbol_id=}` present in loaded documentation inventories "
+ f"not found on [site]({doc_item.url}), inventories may need to be refreshed."
)
await self._dev_log.send(embed=embed)
class QueueItem(NamedTuple):
- """Contains a symbol and the BeautifulSoup object needed to parse it."""
+ """Contains a doc_item and the BeautifulSoup object needed to parse it."""
- symbol: _cog.DocItem
+ doc_item: _cog.DocItem
soup: BeautifulSoup
def __eq__(self, other: Union[QueueItem, _cog.DocItem]):
if isinstance(other, _cog.DocItem):
- return self.symbol == other
+ return self.doc_item == other
return NamedTuple.__eq__(self, other)
@@ -83,14 +83,14 @@ class BatchParser:
"""
Get the Markdown of all symbols on a page and send them to redis when a symbol is requested.
- DocItems are added through the `add_item` method which adds them to the `_page_symbols` dict.
+ DocItems are added through the `add_item` method which adds them to the `_page_doc_items` dict.
`get_markdown` is used to fetch the Markdown; when this is used for the first time on a page,
all of the symbols are queued to be parsed to avoid multiple web requests to the same page.
"""
def __init__(self):
self._queue: List[QueueItem] = []
- self._page_symbols: Dict[str, List[_cog.DocItem]] = defaultdict(list)
+ self._page_doc_items: Dict[str, List[_cog.DocItem]] = defaultdict(list)
self._item_futures: Dict[_cog.DocItem, ParseResultFuture] = {}
self._parse_task = None
@@ -109,14 +109,14 @@ class BatchParser:
Not safe to run while `self.clear` is running.
"""
if doc_item not in self._item_futures:
- self._item_futures.update((symbol, ParseResultFuture()) for symbol in self._page_symbols[doc_item.url])
+ self._item_futures.update((item, ParseResultFuture()) for item in self._page_doc_items[doc_item.url])
self._item_futures[doc_item].user_requested = True
async with bot.instance.http_session.get(doc_item.url) as response:
soup = BeautifulSoup(await response.text(encoding="utf8"), "lxml")
- self._queue.extend(QueueItem(symbol, soup) for symbol in self._page_symbols[doc_item.url])
- log.debug(f"Added symbols from {doc_item.url} to parse queue.")
+ self._queue.extend(QueueItem(item, soup) for item in self._page_doc_items[doc_item.url])
+ log.debug(f"Added items from {doc_item.url} to parse queue.")
if self._parse_task is None:
self._parse_task = asyncio.create_task(self._parse_queue())
@@ -139,7 +139,7 @@ class BatchParser:
item, soup = self._queue.pop()
try:
if (future := self._item_futures[item]).done():
- # Some items are present in the inventories multiple times under different symbols,
+ # Some items are present in the inventories multiple times under different symbol names,
# if we already parsed an equal item, we can just skip it.
continue
@@ -173,7 +173,7 @@ class BatchParser:
def add_item(self, doc_item: _cog.DocItem) -> None:
"""Map a DocItem to its page so that the symbol will be parsed once the page is requested."""
- self._page_symbols[doc_item.url].append(doc_item)
+ self._page_doc_items[doc_item.url].append(doc_item)
async def clear(self) -> None:
"""
@@ -186,7 +186,7 @@ class BatchParser:
if self._parse_task is not None:
self._parse_task.cancel()
self._queue.clear()
- self._page_symbols.clear()
+ self._page_doc_items.clear()
self._item_futures.clear()
async def _cleanup_futures(self) -> None:
diff --git a/bot/exts/info/doc/_cog.py b/bot/exts/info/doc/_cog.py
index 16baa6320..0ff775ac7 100644
--- a/bot/exts/info/doc/_cog.py
+++ b/bot/exts/info/doc/_cog.py
@@ -92,31 +92,31 @@ class DocCog(commands.Cog):
self.base_urls[api_package_name] = base_url
for group, items in package.items():
- for symbol, relative_doc_url in items:
+ for symbol_name, relative_doc_url in items:
# e.g. get 'class' from 'py:class'
group_name = group.split(":")[1]
- if (original_symbol := self.doc_symbols.get(symbol)) is not None:
+ if (original_item := self.doc_symbols.get(symbol_name)) is not None:
replaced_symbol_name = self.ensure_unique_symbol_name(
api_package_name,
group_name,
- original_symbol,
- symbol,
+ original_item,
+ symbol_name,
)
if replaced_symbol_name is not None:
- symbol = replaced_symbol_name
+ symbol_name = replaced_symbol_name
relative_url_path, _, symbol_id = relative_doc_url.partition("#")
# Intern fields that have shared content so we're not storing unique strings for every object
- symbol_item = DocItem(
+ doc_item = DocItem(
api_package_name,
sys.intern(group_name),
base_url,
sys.intern(relative_url_path),
- symbol_id
+ symbol_id,
)
- self.doc_symbols[symbol] = symbol_item
- self.item_fetcher.add_item(symbol_item)
+ self.doc_symbols[symbol_name] = doc_item
+ self.item_fetcher.add_item(doc_item)
log.trace(f"Fetched inventory for {api_package_name}.")
@@ -124,7 +124,7 @@ class DocCog(commands.Cog):
self,
api_package_name: str,
base_url: str,
- inventory_url: str
+ inventory_url: str,
) -> None:
"""
Update the cog's inventory, or reschedule this method to execute again if the remote inventory unreachable.
@@ -144,7 +144,7 @@ class DocCog(commands.Cog):
self.inventory_scheduler.schedule_later(
delay*60,
api_package_name,
- self.update_or_reschedule_inventory(api_package_name, base_url, inventory_url)
+ self.update_or_reschedule_inventory(api_package_name, base_url, inventory_url),
)
else:
self.update_single(api_package_name, base_url, package)
@@ -154,7 +154,7 @@ class DocCog(commands.Cog):
package_name: str,
group_name: str,
original_item: DocItem,
- symbol_name: str
+ symbol_name: str,
) -> Optional[str]:
"""
Ensure `symbol_name` doesn't overwrite an another symbol in `doc_symbols`.
@@ -166,42 +166,42 @@ class DocCog(commands.Cog):
"""
# Certain groups are added as prefixes to disambiguate the symbols.
if group_name in FORCE_PREFIX_GROUPS:
- new_symbol = f"{group_name}.{symbol_name}"
- if new_symbol in self.doc_symbols:
+ new_symbol_name = f"{group_name}.{symbol_name}"
+ if new_symbol_name in self.doc_symbols:
# If there's still a conflict, prefix with package name.
- new_symbol = f"{package_name}.{new_symbol}"
- self.renamed_symbols[symbol_name].append(new_symbol)
- return new_symbol
+ new_symbol_name = f"{package_name}.{new_symbol_name}"
+ self.renamed_symbols[symbol_name].append(new_symbol_name)
+ return new_symbol_name
# The existing symbol with which the current symbol conflicts should have a group prefix.
# It currently doesn't have the group prefix because it's only added once there's a conflict.
elif (original_symbol_group := original_item.group) in FORCE_PREFIX_GROUPS:
- overridden_symbol = f"{original_symbol_group}.{symbol_name}"
- if overridden_symbol in self.doc_symbols:
+ overridden_symbol_name = f"{original_symbol_group}.{symbol_name}"
+ if overridden_symbol_name in self.doc_symbols:
# If there's still a conflict, prefix with package name.
- overridden_symbol = f"{original_item.package}.{overridden_symbol}"
+ overridden_symbol_name = f"{original_item.package}.{overridden_symbol_name}"
- self.doc_symbols[overridden_symbol] = original_item
- self.renamed_symbols[symbol_name].append(overridden_symbol)
+ self.doc_symbols[overridden_symbol_name] = original_item
+ self.renamed_symbols[symbol_name].append(overridden_symbol_name)
elif package_name in PRIORITY_PACKAGES:
- overridden_symbol = f"{original_item.package}.{symbol_name}"
- if overridden_symbol in self.doc_symbols:
+ overridden_symbol_name = f"{original_item.package}.{symbol_name}"
+ if overridden_symbol_name in self.doc_symbols:
# If there's still a conflict, add the symbol's group in the middle.
- overridden_symbol = f"{original_item.package}.{original_item.group}.{symbol_name}"
+ overridden_symbol_name = f"{original_item.package}.{original_item.group}.{symbol_name}"
- self.doc_symbols[overridden_symbol] = original_item
- self.renamed_symbols[symbol_name].append(overridden_symbol)
+ self.doc_symbols[overridden_symbol_name] = original_item
+ self.renamed_symbols[symbol_name].append(overridden_symbol_name)
# If we can't specially handle the symbol through its group or package,
# fall back to prepending its package name to the front.
else:
- new_symbol = f"{package_name}.{symbol_name}"
- if new_symbol in self.doc_symbols:
+ new_symbol_name = f"{package_name}.{symbol_name}"
+ if new_symbol_name in self.doc_symbols:
# If there's still a conflict, add the symbol's group in the middle.
- new_symbol = f"{package_name}.{group_name}.{symbol_name}"
- self.renamed_symbols[symbol_name].append(new_symbol)
- return new_symbol
+ new_symbol_name = f"{package_name}.{group_name}.{symbol_name}"
+ self.renamed_symbols[symbol_name].append(new_symbol_name)
+ return new_symbol_name
async def refresh_inventory(self) -> None:
"""Refresh internal documentation inventory."""
@@ -229,7 +229,7 @@ class DocCog(commands.Cog):
log.debug("Finished inventory refresh.")
self.refresh_event.set()
- async def get_symbol_embed(self, symbol: str) -> Optional[discord.Embed]:
+ async def get_symbol_embed(self, symbol_name: str) -> Optional[discord.Embed]:
"""
Attempt to scrape and fetch the data for the given `symbol`, and build an embed from its contents.
@@ -237,47 +237,47 @@ class DocCog(commands.Cog):
First check the DocRedisCache before querying the cog's `BatchParser`.
"""
- log.trace(f"Building embed for symbol `{symbol}`")
+ log.trace(f"Building embed for symbol `{symbol_name}`")
if not self.refresh_event.is_set():
log.debug("Waiting for inventories to be refreshed before processing item.")
await self.refresh_event.wait()
- symbol_info = self.doc_symbols.get(symbol)
- if symbol_info is None:
- if symbol.count(" "):
+ doc_item = self.doc_symbols.get(symbol_name)
+ if doc_item is None:
+ if symbol_name.count(" "):
# If an invalid symbol contains a space, check if the command was invoked
# in the format !d <symbol> <message>
- symbol = symbol.split(" ", maxsplit=1)[0]
- symbol_info = self.doc_symbols.get(symbol)
- if symbol_info is None:
+ symbol_name = symbol_name.split(" ", maxsplit=1)[0]
+ doc_item = self.doc_symbols.get(symbol_name)
+ if doc_item is None:
log.debug("Symbol does not exist.")
return None
else:
log.debug("Symbol does not exist.")
return None
- self.bot.stats.incr(f"doc_fetches.{symbol_info.package}")
+ self.bot.stats.incr(f"doc_fetches.{doc_item.package}")
with self.symbol_get_event:
- markdown = await doc_cache.get(symbol_info)
+ markdown = await doc_cache.get(doc_item)
if markdown is None:
- log.debug(f"Redis cache miss for symbol `{symbol}`.")
- markdown = await self.item_fetcher.get_markdown(symbol_info)
+ log.debug(f"Redis cache miss for symbol `{symbol_name}`.")
+ markdown = await self.item_fetcher.get_markdown(doc_item)
if markdown is not None:
- await doc_cache.set(symbol_info, markdown)
+ await doc_cache.set(doc_item, markdown)
else:
markdown = "Unable to parse the requested symbol."
embed = discord.Embed(
- title=discord.utils.escape_markdown(symbol),
- url=f"{symbol_info.url}#{symbol_info.symbol_id}",
+ title=discord.utils.escape_markdown(symbol_name),
+ url=f"{doc_item.url}#{doc_item.symbol_id}",
description=markdown
)
# Show all symbols with the same name that were renamed in the footer,
# with a max of 100 chars.
- if symbol in self.renamed_symbols:
- renamed_symbols = ', '.join(self.renamed_symbols[symbol])
+ if symbol_name in self.renamed_symbols:
+ renamed_symbols = ', '.join(self.renamed_symbols[symbol_name])
footer_text = textwrap.shorten("Moved: " + renamed_symbols, 100, placeholder=' ...')
else:
footer_text = ""
@@ -285,12 +285,12 @@ class DocCog(commands.Cog):
return embed
@commands.group(name='docs', aliases=('doc', 'd'), invoke_without_command=True)
- async def docs_group(self, ctx: commands.Context, *, symbol: Optional[str]) -> None:
+ async def docs_group(self, ctx: commands.Context, *, symbol_name: Optional[str]) -> None:
"""Look up documentation for Python symbols."""
- await self.get_command(ctx, symbol=symbol)
+ await self.get_command(ctx, symbol_name=symbol_name)
@docs_group.command(name='getdoc', aliases=('g',))
- async def get_command(self, ctx: commands.Context, *, symbol: Optional[str]) -> None:
+ async def get_command(self, ctx: commands.Context, *, symbol_name: Optional[str]) -> None:
"""
Return a documentation embed for a given symbol.
@@ -302,7 +302,7 @@ class DocCog(commands.Cog):
!docs aiohttp.ClientSession
!docs getdoc aiohttp.ClientSession
"""
- if not symbol:
+ if not symbol_name:
inventory_embed = discord.Embed(
title=f"All inventories (`{len(self.base_urls)}` total)",
colour=discord.Colour.blue()
@@ -317,7 +317,7 @@ class DocCog(commands.Cog):
await ctx.send(embed=inventory_embed)
else:
- symbol = symbol.strip("`")
+ symbol = symbol_name.strip("`")
# Fetching documentation for a symbol (at least for the first time, since
# caching is used) takes quite some time, so let's send typing to indicate
# that we got the command, but are still working on it.
diff --git a/bot/exts/info/doc/_parsing.py b/bot/exts/info/doc/_parsing.py
index 0251b0105..8e1b4d7a1 100644
--- a/bot/exts/info/doc/_parsing.py
+++ b/bot/exts/info/doc/_parsing.py
@@ -232,7 +232,7 @@ def _create_markdown(signatures: Optional[List[str]], description: Iterable[Tag]
def get_symbol_markdown(soup: BeautifulSoup, symbol_data: DocItem) -> Optional[str]:
"""
- Return parsed markdown of the passed symbol using the passed in soup, truncated to fit within a discord message.
+ Return parsed markdown of the passed item using the passed in soup, truncated to fit within a discord message.
The method of parsing and what information gets included depends on the symbol's group.
"""