diff options
| author | 2021-02-03 11:49:53 +0100 | |
|---|---|---|
| committer | 2021-02-03 12:00:37 +0100 | |
| commit | 1df9459c2c9c33f3e6dbcbd1b76415610aade192 (patch) | |
| tree | f0bd52de0c9b4d0525ee719be0bf6b37d60342a0 | |
| parent | Import module directly to avoid circular dependency and use of type (diff) | |
Use more consistent naming for doc items and symbols
Trailing commas were also added where missing
Diffstat (limited to '')
| -rw-r--r-- | bot/exts/info/doc/_batch_parser.py | 32 | ||||
| -rw-r--r-- | bot/exts/info/doc/_cog.py | 108 | ||||
| -rw-r--r-- | bot/exts/info/doc/_parsing.py | 2 | 
3 files changed, 71 insertions, 71 deletions
| diff --git a/bot/exts/info/doc/_batch_parser.py b/bot/exts/info/doc/_batch_parser.py index 072545e66..2edf05ff0 100644 --- a/bot/exts/info/doc/_batch_parser.py +++ b/bot/exts/info/doc/_batch_parser.py @@ -33,27 +33,27 @@ class StaleInventoryNotifier:          await bot.instance.wait_until_guild_available()          self._dev_log = bot.instance.get_channel(Channels.dev_log) -    async def send_warning(self, item: _cog.DocItem) -> None: +    async def send_warning(self, doc_item: _cog.DocItem) -> None:          """Send a warning to dev log is one wasn't already sent for `item`'s url.""" -        if item.url not in self._warned_urls: -            self._warned_urls.add(item.url) +        if doc_item.url not in self._warned_urls: +            self._warned_urls.add(doc_item.url)              await self._init_task              embed = discord.Embed( -                description=f"Doc item `{item.symbol_id=}` present in loaded documentation inventories " -                            f"not found on [site]({item.url}), inventories may need to be refreshed." +                description=f"Doc item `{doc_item.symbol_id=}` present in loaded documentation inventories " +                            f"not found on [site]({doc_item.url}), inventories may need to be refreshed."              )              await self._dev_log.send(embed=embed)  class QueueItem(NamedTuple): -    """Contains a symbol and the BeautifulSoup object needed to parse it.""" +    """Contains a doc_item and the BeautifulSoup object needed to parse it.""" -    symbol: _cog.DocItem +    doc_item: _cog.DocItem      soup: BeautifulSoup      def __eq__(self, other: Union[QueueItem, _cog.DocItem]):          if isinstance(other, _cog.DocItem): -            return self.symbol == other +            return self.doc_item == other          return NamedTuple.__eq__(self, other) @@ -83,14 +83,14 @@ class BatchParser:      """      Get the Markdown of all symbols on a page and send them to redis when a symbol is requested. -    DocItems are added through the `add_item` method which adds them to the `_page_symbols` dict. +    DocItems are added through the `add_item` method which adds them to the `_page_doc_items` dict.      `get_markdown` is used to fetch the Markdown; when this is used for the first time on a page,      all of the symbols are queued to be parsed to avoid multiple web requests to the same page.      """      def __init__(self):          self._queue: List[QueueItem] = [] -        self._page_symbols: Dict[str, List[_cog.DocItem]] = defaultdict(list) +        self._page_doc_items: Dict[str, List[_cog.DocItem]] = defaultdict(list)          self._item_futures: Dict[_cog.DocItem, ParseResultFuture] = {}          self._parse_task = None @@ -109,14 +109,14 @@ class BatchParser:          Not safe to run while `self.clear` is running.          """          if doc_item not in self._item_futures: -            self._item_futures.update((symbol, ParseResultFuture()) for symbol in self._page_symbols[doc_item.url]) +            self._item_futures.update((item, ParseResultFuture()) for item in self._page_doc_items[doc_item.url])              self._item_futures[doc_item].user_requested = True              async with bot.instance.http_session.get(doc_item.url) as response:                  soup = BeautifulSoup(await response.text(encoding="utf8"), "lxml") -            self._queue.extend(QueueItem(symbol, soup) for symbol in self._page_symbols[doc_item.url]) -            log.debug(f"Added symbols from {doc_item.url} to parse queue.") +            self._queue.extend(QueueItem(item, soup) for item in self._page_doc_items[doc_item.url]) +            log.debug(f"Added items from {doc_item.url} to parse queue.")              if self._parse_task is None:                  self._parse_task = asyncio.create_task(self._parse_queue()) @@ -139,7 +139,7 @@ class BatchParser:                  item, soup = self._queue.pop()                  try:                      if (future := self._item_futures[item]).done(): -                        # Some items are present in the inventories multiple times under different symbols, +                        # Some items are present in the inventories multiple times under different symbol names,                          # if we already parsed an equal item, we can just skip it.                          continue @@ -173,7 +173,7 @@ class BatchParser:      def add_item(self, doc_item: _cog.DocItem) -> None:          """Map a DocItem to its page so that the symbol will be parsed once the page is requested.""" -        self._page_symbols[doc_item.url].append(doc_item) +        self._page_doc_items[doc_item.url].append(doc_item)      async def clear(self) -> None:          """ @@ -186,7 +186,7 @@ class BatchParser:          if self._parse_task is not None:              self._parse_task.cancel()          self._queue.clear() -        self._page_symbols.clear() +        self._page_doc_items.clear()          self._item_futures.clear()      async def _cleanup_futures(self) -> None: diff --git a/bot/exts/info/doc/_cog.py b/bot/exts/info/doc/_cog.py index 16baa6320..0ff775ac7 100644 --- a/bot/exts/info/doc/_cog.py +++ b/bot/exts/info/doc/_cog.py @@ -92,31 +92,31 @@ class DocCog(commands.Cog):          self.base_urls[api_package_name] = base_url          for group, items in package.items(): -            for symbol, relative_doc_url in items: +            for symbol_name, relative_doc_url in items:                  # e.g. get 'class' from 'py:class'                  group_name = group.split(":")[1] -                if (original_symbol := self.doc_symbols.get(symbol)) is not None: +                if (original_item := self.doc_symbols.get(symbol_name)) is not None:                      replaced_symbol_name = self.ensure_unique_symbol_name(                          api_package_name,                          group_name, -                        original_symbol, -                        symbol, +                        original_item, +                        symbol_name,                      )                      if replaced_symbol_name is not None: -                        symbol = replaced_symbol_name +                        symbol_name = replaced_symbol_name                  relative_url_path, _, symbol_id = relative_doc_url.partition("#")                  # Intern fields that have shared content so we're not storing unique strings for every object -                symbol_item = DocItem( +                doc_item = DocItem(                      api_package_name,                      sys.intern(group_name),                      base_url,                      sys.intern(relative_url_path), -                    symbol_id +                    symbol_id,                  ) -                self.doc_symbols[symbol] = symbol_item -                self.item_fetcher.add_item(symbol_item) +                self.doc_symbols[symbol_name] = doc_item +                self.item_fetcher.add_item(doc_item)          log.trace(f"Fetched inventory for {api_package_name}.") @@ -124,7 +124,7 @@ class DocCog(commands.Cog):              self,              api_package_name: str,              base_url: str, -            inventory_url: str +            inventory_url: str,      ) -> None:          """          Update the cog's inventory, or reschedule this method to execute again if the remote inventory unreachable. @@ -144,7 +144,7 @@ class DocCog(commands.Cog):              self.inventory_scheduler.schedule_later(                  delay*60,                  api_package_name, -                self.update_or_reschedule_inventory(api_package_name, base_url, inventory_url) +                self.update_or_reschedule_inventory(api_package_name, base_url, inventory_url),              )          else:              self.update_single(api_package_name, base_url, package) @@ -154,7 +154,7 @@ class DocCog(commands.Cog):              package_name: str,              group_name: str,              original_item: DocItem, -            symbol_name: str +            symbol_name: str,      ) -> Optional[str]:          """          Ensure `symbol_name` doesn't overwrite an another symbol in `doc_symbols`. @@ -166,42 +166,42 @@ class DocCog(commands.Cog):          """          # Certain groups are added as prefixes to disambiguate the symbols.          if group_name in FORCE_PREFIX_GROUPS: -            new_symbol = f"{group_name}.{symbol_name}" -            if new_symbol in self.doc_symbols: +            new_symbol_name = f"{group_name}.{symbol_name}" +            if new_symbol_name in self.doc_symbols:                  # If there's still a conflict, prefix with package name. -                new_symbol = f"{package_name}.{new_symbol}" -            self.renamed_symbols[symbol_name].append(new_symbol) -            return new_symbol +                new_symbol_name = f"{package_name}.{new_symbol_name}" +            self.renamed_symbols[symbol_name].append(new_symbol_name) +            return new_symbol_name          # The existing symbol with which the current symbol conflicts should have a group prefix.          # It currently doesn't have the group prefix because it's only added once there's a conflict.          elif (original_symbol_group := original_item.group) in FORCE_PREFIX_GROUPS: -            overridden_symbol = f"{original_symbol_group}.{symbol_name}" -            if overridden_symbol in self.doc_symbols: +            overridden_symbol_name = f"{original_symbol_group}.{symbol_name}" +            if overridden_symbol_name in self.doc_symbols:                  # If there's still a conflict, prefix with package name. -                overridden_symbol = f"{original_item.package}.{overridden_symbol}" +                overridden_symbol_name = f"{original_item.package}.{overridden_symbol_name}" -            self.doc_symbols[overridden_symbol] = original_item -            self.renamed_symbols[symbol_name].append(overridden_symbol) +            self.doc_symbols[overridden_symbol_name] = original_item +            self.renamed_symbols[symbol_name].append(overridden_symbol_name)          elif package_name in PRIORITY_PACKAGES: -            overridden_symbol = f"{original_item.package}.{symbol_name}" -            if overridden_symbol in self.doc_symbols: +            overridden_symbol_name = f"{original_item.package}.{symbol_name}" +            if overridden_symbol_name in self.doc_symbols:                  # If there's still a conflict, add the symbol's group in the middle. -                overridden_symbol = f"{original_item.package}.{original_item.group}.{symbol_name}" +                overridden_symbol_name = f"{original_item.package}.{original_item.group}.{symbol_name}" -            self.doc_symbols[overridden_symbol] = original_item -            self.renamed_symbols[symbol_name].append(overridden_symbol) +            self.doc_symbols[overridden_symbol_name] = original_item +            self.renamed_symbols[symbol_name].append(overridden_symbol_name)          # If we can't specially handle the symbol through its group or package,          # fall back to prepending its package name to the front.          else: -            new_symbol = f"{package_name}.{symbol_name}" -            if new_symbol in self.doc_symbols: +            new_symbol_name = f"{package_name}.{symbol_name}" +            if new_symbol_name in self.doc_symbols:                  # If there's still a conflict, add the symbol's group in the middle. -                new_symbol = f"{package_name}.{group_name}.{symbol_name}" -            self.renamed_symbols[symbol_name].append(new_symbol) -            return new_symbol +                new_symbol_name = f"{package_name}.{group_name}.{symbol_name}" +            self.renamed_symbols[symbol_name].append(new_symbol_name) +            return new_symbol_name      async def refresh_inventory(self) -> None:          """Refresh internal documentation inventory.""" @@ -229,7 +229,7 @@ class DocCog(commands.Cog):          log.debug("Finished inventory refresh.")          self.refresh_event.set() -    async def get_symbol_embed(self, symbol: str) -> Optional[discord.Embed]: +    async def get_symbol_embed(self, symbol_name: str) -> Optional[discord.Embed]:          """          Attempt to scrape and fetch the data for the given `symbol`, and build an embed from its contents. @@ -237,47 +237,47 @@ class DocCog(commands.Cog):          First check the DocRedisCache before querying the cog's `BatchParser`.          """ -        log.trace(f"Building embed for symbol `{symbol}`") +        log.trace(f"Building embed for symbol `{symbol_name}`")          if not self.refresh_event.is_set():              log.debug("Waiting for inventories to be refreshed before processing item.")              await self.refresh_event.wait() -        symbol_info = self.doc_symbols.get(symbol) -        if symbol_info is None: -            if symbol.count(" "): +        doc_item = self.doc_symbols.get(symbol_name) +        if doc_item is None: +            if symbol_name.count(" "):                  # If an invalid symbol contains a space, check if the command was invoked                  # in the format !d <symbol> <message> -                symbol = symbol.split(" ", maxsplit=1)[0] -                symbol_info = self.doc_symbols.get(symbol) -                if symbol_info is None: +                symbol_name = symbol_name.split(" ", maxsplit=1)[0] +                doc_item = self.doc_symbols.get(symbol_name) +                if doc_item is None:                      log.debug("Symbol does not exist.")                      return None              else:                  log.debug("Symbol does not exist.")                  return None -        self.bot.stats.incr(f"doc_fetches.{symbol_info.package}") +        self.bot.stats.incr(f"doc_fetches.{doc_item.package}")          with self.symbol_get_event: -            markdown = await doc_cache.get(symbol_info) +            markdown = await doc_cache.get(doc_item)          if markdown is None: -            log.debug(f"Redis cache miss for symbol `{symbol}`.") -            markdown = await self.item_fetcher.get_markdown(symbol_info) +            log.debug(f"Redis cache miss for symbol `{symbol_name}`.") +            markdown = await self.item_fetcher.get_markdown(doc_item)              if markdown is not None: -                await doc_cache.set(symbol_info, markdown) +                await doc_cache.set(doc_item, markdown)              else:                  markdown = "Unable to parse the requested symbol."          embed = discord.Embed( -            title=discord.utils.escape_markdown(symbol), -            url=f"{symbol_info.url}#{symbol_info.symbol_id}", +            title=discord.utils.escape_markdown(symbol_name), +            url=f"{doc_item.url}#{doc_item.symbol_id}",              description=markdown          )          # Show all symbols with the same name that were renamed in the footer,          # with a max of 100 chars. -        if symbol in self.renamed_symbols: -            renamed_symbols = ', '.join(self.renamed_symbols[symbol]) +        if symbol_name in self.renamed_symbols: +            renamed_symbols = ', '.join(self.renamed_symbols[symbol_name])              footer_text = textwrap.shorten("Moved: " + renamed_symbols, 100, placeholder=' ...')          else:              footer_text = "" @@ -285,12 +285,12 @@ class DocCog(commands.Cog):          return embed      @commands.group(name='docs', aliases=('doc', 'd'), invoke_without_command=True) -    async def docs_group(self, ctx: commands.Context, *, symbol: Optional[str]) -> None: +    async def docs_group(self, ctx: commands.Context, *, symbol_name: Optional[str]) -> None:          """Look up documentation for Python symbols.""" -        await self.get_command(ctx, symbol=symbol) +        await self.get_command(ctx, symbol_name=symbol_name)      @docs_group.command(name='getdoc', aliases=('g',)) -    async def get_command(self, ctx: commands.Context, *, symbol: Optional[str]) -> None: +    async def get_command(self, ctx: commands.Context, *, symbol_name: Optional[str]) -> None:          """          Return a documentation embed for a given symbol. @@ -302,7 +302,7 @@ class DocCog(commands.Cog):              !docs aiohttp.ClientSession              !docs getdoc aiohttp.ClientSession          """ -        if not symbol: +        if not symbol_name:              inventory_embed = discord.Embed(                  title=f"All inventories (`{len(self.base_urls)}` total)",                  colour=discord.Colour.blue() @@ -317,7 +317,7 @@ class DocCog(commands.Cog):                  await ctx.send(embed=inventory_embed)          else: -            symbol = symbol.strip("`") +            symbol = symbol_name.strip("`")              # Fetching documentation for a symbol (at least for the first time, since              # caching is used) takes quite some time, so let's send typing to indicate              # that we got the command, but are still working on it. diff --git a/bot/exts/info/doc/_parsing.py b/bot/exts/info/doc/_parsing.py index 0251b0105..8e1b4d7a1 100644 --- a/bot/exts/info/doc/_parsing.py +++ b/bot/exts/info/doc/_parsing.py @@ -232,7 +232,7 @@ def _create_markdown(signatures: Optional[List[str]], description: Iterable[Tag]  def get_symbol_markdown(soup: BeautifulSoup, symbol_data: DocItem) -> Optional[str]:      """ -    Return parsed markdown of the passed symbol using the passed in soup, truncated to fit within a discord message. +    Return parsed markdown of the passed item using the passed in soup, truncated to fit within a discord message.      The method of parsing and what information gets included depends on the symbol's group.      """ | 
