diff options
| author |  Numerlor <[email protected]> | 2021-02-03 12:07:13 +0100 |
|---|---|---|
| committer | Numerlor <[email protected]> | 2021-02-03 12:07:13 +0100 |
| commit | ef5a98595ec647198f3d06375d2c1d4a5a54bf02 (patch) | |
| tree | 8944eac90366d8f7496b659edfea345aa8a20ef4 | |
| parent | Remove url lock (diff) | |
Move BeautifulSoup parsing into an executor
| -rw-r--r-- | bot/exts/info/doc/_batch_parser.py | 5 |
1 files changed, 4 insertions, 1 deletions
diff --git a/bot/exts/info/doc/_batch_parser.py b/bot/exts/info/doc/_batch_parser.py index c16cf6d28..13e197587 100644 --- a/bot/exts/info/doc/_batch_parser.py +++ b/bot/exts/info/doc/_batch_parser.py @@ -111,7 +111,10 @@ class BatchParser: self._item_futures[doc_item].user_requested = True async with bot.instance.http_session.get(doc_item.url) as response: - soup = BeautifulSoup(await response.text(encoding="utf8"), "lxml") + soup = await bot.instance.loop.run_in_executor( + None, + partial(BeautifulSoup, await response.text(encoding="utf8"), "lxml") + ) self._queue.extend(QueueItem(item, soup) for item in self._page_doc_items[doc_item.url]) log.debug(f"Added items from {doc_item.url} to parse queue.") |