From 8f9f25a796f6cc07f01f8f7f56e825cb5ebf56c8 Mon Sep 17 00:00:00 2001 From: Hassan Abouelela Date: Sat, 23 Apr 2022 14:24:20 +0400 Subject: Speed Up Sync Cog Loading The user syncer was blocking the startup of the sync cog due to having to perform thousands of pointless member fetch requests. This speeds up that process by increasing the probability that the cache is up-to-date using `Guild.chunked`, and limiting the fetches to members who were in the guild during the previous sync only. Co-authored-by: ChrisJL Co-authored-by: wookie184 Signed-off-by: Hassan Abouelela --- bot/exts/backend/sync/_cog.py | 6 ++++++ bot/exts/backend/sync/_syncers.py | 13 +++++++++++-- tests/helpers.py | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/bot/exts/backend/sync/_cog.py b/bot/exts/backend/sync/_cog.py index a5bf82397..4ec822d3f 100644 --- a/bot/exts/backend/sync/_cog.py +++ b/bot/exts/backend/sync/_cog.py @@ -1,3 +1,4 @@ +import asyncio from typing import Any, Dict from botcore.site_api import ResponseCodeError @@ -27,6 +28,11 @@ class Sync(Cog): if guild is None: return + log.info("Waiting for guild to be chunked to start syncers.") + while not guild.chunked: + await asyncio.sleep(10) + log.info("Starting syncers.") + for syncer in (_syncers.RoleSyncer, _syncers.UserSyncer): await syncer.sync(guild) diff --git a/bot/exts/backend/sync/_syncers.py b/bot/exts/backend/sync/_syncers.py index e1c4541ef..799137cb9 100644 --- a/bot/exts/backend/sync/_syncers.py +++ b/bot/exts/backend/sync/_syncers.py @@ -2,6 +2,7 @@ import abc import typing as t from collections import namedtuple +import discord.errors from botcore.site_api import ResponseCodeError from discord import Guild from discord.ext.commands import Context @@ -9,7 +10,6 @@ from more_itertools import chunked import bot from bot.log import get_logger -from bot.utils.members import get_or_fetch_member log = get_logger(__name__) @@ -157,7 +157,16 @@ class UserSyncer(Syncer): if db_user[db_field] != guild_value: updated_fields[db_field] = guild_value - if guild_user := await get_or_fetch_member(guild, db_user["id"]): + guild_user = guild.get_member(db_user["id"]) + if not guild_user and db_user["in_guild"]: + # The member was in the guild during the last sync. + # We try to fetch them to verify cache integrity. + try: + guild_user = await guild.fetch_member(db_user["id"]) + except discord.errors.NotFound: + guild_user = None + + if guild_user: seen_guild_users.add(guild_user.id) maybe_update("name", guild_user.name) diff --git a/tests/helpers.py b/tests/helpers.py index a6e4bdd66..5f3111616 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -171,7 +171,7 @@ class MockGuild(CustomMockMixin, unittest.mock.Mock, HashableMixin): spec_set = guild_instance def __init__(self, roles: Optional[Iterable[MockRole]] = None, **kwargs) -> None: - default_kwargs = {'id': next(self.discord_id), 'members': []} + default_kwargs = {'id': next(self.discord_id), 'members': [], "chunked": True} super().__init__(**collections.ChainMap(kwargs, default_kwargs)) self.roles = [MockRole(name="@everyone", position=1, id=0)] -- cgit v1.2.3