From 681027b61663bcdff5b174aa3e06f34b54f05349 Mon Sep 17 00:00:00 2001 From: RohanJnr Date: Mon, 21 Sep 2020 19:21:13 +0530 Subject: refactor code to GET users from site endpoint `bot/users` with pagination Added method to recursively GET users if paginated and another method to parse URL and return endpoint and query parameters. --- bot/cogs/sync/syncers.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/bot/cogs/sync/syncers.py b/bot/cogs/sync/syncers.py index f7ba811bc..156c32a15 100644 --- a/bot/cogs/sync/syncers.py +++ b/bot/cogs/sync/syncers.py @@ -4,6 +4,7 @@ import logging import typing as t from collections import namedtuple from functools import partial +from urllib.parse import parse_qsl, urlparse import discord from discord import Guild, HTTPException, Member, Message, Reaction, User @@ -287,7 +288,8 @@ class UserSyncer(Syncer): async def _get_diff(self, guild: Guild) -> _Diff: """Return the difference of users between the cache of `guild` and the database.""" log.trace("Getting the diff for users.") - users = await self.bot.api_client.get('bot/users') + + users = await self._get_users() # Pack DB roles and guild roles into one common, hashable format. # They're hashable so that they're easily comparable with sets later. @@ -336,6 +338,32 @@ class UserSyncer(Syncer): return _Diff(users_to_create, users_to_update, None) + async def _get_users(self, endpoint: str = "bot/users", query_params: dict = None) -> t.List[dict]: + """GET all users recursively.""" + users: list = [] + response: dict = await self.bot.api_client.get(endpoint, params=query_params) + users.extend(response["results"]) + + # The `response` is paginated, hence check if next page exists. + if (next_page_url := response["next"]) is not None: + next_endpoint, query_params = self.get_endpoint(next_page_url) + users.extend(await self._get_users(next_endpoint, query_params)) + + return users + + @staticmethod + def get_endpoint(url: str) -> tuple: + """Extract the API endpoint and query params from a URL.""" + url = urlparse(url) + + # Do not include starting `/` for endpoint. + endpoint = url.path[1:] + + # Query params. + params = parse_qsl(url.query) + + return endpoint, params + async def _sync(self, diff: _Diff) -> None: """Synchronise the database with the user cache of `guild`.""" log.trace("Syncing created users...") -- cgit v1.2.3 From ec0db2dd98e55f8bf5ba1c07375e196933129f99 Mon Sep 17 00:00:00 2001 From: RohanJnr Date: Tue, 22 Sep 2020 21:55:09 +0530 Subject: Refactor code to make use of bulk create and update API endpoints. instead of creating and updating a single user at a time, a list of dicts will be sent for bulk update and creation. --- bot/exts/backend/sync/_syncers.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bot/exts/backend/sync/_syncers.py b/bot/exts/backend/sync/_syncers.py index 156c32a15..7d1a8eacc 100644 --- a/bot/exts/backend/sync/_syncers.py +++ b/bot/exts/backend/sync/_syncers.py @@ -367,9 +367,10 @@ class UserSyncer(Syncer): async def _sync(self, diff: _Diff) -> None: """Synchronise the database with the user cache of `guild`.""" log.trace("Syncing created users...") - for user in diff.created: - await self.bot.api_client.post('bot/users', json=user._asdict()) + if diff.created: + created: list = [user._asdict() for user in diff.created] + await self.bot.api_client.post("bot/users", json=created) - log.trace("Syncing updated users...") - for user in diff.updated: - await self.bot.api_client.put(f'bot/users/{user.id}', json=user._asdict()) + if diff.updated: + updated = [user._asdict() for user in diff.created] + await self.bot.api_client.patch("bot/users/bulk_patch", json=updated) -- cgit v1.2.3 From eca87e32948142863c562664bde262bf9054ca94 Mon Sep 17 00:00:00 2001 From: RohanJnr Date: Tue, 22 Sep 2020 22:19:27 +0530 Subject: fix type and add variable type hinting --- bot/exts/backend/sync/_syncers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bot/exts/backend/sync/_syncers.py b/bot/exts/backend/sync/_syncers.py index 7d1a8eacc..cf75b6407 100644 --- a/bot/exts/backend/sync/_syncers.py +++ b/bot/exts/backend/sync/_syncers.py @@ -370,7 +370,6 @@ class UserSyncer(Syncer): if diff.created: created: list = [user._asdict() for user in diff.created] await self.bot.api_client.post("bot/users", json=created) - if diff.updated: - updated = [user._asdict() for user in diff.created] + updated: list = [user._asdict() for user in diff.updated] await self.bot.api_client.patch("bot/users/bulk_patch", json=updated) -- cgit v1.2.3 From 3f87c52f484afc1316e87f67f4055d5d615b054a Mon Sep 17 00:00:00 2001 From: RohanJnr Date: Fri, 25 Sep 2020 15:29:14 +0530 Subject: Update users on bot start via HTTP PATCH method and send only user ID and the modified user data. --- bot/exts/backend/sync/_syncers.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/bot/exts/backend/sync/_syncers.py b/bot/exts/backend/sync/_syncers.py index cf75b6407..512efaa3d 100644 --- a/bot/exts/backend/sync/_syncers.py +++ b/bot/exts/backend/sync/_syncers.py @@ -316,9 +316,18 @@ class UserSyncer(Syncer): for db_user in db_users.values(): guild_user = guild_users.get(db_user.id) + if guild_user is not None: if db_user != guild_user: - users_to_update.add(guild_user) + fields_to_none: dict = {} + + for field in _User._fields: + # Set un-changed values to None except ID to speed up API PATCH method. + if getattr(db_user, field) == getattr(guild_user, field) and field != "id": + fields_to_none[field] = None + + new_api_user = guild_user._replace(**fields_to_none) + users_to_update.add(new_api_user) elif db_user.in_guild: # The user is known in the DB but not the guild, and the @@ -326,7 +335,13 @@ class UserSyncer(Syncer): # This means that the user has left since the last sync. # Update the `in_guild` attribute of the user on the site # to signify that the user left. - new_api_user = db_user._replace(in_guild=False) + + # Set un-changed fields to None except ID as it is required by the API. + fields_to_none: dict = {field: None for field in db_user._fields if field not in ["id", "in_guild"]} + new_api_user = db_user._replace( + in_guild=False, + **fields_to_none + ) users_to_update.add(new_api_user) new_user_ids = set(guild_users.keys()) - set(db_users.keys()) @@ -364,6 +379,15 @@ class UserSyncer(Syncer): return endpoint, params + @staticmethod + def patch_dict(user: _User) -> dict: + """Convert namedtuple to dict by omitting None values.""" + user_dict: dict = {} + for field in user._fields: + if (value := getattr(user, field)) is not None: + user_dict[field] = value + return user_dict + async def _sync(self, diff: _Diff) -> None: """Synchronise the database with the user cache of `guild`.""" log.trace("Syncing created users...") @@ -371,5 +395,5 @@ class UserSyncer(Syncer): created: list = [user._asdict() for user in diff.created] await self.bot.api_client.post("bot/users", json=created) if diff.updated: - updated: list = [user._asdict() for user in diff.updated] + updated: list = [self.patch_dict(user) for user in diff.updated] await self.bot.api_client.patch("bot/users/bulk_patch", json=updated) -- cgit v1.2.3 From 56089920fb7ece152a97e6dc71968bb875c28c33 Mon Sep 17 00:00:00 2001 From: RohanJnr Date: Sun, 27 Sep 2020 22:51:28 +0530 Subject: modify tests to use paginated response. --- tests/bot/exts/backend/sync/test_users.py | 43 ++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/tests/bot/exts/backend/sync/test_users.py b/tests/bot/exts/backend/sync/test_users.py index c0a1da35c..4ebc8b82f 100644 --- a/tests/bot/exts/backend/sync/test_users.py +++ b/tests/bot/exts/backend/sync/test_users.py @@ -41,6 +41,7 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): return guild async def test_empty_diff_for_no_users(self): + # TODO: need to fix this test. """When no users are given, an empty diff should be returned.""" guild = self.get_guild() @@ -51,7 +52,12 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): async def test_empty_diff_for_identical_users(self): """No differences should be found if the users in the guild and DB are identical.""" - self.bot.api_client.get.return_value = [fake_user()] + self.bot.api_client.get.return_value = { + "count": 3, + "next": None, + "previous": None, + "results": [fake_user()] + } guild = self.get_guild(fake_user()) actual_diff = await self.syncer._get_diff(guild) @@ -63,7 +69,12 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): """Only updated users should be added to the 'updated' set of the diff.""" updated_user = fake_user(id=99, name="new") - self.bot.api_client.get.return_value = [fake_user(id=99, name="old"), fake_user()] + self.bot.api_client.get.return_value = { + "count": 3, + "next": None, + "previous": None, + "results": [fake_user(id=99, name="old"), fake_user()] + } guild = self.get_guild(updated_user, fake_user()) actual_diff = await self.syncer._get_diff(guild) @@ -75,7 +86,12 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): """Only new users should be added to the 'created' set of the diff.""" new_user = fake_user(id=99, name="new") - self.bot.api_client.get.return_value = [fake_user()] + self.bot.api_client.get.return_value = { + "count": 3, + "next": None, + "previous": None, + "results": [fake_user()] + } guild = self.get_guild(fake_user(), new_user) actual_diff = await self.syncer._get_diff(guild) @@ -87,7 +103,12 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): """When a user leaves the guild, the `in_guild` flag is updated to `False`.""" leaving_user = fake_user(id=63, in_guild=False) - self.bot.api_client.get.return_value = [fake_user(), fake_user(id=63)] + self.bot.api_client.get.return_value = { + "count": 3, + "next": None, + "previous": None, + "results": [fake_user(), fake_user(id=63)] + } guild = self.get_guild(fake_user()) actual_diff = await self.syncer._get_diff(guild) @@ -101,7 +122,12 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): updated_user = fake_user(id=55, name="updated") leaving_user = fake_user(id=63, in_guild=False) - self.bot.api_client.get.return_value = [fake_user(), fake_user(id=55), fake_user(id=63)] + self.bot.api_client.get.return_value = { + "count": 3, + "next": None, + "previous": None, + "results": [fake_user(), fake_user(id=55), fake_user(id=63)] + } guild = self.get_guild(fake_user(), new_user, updated_user) actual_diff = await self.syncer._get_diff(guild) @@ -111,7 +137,12 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): async def test_empty_diff_for_db_users_not_in_guild(self): """When the DB knows a user the guild doesn't, no difference is found.""" - self.bot.api_client.get.return_value = [fake_user(), fake_user(id=63, in_guild=False)] + self.bot.api_client.get.return_value = { + "count": 3, + "next": None, + "previous": None, + "results": [fake_user(), fake_user(id=63, in_guild=False)] + } guild = self.get_guild(fake_user()) actual_diff = await self.syncer._get_diff(guild) -- cgit v1.2.3 From ba4778d1d618b37ca190c921bcec571319e2914e Mon Sep 17 00:00:00 2001 From: RohanJnr Date: Thu, 1 Oct 2020 15:55:55 +0530 Subject: remove redundant type hints and improve existing function annotations --- bot/exts/backend/sync/_syncers.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bot/exts/backend/sync/_syncers.py b/bot/exts/backend/sync/_syncers.py index 512efaa3d..ea0f2bcb6 100644 --- a/bot/exts/backend/sync/_syncers.py +++ b/bot/exts/backend/sync/_syncers.py @@ -353,9 +353,9 @@ class UserSyncer(Syncer): return _Diff(users_to_create, users_to_update, None) - async def _get_users(self, endpoint: str = "bot/users", query_params: dict = None) -> t.List[dict]: + async def _get_users(self, endpoint: str = "bot/users", query_params: list = None) -> t.List[dict]: """GET all users recursively.""" - users: list = [] + users = [] response: dict = await self.bot.api_client.get(endpoint, params=query_params) users.extend(response["results"]) @@ -363,11 +363,10 @@ class UserSyncer(Syncer): if (next_page_url := response["next"]) is not None: next_endpoint, query_params = self.get_endpoint(next_page_url) users.extend(await self._get_users(next_endpoint, query_params)) - return users @staticmethod - def get_endpoint(url: str) -> tuple: + def get_endpoint(url: str) -> t.Tuple[str, t.List[tuple]]: """Extract the API endpoint and query params from a URL.""" url = urlparse(url) @@ -380,9 +379,9 @@ class UserSyncer(Syncer): return endpoint, params @staticmethod - def patch_dict(user: _User) -> dict: + def patch_dict(user: _User) -> t.Dict[str, t.Union[int, str, tuple, bool]]: """Convert namedtuple to dict by omitting None values.""" - user_dict: dict = {} + user_dict = {} for field in user._fields: if (value := getattr(user, field)) is not None: user_dict[field] = value @@ -392,8 +391,9 @@ class UserSyncer(Syncer): """Synchronise the database with the user cache of `guild`.""" log.trace("Syncing created users...") if diff.created: - created: list = [user._asdict() for user in diff.created] + created = [user._asdict() for user in diff.created] await self.bot.api_client.post("bot/users", json=created) + log.trace("Syncing updated users...") if diff.updated: - updated: list = [self.patch_dict(user) for user in diff.updated] + updated = [self.patch_dict(user) for user in diff.updated] await self.bot.api_client.patch("bot/users/bulk_patch", json=updated) -- cgit v1.2.3 From 0820a81057a8945f33cb386e2010ed78102c9c42 Mon Sep 17 00:00:00 2001 From: RohanJnr Date: Fri, 2 Oct 2020 21:39:25 +0530 Subject: update UserSyncerDiffTests Tests to use changes made to API calls. --- tests/bot/exts/backend/sync/test_users.py | 38 ++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/tests/bot/exts/backend/sync/test_users.py b/tests/bot/exts/backend/sync/test_users.py index 4ebc8b82f..e60c3a24d 100644 --- a/tests/bot/exts/backend/sync/test_users.py +++ b/tests/bot/exts/backend/sync/test_users.py @@ -16,6 +16,16 @@ def fake_user(**kwargs): return kwargs +def fake_none_user(**kwargs): + kwargs.setdefault("id", None) + kwargs.setdefault("name", None) + kwargs.setdefault("discriminator", None) + kwargs.setdefault("roles", None) + kwargs.setdefault("in_guild", None) + + return kwargs + + class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): """Tests for determining differences between users in the DB and users in the Guild cache.""" @@ -41,8 +51,13 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): return guild async def test_empty_diff_for_no_users(self): - # TODO: need to fix this test. """When no users are given, an empty diff should be returned.""" + self.bot.api_client.get.return_value = { + "count": 3, + "next": None, + "previous": None, + "results": [] + } guild = self.get_guild() actual_diff = await self.syncer._get_diff(guild) @@ -68,6 +83,7 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): async def test_diff_for_updated_users(self): """Only updated users should be added to the 'updated' set of the diff.""" updated_user = fake_user(id=99, name="new") + updated_user_none = fake_none_user(id=99, name="new") self.bot.api_client.get.return_value = { "count": 3, @@ -78,7 +94,7 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): guild = self.get_guild(updated_user, fake_user()) actual_diff = await self.syncer._get_diff(guild) - expected_diff = (set(), {_User(**updated_user)}, None) + expected_diff = (set(), {_User(**updated_user_none)}, None) self.assertEqual(actual_diff, expected_diff) @@ -101,7 +117,7 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): async def test_diff_sets_in_guild_false_for_leaving_users(self): """When a user leaves the guild, the `in_guild` flag is updated to `False`.""" - leaving_user = fake_user(id=63, in_guild=False) + leaving_user_none = fake_none_user(id=63, in_guild=False) self.bot.api_client.get.return_value = { "count": 3, @@ -112,15 +128,18 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): guild = self.get_guild(fake_user()) actual_diff = await self.syncer._get_diff(guild) - expected_diff = (set(), {_User(**leaving_user)}, None) + expected_diff = (set(), {_User(**leaving_user_none)}, None) self.assertEqual(actual_diff, expected_diff) async def test_diff_for_new_updated_and_leaving_users(self): """When users are added, updated, and removed, all of them are returned properly.""" new_user = fake_user(id=99, name="new") + updated_user = fake_user(id=55, name="updated") - leaving_user = fake_user(id=63, in_guild=False) + updated_user_none = fake_none_user(id=55, name="updated") + + leaving_user_none = fake_none_user(id=63, in_guild=False) self.bot.api_client.get.return_value = { "count": 3, @@ -131,7 +150,14 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): guild = self.get_guild(fake_user(), new_user, updated_user) actual_diff = await self.syncer._get_diff(guild) - expected_diff = ({_User(**new_user)}, {_User(**updated_user), _User(**leaving_user)}, None) + expected_diff = ( + {_User(**new_user)}, + { + _User(**updated_user_none), + _User(**leaving_user_none) + }, + None + ) self.assertEqual(actual_diff, expected_diff) -- cgit v1.2.3 From 20c85e6fc46ab34fdce23e393a12e275a82a25fa Mon Sep 17 00:00:00 2001 From: RohanJnr Date: Fri, 2 Oct 2020 23:37:15 +0530 Subject: Refactor unit tests UserSyncerSyncTests to use changes made to UserSyncer in _syncers.py --- tests/bot/exts/backend/sync/test_users.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/bot/exts/backend/sync/test_users.py b/tests/bot/exts/backend/sync/test_users.py index e60c3a24d..c3a486743 100644 --- a/tests/bot/exts/backend/sync/test_users.py +++ b/tests/bot/exts/backend/sync/test_users.py @@ -1,5 +1,4 @@ import unittest -from unittest import mock from bot.exts.backend.sync._syncers import UserSyncer, _Diff, _User from tests import helpers @@ -192,9 +191,9 @@ class UserSyncerSyncTests(unittest.IsolatedAsyncioTestCase): diff = _Diff(user_tuples, set(), None) await self.syncer._sync(diff) - calls = [mock.call("bot/users", json=user) for user in users] - self.bot.api_client.post.assert_has_calls(calls, any_order=True) - self.assertEqual(self.bot.api_client.post.call_count, len(users)) + # Convert namedtuples to dicts as done in self.syncer._sync method. + created = [user._asdict() for user in diff.created] + self.bot.api_client.post.assert_called_once_with("bot/users", json=created) self.bot.api_client.put.assert_not_called() self.bot.api_client.delete.assert_not_called() @@ -207,9 +206,8 @@ class UserSyncerSyncTests(unittest.IsolatedAsyncioTestCase): diff = _Diff(set(), user_tuples, None) await self.syncer._sync(diff) - calls = [mock.call(f"bot/users/{user['id']}", json=user) for user in users] - self.bot.api_client.put.assert_has_calls(calls, any_order=True) - self.assertEqual(self.bot.api_client.put.call_count, len(users)) + updated = [self.syncer.patch_dict(user) for user in diff.updated] + self.bot.api_client.patch.assert_called_once_with("bot/users/bulk_patch", json=updated) self.bot.api_client.post.assert_not_called() self.bot.api_client.delete.assert_not_called() -- cgit v1.2.3 From 0f2acbc651b400c29ebdabdfab7f6f7e2debe68e Mon Sep 17 00:00:00 2001 From: RohanJnr Date: Fri, 2 Oct 2020 23:48:46 +0530 Subject: remove un-used variable --- bot/exts/backend/sync/_syncers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bot/exts/backend/sync/_syncers.py b/bot/exts/backend/sync/_syncers.py index 759af96d7..ae7d5d893 100644 --- a/bot/exts/backend/sync/_syncers.py +++ b/bot/exts/backend/sync/_syncers.py @@ -2,7 +2,6 @@ import abc import logging import typing as t from collections import namedtuple -from functools import partial from urllib.parse import parse_qsl, urlparse from discord import Guild -- cgit v1.2.3 From 7e7a801366e2bf8f1190fae91f93729b33f32895 Mon Sep 17 00:00:00 2001 From: RohanJnr Date: Wed, 7 Oct 2020 23:02:39 +0530 Subject: improve code efficiency and use updated API changes to pagination --- bot/exts/backend/sync/_syncers.py | 146 +++++++++++++------------------------- 1 file changed, 48 insertions(+), 98 deletions(-) diff --git a/bot/exts/backend/sync/_syncers.py b/bot/exts/backend/sync/_syncers.py index ae7d5d893..70887a217 100644 --- a/bot/exts/backend/sync/_syncers.py +++ b/bot/exts/backend/sync/_syncers.py @@ -2,7 +2,6 @@ import abc import logging import typing as t from collections import namedtuple -from urllib.parse import parse_qsl, urlparse from discord import Guild from discord.ext.commands import Context @@ -15,7 +14,6 @@ log = logging.getLogger(__name__) # These objects are declared as namedtuples because tuples are hashable, # something that we make use of when diffing site roles against guild roles. _Role = namedtuple('Role', ('id', 'name', 'colour', 'permissions', 'position')) -_User = namedtuple('User', ('id', 'name', 'discriminator', 'roles', 'in_guild')) _Diff = namedtuple('Diff', ('created', 'updated', 'deleted')) @@ -42,11 +40,7 @@ class Syncer(abc.ABC): raise NotImplementedError # pragma: no cover async def sync(self, guild: Guild, ctx: t.Optional[Context] = None) -> None: - """ - Synchronise the database with the cache of `guild`. - - If `ctx` is given, send a message with the results. - """ + """If `ctx` is given, send a message with the results.""" log.info(f"Starting {self.name} syncer.") if ctx: @@ -136,111 +130,67 @@ class UserSyncer(Syncer): """Return the difference of users between the cache of `guild` and the database.""" log.trace("Getting the diff for users.") - users = await self._get_users() + users_to_create = [] + users_to_update = [] + seen_guild_users = set() - # Pack DB roles and guild roles into one common, hashable format. - # They're hashable so that they're easily comparable with sets later. - db_users = { - user_dict['id']: _User( - roles=tuple(sorted(user_dict.pop('roles'))), - **user_dict - ) - for user_dict in users - } - guild_users = { - member.id: _User( - id=member.id, - name=member.name, - discriminator=int(member.discriminator), - roles=tuple(sorted(role.id for role in member.roles)), - in_guild=True - ) - for member in guild.members - } + async for db_user in self._get_users(): + updated_fields = {} - users_to_create = set() - users_to_update = set() - - for db_user in db_users.values(): - guild_user = guild_users.get(db_user.id) - - if guild_user is not None: - if db_user != guild_user: - fields_to_none: dict = {} - - for field in _User._fields: - # Set un-changed values to None except ID to speed up API PATCH method. - if getattr(db_user, field) == getattr(guild_user, field) and field != "id": - fields_to_none[field] = None - - new_api_user = guild_user._replace(**fields_to_none) - users_to_update.add(new_api_user) - - elif db_user.in_guild: - # The user is known in the DB but not the guild, and the - # DB currently specifies that the user is a member of the guild. - # This means that the user has left since the last sync. - # Update the `in_guild` attribute of the user on the site - # to signify that the user left. - - # Set un-changed fields to None except ID as it is required by the API. - fields_to_none: dict = {field: None for field in db_user._fields if field not in ["id", "in_guild"]} - new_api_user = db_user._replace( - in_guild=False, - **fields_to_none - ) - users_to_update.add(new_api_user) - - new_user_ids = set(guild_users.keys()) - set(db_users.keys()) - for user_id in new_user_ids: - # The user is known on the guild but not on the API. This means - # that the user has joined since the last sync. Create it. - new_user = guild_users[user_id] - users_to_create.add(new_user) + def maybe_update(db_field: str, guild_value: t.Union[str, int]) -> None: + if db_user[db_field] != guild_value: + updated_fields[db_field] = guild_value - return _Diff(users_to_create, users_to_update, None) + if guild_user := guild.get_member(db_user["id"]): + seen_guild_users.add(guild_user.id) + + maybe_update("name", guild_user.name) + maybe_update("discriminator", int(guild_user.discriminator)) + maybe_update("in_guild", True) - async def _get_users(self, endpoint: str = "bot/users", query_params: list = None) -> t.List[dict]: - """GET all users recursively.""" - users = [] - response: dict = await self.bot.api_client.get(endpoint, params=query_params) - users.extend(response["results"]) + guild_roles = [role.id for role in guild_user.roles] + if set(db_user["roles"]) != set(guild_roles): + updated_fields["roles"] = guild_roles - # The `response` is paginated, hence check if next page exists. - if (next_page_url := response["next"]) is not None: - next_endpoint, query_params = self.get_endpoint(next_page_url) - users.extend(await self._get_users(next_endpoint, query_params)) - return users + elif db_user["in_guild"]: + updated_fields["in_guild"] = False - @staticmethod - def get_endpoint(url: str) -> t.Tuple[str, t.List[tuple]]: - """Extract the API endpoint and query params from a URL.""" - url = urlparse(url) + if updated_fields and updated_fields not in users_to_update: + updated_fields["id"] = db_user["id"] + users_to_update.append(updated_fields) - # Do not include starting `/` for endpoint. - endpoint = url.path[1:] + for member in guild.members: + if member.id not in seen_guild_users: + new_user = { + "id": member.id, + "name": member.name, + "discriminator": int(member.discriminator), + "roles": [role.id for role in member.roles], + "in_guild": True + } + if new_user not in users_to_create: + users_to_create.append(new_user) - # Query params. - params = parse_qsl(url.query) + return _Diff(users_to_create, users_to_update, None) - return endpoint, params + async def _get_users(self) -> t.AsyncIterable: + """GET users from database.""" + query_params = { + "page": 1 + } + while query_params["page"]: + res = await self.bot.api_client.get("bot/users", params=query_params) + for user in res["results"]: + yield user - @staticmethod - def patch_dict(user: _User) -> t.Dict[str, t.Union[int, str, tuple, bool]]: - """Convert namedtuple to dict by omitting None values.""" - user_dict = {} - for field in user._fields: - if (value := getattr(user, field)) is not None: - user_dict[field] = value - return user_dict + query_params["page"] = res["next_page_no"] async def _sync(self, diff: _Diff) -> None: """Synchronise the database with the user cache of `guild`.""" log.trace("Syncing created users...") if diff.created: - created = [user._asdict() for user in diff.created] - await self.bot.api_client.post("bot/users", json=created) + await self.bot.api_client.post("bot/users", json=diff.created) + log.trace("Syncing updated users...") if diff.updated: - updated = [self.patch_dict(user) for user in diff.updated] - await self.bot.api_client.patch("bot/users/bulk_patch", json=updated) + await self.bot.api_client.patch("bot/users/bulk_patch", json=diff.updated) -- cgit v1.2.3 From 6ee08368186716804121cb456783e3bc56ced7f3 Mon Sep 17 00:00:00 2001 From: RohanJnr Date: Wed, 7 Oct 2020 23:20:13 +0530 Subject: Refactor tests to use updated changes to syncer.py and API. --- tests/bot/exts/backend/sync/test_users.py | 117 +++++++++++++++--------------- 1 file changed, 59 insertions(+), 58 deletions(-) diff --git a/tests/bot/exts/backend/sync/test_users.py b/tests/bot/exts/backend/sync/test_users.py index c3a486743..9f380a15d 100644 --- a/tests/bot/exts/backend/sync/test_users.py +++ b/tests/bot/exts/backend/sync/test_users.py @@ -1,6 +1,6 @@ import unittest -from bot.exts.backend.sync._syncers import UserSyncer, _Diff, _User +from bot.exts.backend.sync._syncers import UserSyncer, _Diff from tests import helpers @@ -9,22 +9,12 @@ def fake_user(**kwargs): kwargs.setdefault("id", 43) kwargs.setdefault("name", "bob the test man") kwargs.setdefault("discriminator", 1337) - kwargs.setdefault("roles", (666,)) + kwargs.setdefault("roles", [666]) kwargs.setdefault("in_guild", True) return kwargs -def fake_none_user(**kwargs): - kwargs.setdefault("id", None) - kwargs.setdefault("name", None) - kwargs.setdefault("discriminator", None) - kwargs.setdefault("roles", None) - kwargs.setdefault("in_guild", None) - - return kwargs - - class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): """Tests for determining differences between users in the DB and users in the Guild cache.""" @@ -49,18 +39,26 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): return guild + @staticmethod + def get_mock_member(member: dict): + member = member.copy() + del member["in_guild"] + mock_member = helpers.MockMember(**member) + mock_member.roles = [helpers.MockRole(id=role_id) for role_id in member["roles"]] + return mock_member + async def test_empty_diff_for_no_users(self): """When no users are given, an empty diff should be returned.""" self.bot.api_client.get.return_value = { "count": 3, - "next": None, - "previous": None, + "next_page_no": None, + "previous_page_no": None, "results": [] } guild = self.get_guild() actual_diff = await self.syncer._get_diff(guild) - expected_diff = (set(), set(), None) + expected_diff = ([], [], None) self.assertEqual(actual_diff, expected_diff) @@ -68,66 +66,75 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): """No differences should be found if the users in the guild and DB are identical.""" self.bot.api_client.get.return_value = { "count": 3, - "next": None, - "previous": None, + "next_page_no": None, + "previous_page_no": None, "results": [fake_user()] } guild = self.get_guild(fake_user()) + guild.get_member.return_value = self.get_mock_member(fake_user()) actual_diff = await self.syncer._get_diff(guild) - expected_diff = (set(), set(), None) + expected_diff = ([], [], None) self.assertEqual(actual_diff, expected_diff) async def test_diff_for_updated_users(self): """Only updated users should be added to the 'updated' set of the diff.""" updated_user = fake_user(id=99, name="new") - updated_user_none = fake_none_user(id=99, name="new") self.bot.api_client.get.return_value = { "count": 3, - "next": None, - "previous": None, + "next_page_no": None, + "previous_page_no": None, "results": [fake_user(id=99, name="old"), fake_user()] } guild = self.get_guild(updated_user, fake_user()) + guild.get_member.side_effect = [ + self.get_mock_member(updated_user), + self.get_mock_member(fake_user()) + ] actual_diff = await self.syncer._get_diff(guild) - expected_diff = (set(), {_User(**updated_user_none)}, None) + expected_diff = ([], [{"id": 99, "name": "new"}], None) self.assertEqual(actual_diff, expected_diff) async def test_diff_for_new_users(self): - """Only new users should be added to the 'created' set of the diff.""" + """Only new users should be added to the 'created' list of the diff.""" new_user = fake_user(id=99, name="new") self.bot.api_client.get.return_value = { "count": 3, - "next": None, - "previous": None, + "next_page_no": None, + "previous_page_no": None, "results": [fake_user()] } guild = self.get_guild(fake_user(), new_user) - + guild.get_member.side_effect = [ + self.get_mock_member(fake_user()), + self.get_mock_member(new_user) + ] actual_diff = await self.syncer._get_diff(guild) - expected_diff = ({_User(**new_user)}, set(), None) + expected_diff = ([new_user], [], None) self.assertEqual(actual_diff, expected_diff) async def test_diff_sets_in_guild_false_for_leaving_users(self): """When a user leaves the guild, the `in_guild` flag is updated to `False`.""" - leaving_user_none = fake_none_user(id=63, in_guild=False) - self.bot.api_client.get.return_value = { "count": 3, - "next": None, - "previous": None, + "next_page_no": None, + "previous_page_no": None, "results": [fake_user(), fake_user(id=63)] } guild = self.get_guild(fake_user()) + guild.get_member.side_effect = [ + self.get_mock_member(fake_user()), + None + ] actual_diff = await self.syncer._get_diff(guild) - expected_diff = (set(), {_User(**leaving_user_none)}, None) + expected_diff = ([], [{"id": 63, "in_guild": False}], None) self.assertEqual(actual_diff, expected_diff) @@ -136,42 +143,41 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase): new_user = fake_user(id=99, name="new") updated_user = fake_user(id=55, name="updated") - updated_user_none = fake_none_user(id=55, name="updated") - - leaving_user_none = fake_none_user(id=63, in_guild=False) self.bot.api_client.get.return_value = { "count": 3, - "next": None, - "previous": None, + "next_page_no": None, + "previous_page_no": None, "results": [fake_user(), fake_user(id=55), fake_user(id=63)] } guild = self.get_guild(fake_user(), new_user, updated_user) + guild.get_member.side_effect = [ + self.get_mock_member(fake_user()), + self.get_mock_member(updated_user), + None + ] actual_diff = await self.syncer._get_diff(guild) - expected_diff = ( - {_User(**new_user)}, - { - _User(**updated_user_none), - _User(**leaving_user_none) - }, - None - ) + expected_diff = ([new_user], [{"id": 55, "name": "updated"}, {"id": 63, "in_guild": False}], None) self.assertEqual(actual_diff, expected_diff) async def test_empty_diff_for_db_users_not_in_guild(self): - """When the DB knows a user the guild doesn't, no difference is found.""" + """When the DB knows a user, but the guild doesn't, no difference is found.""" self.bot.api_client.get.return_value = { "count": 3, - "next": None, - "previous": None, + "next_page_no": None, + "previous_page_no": None, "results": [fake_user(), fake_user(id=63, in_guild=False)] } guild = self.get_guild(fake_user()) + guild.get_member.side_effect = [ + self.get_mock_member(fake_user()), + None + ] actual_diff = await self.syncer._get_diff(guild) - expected_diff = (set(), set(), None) + expected_diff = ([], [], None) self.assertEqual(actual_diff, expected_diff) @@ -187,13 +193,10 @@ class UserSyncerSyncTests(unittest.IsolatedAsyncioTestCase): """Only POST requests should be made with the correct payload.""" users = [fake_user(id=111), fake_user(id=222)] - user_tuples = {_User(**user) for user in users} - diff = _Diff(user_tuples, set(), None) + diff = _Diff(users, [], None) await self.syncer._sync(diff) - # Convert namedtuples to dicts as done in self.syncer._sync method. - created = [user._asdict() for user in diff.created] - self.bot.api_client.post.assert_called_once_with("bot/users", json=created) + self.bot.api_client.post.assert_called_once_with("bot/users", json=diff.created) self.bot.api_client.put.assert_not_called() self.bot.api_client.delete.assert_not_called() @@ -202,12 +205,10 @@ class UserSyncerSyncTests(unittest.IsolatedAsyncioTestCase): """Only PUT requests should be made with the correct payload.""" users = [fake_user(id=111), fake_user(id=222)] - user_tuples = {_User(**user) for user in users} - diff = _Diff(set(), user_tuples, None) + diff = _Diff([], users, None) await self.syncer._sync(diff) - updated = [self.syncer.patch_dict(user) for user in diff.updated] - self.bot.api_client.patch.assert_called_once_with("bot/users/bulk_patch", json=updated) + self.bot.api_client.patch.assert_called_once_with("bot/users/bulk_patch", json=diff.updated) self.bot.api_client.post.assert_not_called() self.bot.api_client.delete.assert_not_called() -- cgit v1.2.3 From 586aeb66e9156259efbdfed43c11b66003185ad2 Mon Sep 17 00:00:00 2001 From: RohanJnr Date: Thu, 8 Oct 2020 10:40:11 +0530 Subject: remove redundant if statement --- bot/exts/backend/sync/_syncers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bot/exts/backend/sync/_syncers.py b/bot/exts/backend/sync/_syncers.py index 70887a217..3a7719559 100644 --- a/bot/exts/backend/sync/_syncers.py +++ b/bot/exts/backend/sync/_syncers.py @@ -168,8 +168,7 @@ class UserSyncer(Syncer): "roles": [role.id for role in member.roles], "in_guild": True } - if new_user not in users_to_create: - users_to_create.append(new_user) + users_to_create.append(new_user) return _Diff(users_to_create, users_to_update, None) -- cgit v1.2.3 From 72819f275658f0637deb2d7fba9a838d65294203 Mon Sep 17 00:00:00 2001 From: RohanJnr Date: Thu, 8 Oct 2020 10:41:52 +0530 Subject: remove redundant if statement --- bot/exts/backend/sync/_syncers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/exts/backend/sync/_syncers.py b/bot/exts/backend/sync/_syncers.py index 3a7719559..c32038f4e 100644 --- a/bot/exts/backend/sync/_syncers.py +++ b/bot/exts/backend/sync/_syncers.py @@ -155,7 +155,7 @@ class UserSyncer(Syncer): elif db_user["in_guild"]: updated_fields["in_guild"] = False - if updated_fields and updated_fields not in users_to_update: + if updated_fields: updated_fields["id"] = db_user["id"] users_to_update.append(updated_fields) -- cgit v1.2.3 From 9af0883deeb57b08044400335c759a206d5833fb Mon Sep 17 00:00:00 2001 From: RohanJnr Date: Thu, 8 Oct 2020 10:49:06 +0530 Subject: update documentation --- bot/exts/backend/sync/_syncers.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/bot/exts/backend/sync/_syncers.py b/bot/exts/backend/sync/_syncers.py index c32038f4e..38468c2b1 100644 --- a/bot/exts/backend/sync/_syncers.py +++ b/bot/exts/backend/sync/_syncers.py @@ -40,7 +40,11 @@ class Syncer(abc.ABC): raise NotImplementedError # pragma: no cover async def sync(self, guild: Guild, ctx: t.Optional[Context] = None) -> None: - """If `ctx` is given, send a message with the results.""" + """ + Synchronise the database with the cache of `guild`. + + If `ctx` is given, send a message with the results. + """ log.info(f"Starting {self.name} syncer.") if ctx: @@ -135,9 +139,11 @@ class UserSyncer(Syncer): seen_guild_users = set() async for db_user in self._get_users(): + # Store user fields which are to be updated. updated_fields = {} def maybe_update(db_field: str, guild_value: t.Union[str, int]) -> None: + # Equalize DB user and guild user attributes. if db_user[db_field] != guild_value: updated_fields[db_field] = guild_value @@ -153,6 +159,11 @@ class UserSyncer(Syncer): updated_fields["roles"] = guild_roles elif db_user["in_guild"]: + # The user is known in the DB but not the guild, and the + # DB currently specifies that the user is a member of the guild. + # This means that the user has left since the last sync. + # Update the `in_guild` attribute of the user on the site + # to signify that the user left. updated_fields["in_guild"] = False if updated_fields: @@ -161,6 +172,8 @@ class UserSyncer(Syncer): for member in guild.members: if member.id not in seen_guild_users: + # The user is known on the guild but not on the API. This means + # that the user has joined since the last sync. Create it. new_user = { "id": member.id, "name": member.name, -- cgit v1.2.3