From 12fe15baa5f3ea5d35d43d27ff671dca5fe58334 Mon Sep 17 00:00:00 2001 From: Amrou Bellalouna Date: Thu, 9 Mar 2023 07:34:54 +0100 Subject: Merge #2408: Scaffold server config via a bootstrapping script Refactor configuration into a pydantic-based python constants file, and add a utility to auto-populate guild data. Squashed commits: * use basic config for demo purposes * fix guiding comments * update var names for proper context reflection * fix wront iteration var * add all roles, & channels * load categories * separate sections in env file * ignore .env.server * rename change_log to changelog This also adds a default env file to look for * remove instantiation of webhooks * add most of the default configs These will mostly be fetched from the .env.default file, which won't be bootstrapped * warn when categories/roles/channels are not found * add env file to keep server defaults * fix malformatted value in the .env.default * add default server env variables * update the sections formatting in default env file * fallback to server env when loading constants * add guild basic defaults * update change_log channel name to changelog * add the Guid settings prefix * make _Guild inherit from EnvConfig * add webhook defaults * add python_news defaults to the server env * ad missing webhooks prefix * update bootstrapper logger name * update priority of the env loaded files According to Pydantic's docs: "Later files in the list/tuple will take priority over earlier files." * warn user that default value from PyDis' config will be used * add colours default config * add antispam config * update antispam references * add redis default cfg * add Stats, Cooldowns and CleanMessages consts This also includes their default values * add Metabase to constants This also includes its default values * add URLS to constants This also includes its default values * use the Field class to provide defaults This avoids overriding & changing the `fields` of the `Config` class "dynamically" * add keys constant class * add Guild conf * replace dash with underscore in script * appease linter * transform attributes of AntiSpam to dict when needed This ensures that the application stays backwards compatible * add root_validator for the colours class This enables the conversion from hex to int easily since it's not a supported type by pydantic * reinstate the role & channels combinations * rename URLS to URLs * add emojis & icons constants * add filter constants & their default values * remove all useless spaces * instantiate the keys class * add bot prefix to default env file * fetch Bot constants from env vars instead of the prefix ones * add Miscellaneous config * instantiate poor forgotten Miscellaneous config * add final touches to the constants module This includes removing dups, adding missing channels & fixing type casts * move all default values to constants.py This is done by using the `Field` class. It allows us to 1. Set defaults, in case the variables are not configured 2. Load them from a env variable under a specific name (for backwards comp) 3. load it from any env variable file that contains the right prefix * ignore all .env files * load BOT_TOKEN & GUILD_ID from .env * allow _GUILD to read its id from the `GUILD_ID` env var * base Webhooks settings off of a Webhook model * create necessary webhooks if non existent * appease flake8 docstrings error * make the script idempotent * update type hints * uppercase all consts * make webhook channel optional * add httpx to its own dependency group This group will be optional & only related to the bootstrapper * replace requests with httpx * pass client as param * include raise_for_status as a response hook * rename get_webhook to webhook_exists * update docstring of the constants module * use "." as a separator * update script to account for already created webhooks * make ANTI_SPAM_RULES a module level constant This ensures that flake8 doesn't complain about making a function call in the function's signature * remove the manual resolving of .env paths * update usages of AntiSpam constants * remove forgotten assignment of rule_config * remove useless assignments of env file names * delete default config-default.yml * update docstrings of CodeBlockCog to reference constants.py * add a poetry task that runs the bootstrapping script * add python-dotenv to the config-bootstrap group * update hook name to _raise_for_status * construct site_api in _URLs * remove __name__ == '__main__'guard * Revert "construct site_api in _URLs" This reverts commit 1c555c4280c6a0bdd452319cbd3ffcd0370f5d48. * remove usage of the Field class * update env var keys that the bootstrapping script needs * use API_KEYS.SITE_API as env var in docker compose instead of BOT_API_KEY * use basic config for demo purposes * fix guiding comments * update var names for proper context reflection * fix wront iteration var * add all roles, & channels * load categories * separate sections in env file * ignore .env.server * rename change_log to changelog This also adds a default env file to look for * remove instantiation of webhooks * add most of the default configs These will mostly be fetched from the .env.default file, which won't be bootstrapped * warn when categories/roles/channels are not found * add env file to keep server defaults * fix malformatted value in the .env.default * add default server env variables * update the sections formatting in default env file * fallback to server env when loading constants * add guild basic defaults * update change_log channel name to changelog * add the Guid settings prefix * make _Guild inherit from EnvConfig * add webhook defaults * add python_news defaults to the server env * ad missing webhooks prefix * update bootstrapper logger name * update priority of the env loaded files According to Pydantic's docs: "Later files in the list/tuple will take priority over earlier files." * warn user that default value from PyDis' config will be used * add colours default config * add antispam config * update antispam references * add redis default cfg * add Stats, Cooldowns and CleanMessages consts This also includes their default values * add Metabase to constants This also includes its default values * add URLS to constants This also includes its default values * use the Field class to provide defaults This avoids overriding & changing the `fields` of the `Config` class "dynamically" * add keys constant class * add Guild conf * replace dash with underscore in script * appease linter * transform attributes of AntiSpam to dict when needed This ensures that the application stays backwards compatible * add root_validator for the colours class This enables the conversion from hex to int easily since it's not a supported type by pydantic * reinstate the role & channels combinations * rename URLS to URLs * add emojis & icons constants * add filter constants & their default values * remove all useless spaces * instantiate the keys class * add bot prefix to default env file * fetch Bot constants from env vars instead of the prefix ones * add Miscellaneous config * instantiate poor forgotten Miscellaneous config * add final touches to the constants module This includes removing dups, adding missing channels & fixing type casts * move all default values to constants.py This is done by using the `Field` class. It allows us to 1. Set defaults, in case the variables are not configured 2. Load them from a env variable under a specific name (for backwards comp) 3. load it from any env variable file that contains the right prefix * ignore all .env files * load BOT_TOKEN & GUILD_ID from .env * allow _GUILD to read its id from the `GUILD_ID` env var * base Webhooks settings off of a Webhook model * create necessary webhooks if non existent * appease flake8 docstrings error * make the script idempotent * update type hints * uppercase all consts * make webhook channel optional * add httpx to its own dependency group This group will be optional & only related to the bootstrapper * replace requests with httpx * pass client as param * include raise_for_status as a response hook * rename get_webhook to webhook_exists * update docstring of the constants module * use "." as a separator * update script to account for already created webhooks * make ANTI_SPAM_RULES a module level constant This ensures that flake8 doesn't complain about making a function call in the function's signature * remove the manual resolving of .env paths * update usages of AntiSpam constants * remove forgotten assignment of rule_config * remove useless assignments of env file names * delete default config-default.yml * update docstrings of CodeBlockCog to reference constants.py * add a poetry task that runs the bootstrapping script * add python-dotenv to the config-bootstrap group * update hook name to _raise_for_status * construct site_api in _URLs * remove __name__ == '__main__'guard * Revert "construct site_api in _URLs" This reverts commit 1c555c4280c6a0bdd452319cbd3ffcd0370f5d48. * remove usage of the Field class * update env var keys that the bootstrapping script needs * use API_KEYS.SITE_API as env var in docker compose instead of BOT_API_KEY * relock dependencies * update snekbox's defaults * add support for ot channels * rename help_system_forum to python_help * rename nomination_archive to nomination_voting_archive * rename appeals2 to appeals_2 * yeet sprinters role out * rename all big_brother_logs instances to big_brother The purpose is to adhere to what we have in prod * rename bootstrap_config.py to botstrap.py * update module name of the configure poetry task * update error messages to reflect the new keys needed for env variables * install dotenv as an extra with pydantic * update all prefixes to "_" (underscore) * log tuple of (channel_name, channel_id) in the config verifier * update needed default values for docker compose env var * relock dependencies * update forgotten delimiters & env prefixes --- botstrap.py | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 botstrap.py (limited to 'botstrap.py') diff --git a/botstrap.py b/botstrap.py new file mode 100644 index 000000000..4b00be9aa --- /dev/null +++ b/botstrap.py @@ -0,0 +1,164 @@ +import os +import re +from pathlib import Path + +from dotenv import load_dotenv +from httpx import Client, HTTPStatusError, Response + +from bot.constants import Webhooks, _Categories, _Channels, _Roles +from bot.log import get_logger + +load_dotenv() +log = get_logger("Config Bootstrapper") + +env_file_path = Path(".env.server") +BOT_TOKEN = os.getenv("BOT_TOKEN", None) +GUILD_ID = os.getenv("GUILD_ID", None) + + +if not BOT_TOKEN: + message = ( + "Couldn't find BOT_TOKEN in the environment variables." + "Make sure to add it to the `.env` file likewise: `BOT_TOKEN=value_of_your_bot_token`" + ) + log.warning(message) + raise ValueError(message) + +if not GUILD_ID: + message = ( + "Couldn't find GUILD_ID in the environment variables." + "Make sure to add it to the `.env` file likewise: `GUILD_ID=value_of_your_discord_server_id`" + ) + log.warning(message) + raise ValueError(message) + + +class DiscordClient(Client): + """An HTTP client to communicate with Discord's APIs.""" + + def __init__(self): + super().__init__( + base_url="https://discord.com/api/v10", + headers={"Authorization": f"Bot {BOT_TOKEN}"}, + event_hooks={"response": [self._raise_for_status]} + ) + + @staticmethod + def _raise_for_status(response: Response) -> None: + response.raise_for_status() + + +def get_all_roles(guild_id: int | str, client: DiscordClient) -> dict: + """Fetches all the roles in a guild.""" + result = {} + + response = client.get(f"guilds/{guild_id}/roles") + roles = response.json() + + for role in roles: + name = "_".join(part.lower() for part in role["name"].split(" ")).replace("-", "_") + result[name] = role["id"] + + return result + + +def get_all_channels_and_categories( + guild_id: int | str, + client: DiscordClient +) -> tuple[dict[str, str], dict[str, str]]: + """Fetches all the text channels & categories in a guild.""" + off_topic_channel_name_regex = r"ot\d{1}(_.*)+" + off_topic_count = 0 + channels = {} # could be text channels only as well + categories = {} + + response = client.get(f"guilds/{guild_id}/channels") + server_channels = response.json() + + for channel in server_channels: + channel_type = channel["type"] + name = "_".join(part.lower() for part in channel["name"].split(" ")).replace("-", "_") + if re.match(off_topic_channel_name_regex, name): + name = f"off_topic_{off_topic_count}" + off_topic_count += 1 + + if channel_type == 4: + categories[name] = channel["id"] + else: + channels[name] = channel["id"] + + return channels, categories + + +def webhook_exists(webhook_id_: int, client: DiscordClient) -> bool: + """A predicate that indicates whether a webhook exists already or not.""" + try: + client.get(f"webhooks/{webhook_id_}") + return True + except HTTPStatusError: + return False + + +def create_webhook(name: str, channel_id_: int, client: DiscordClient) -> str: + """Creates a new webhook for a particular channel.""" + payload = {"name": name} + + response = client.post(f"channels/{channel_id_}/webhooks", json=payload) + new_webhook = response.json() + return new_webhook["id"] + + +with DiscordClient() as discord_client: + config_str = "#Roles\n" + + all_roles = get_all_roles(guild_id=GUILD_ID, client=discord_client) + + for role_name in _Roles.__fields__: + + role_id = all_roles.get(role_name, None) + if not role_id: + log.warning(f"Couldn't find the role {role_name} in the guild, PyDis' default values will be used.") + continue + + config_str += f"roles_{role_name}={role_id}\n" + + all_channels, all_categories = get_all_channels_and_categories(guild_id=GUILD_ID, client=discord_client) + + config_str += "\n#Channels\n" + + for channel_name in _Channels.__fields__: + channel_id = all_channels.get(channel_name, None) + if not channel_id: + log.warning( + f"Couldn't find the channel {channel_name} in the guild, PyDis' default values will be used." + ) + continue + + config_str += f"channels_{channel_name}={channel_id}\n" + + config_str += "\n#Categories\n" + + for category_name in _Categories.__fields__: + category_id = all_categories.get(category_name, None) + if not category_id: + log.warning( + f"Couldn't find the category {category_name} in the guild, PyDis' default values will be used." + ) + continue + + config_str += f"categories_{category_name}={category_id}\n" + + env_file_path.write_text(config_str) + + config_str += "\n#Webhooks\n" + + for webhook_name, webhook_model in Webhooks: + webhook = webhook_exists(webhook_model.id, client=discord_client) + if not webhook: + webhook_channel_id = int(all_channels[webhook_name]) + webhook_id = create_webhook(webhook_name, webhook_channel_id, client=discord_client) + else: + webhook_id = webhook_model.id + config_str += f"webhooks_{webhook_name}.id={webhook_id}\n" + + env_file_path.write_text(config_str) -- cgit v1.2.3