Merge pull request #1436 from python-discord/startup-errors

Helpful Startup Errors

4 files changed, 59 insertions, 5 deletions

diff --git a/bot/__main__.py b/bot/__main__.py
index 257216fa7..9317563c8 100644
--- a/bot/__main__.py
+++ b/bot/__main__.py
@@ -1,10 +1,28 @@
+import logging
+
+import aiohttp
+
 import bot
 from bot import constants
-from bot.bot import Bot
+from bot.bot import Bot, StartupError
 from bot.log import setup_sentry
 
 setup_sentry()
 
-bot.instance = Bot.create()
-bot.instance.load_extensions()
-bot.instance.run(constants.Bot.token)
+try:
+    bot.instance = Bot.create()
+    bot.instance.load_extensions()
+    bot.instance.run(constants.Bot.token)
+except StartupError as e:
+    message = "Unknown Startup Error Occurred."
+    if isinstance(e.exception, (aiohttp.ClientConnectorError, aiohttp.ServerDisconnectedError)):
+        message = "Could not connect to site API. Is it running?"
+    elif isinstance(e.exception, OSError):
+        message = "Could not connect to Redis. Is it running?"
+
+    # The exception is logged with an empty message so the actual message is visible at the bottom
+    log = logging.getLogger("bot")
+    log.fatal("", exc_info=e.exception)
+    log.fatal(message)
+
+    exit(69)
diff --git a/bot/bot.py b/bot/bot.py
index d5f108575..3a2af472d 100644
--- a/bot/bot.py
+++ b/bot/bot.py
@@ -19,6 +19,14 @@ log = logging.getLogger('bot')
 LOCALHOST = "127.0.0.1"
 
 
+class StartupError(Exception):
+    """Exception class for startup errors."""
+
+    def __init__(self, base: Exception):
+        super().__init__()
+        self.exception = base
+
+
 class Bot(commands.Bot):
     """A subclass of `discord.ext.commands.Bot` with an aiohttp session and an API client."""
 
@@ -81,6 +89,22 @@ class Bot(commands.Bot):
         for item in full_cache:
             self.insert_item_into_filter_list_cache(item)
 
+    async def ping_services(self) -> None:
+        """A helper to make sure all the services the bot relies on are available on startup."""
+        # Connect Site/API
+        attempts = 0
+        while True:
+            try:
+                log.info(f"Attempting site connection: {attempts + 1}/{constants.URLs.connect_max_retries}")
+                await self.api_client.get("healthcheck")
+                break
+
+            except (aiohttp.ClientConnectorError, aiohttp.ServerDisconnectedError):
+                attempts += 1
+                if attempts == constants.URLs.connect_max_retries:
+                    raise
+                await asyncio.sleep(constants.URLs.connect_cooldown)
+
     @classmethod
     def create(cls) -> "Bot":
         """Create and return an instance of a Bot."""
@@ -223,6 +247,11 @@ class Bot(commands.Bot):
             # here. Normally, this shouldn't happen.
             await self.redis_session.connect()
 
+        try:
+            await self.ping_services()
+        except Exception as e:
+            raise StartupError(e)
+
         # Build the FilterList cache
         await self.cache_filter_list_data()
 
@@ -318,5 +347,8 @@ def _create_redis_session(loop: asyncio.AbstractEventLoop) -> RedisSession:
         use_fakeredis=constants.Redis.use_fakeredis,
         global_namespace="bot",
     )
-    loop.run_until_complete(redis_session.connect())
+    try:
+        loop.run_until_complete(redis_session.connect())
+    except OSError as e:
+        raise StartupError(e)
     return redis_session
diff --git a/bot/constants.py b/bot/constants.py
index 69bc82b89..7cf31e835 100644
--- a/bot/constants.py
+++ b/bot/constants.py
@@ -531,6 +531,8 @@ class URLs(metaclass=YAMLGetter):
     github_bot_repo: str
 
     # Base site vars
+    connect_max_retries: int
+    connect_cooldown: int
     site: str
     site_api: str
     site_schema: str
diff --git a/config-default.yml b/config-default.yml
index 7d9afaa0e..a9fb2262e 100644
--- a/config-default.yml
+++ b/config-default.yml
@@ -338,6 +338,8 @@ keys:
 
 urls:
     # PyDis site vars
+    connect_max_retries:       3
+    connect_cooldown:          5
     site:        &DOMAIN       "pythondiscord.com"
     site_api:    &API          "pydis-api.default.svc.cluster.local"
     site_api_schema:           "http://"