Aiohttp fixes (#27)

* Attempt to fix issues with aiohttp spawning way too many threads

* Add aiodns to requirements

* Math doesn't work for me

`Fatal Python error: failed to get random numbers to initialize Python`

8 files changed, 71 insertions, 79 deletions

diff --git a/bot/__main__.py b/bot/__main__.py
index 766540bc6..8025f66a4 100644
--- a/bot/__main__.py
+++ b/bot/__main__.py
@@ -1,6 +1,8 @@
 # coding=utf-8
 import os
 
+from aiohttp import AsyncResolver, ClientSession, TCPConnector
+
 from discord import Game
 from discord.ext.commands import AutoShardedBot, when_mentioned_or
 
@@ -19,8 +21,12 @@ bot = AutoShardedBot(
     formatter=Formatter()
 )
 
+# Make cog names case-insensitive
 bot.cogs = CaseInsensitiveDict()
 
+# Global aiohttp session for all cogs - uses asyncio for DNS resolution instead of threads, so we don't *spam threads*
+bot.http_session = ClientSession(connector=TCPConnector(resolver=AsyncResolver()))
+
 # Internal/debug
 bot.load_extension("bot.cogs.logging")
 bot.load_extension("bot.cogs.security")
@@ -39,3 +45,5 @@ bot.load_extension("bot.cogs.tags")
 bot.load_extension("bot.cogs.verification")
 
 bot.run(os.environ.get("BOT_TOKEN"))
+
+bot.http_session.close()  # Close the aiohttp session when the bot finishes running
diff --git a/bot/cogs/bot.py b/bot/cogs/bot.py
index 0ead73721..a29aff9d5 100644
--- a/bot/cogs/bot.py
+++ b/bot/cogs/bot.py
@@ -1,9 +1,9 @@
 # coding=utf-8
-# import ast
+import ast
 import re
-# import time
+import time
 
-from discord import Embed  # Message
+from discord import Embed, Message
 from discord.ext.commands import AutoShardedBot, Context, command, group
 
 from dulwich.repo import Repo
@@ -104,7 +104,6 @@ class Bot:
                 content = content.strip()
                 return content
 
-    """
     async def on_message(self, msg: Message):
         if msg.channel.id in self.channel_cooldowns:
             on_cooldown = time.time() - self.channel_cooldowns[msg.channel.id] < 300
@@ -136,7 +135,6 @@ class Bot:
                 except SyntaxError:
                     # todo: add logging
                     pass
-    """
 
 
 def setup(bot):
diff --git a/bot/cogs/clickup.py b/bot/cogs/clickup.py
index aa6fa4a62..db94a96fa 100644
--- a/bot/cogs/clickup.py
+++ b/bot/cogs/clickup.py
@@ -1,6 +1,4 @@
 # coding=utf-8
-from aiohttp import ClientSession
-
 from discord import Colour, Embed
 from discord.ext.commands import AutoShardedBot, Context, command
 
@@ -41,9 +39,10 @@ class ClickUp:
         self.lists = CaseInsensitiveDict()
 
     async def on_ready(self):
-        with ClientSession() as session:
-            response = await session.get(PROJECTS_URL.format(space_id=CLICKUP_SPACE), headers=HEADERS)
-            result = await response.json()
+        response = await self.bot.http_session.get(
+            PROJECTS_URL.format(space_id=CLICKUP_SPACE), headers=HEADERS
+        )
+        result = await response.json()
 
         if "err" in result:
             print(f"Failed to get ClickUp lists: `{result['ECODE']}`: {result['err']}")
@@ -89,9 +88,10 @@ class ClickUp:
         if status and status != "*":
             params["statuses[]"] = status
 
-        with ClientSession() as session:
-            response = await session.get(GET_TASKS_URL.format(team_id=CLICKUP_TEAM), headers=HEADERS, params=params)
-            result = await response.json()
+        response = await self.bot.http_session.get(
+            GET_TASKS_URL.format(team_id=CLICKUP_TEAM), headers=HEADERS, params=params
+        )
+        result = await response.json()
 
         if "err" in result:
             embed.description = f"`{result['ECODE']}`: {result['err']}"
@@ -135,9 +135,10 @@ class ClickUp:
         params.add("statuses[]", "review")
         params.add("statuses[]", "Closed")
 
-        with ClientSession() as session:
-            response = await session.get(GET_TASKS_URL.format(team_id=CLICKUP_TEAM), headers=HEADERS, params=params)
-            result = await response.json()
+        response = await self.bot.http_session.get(
+            GET_TASKS_URL.format(team_id=CLICKUP_TEAM), headers=HEADERS, params=params
+        )
+        result = await response.json()
 
         if "err" in result:
             embed.description = f"`{result['ECODE']}`: {result['err']}"
@@ -184,9 +185,10 @@ class ClickUp:
         Get a list of every member of the team
         """
 
-        with ClientSession() as session:
-            response = await session.get(TEAM_URL.format(team_id=CLICKUP_TEAM), headers=HEADERS)
-            result = await response.json()
+        response = await self.bot.http_session.get(
+            TEAM_URL.format(team_id=CLICKUP_TEAM), headers=HEADERS
+        )
+        result = await response.json()
 
         if "err" in result:
             embed = Embed(
@@ -219,9 +221,10 @@ class ClickUp:
         Get all the lists belonging to the ClickUp space
         """
 
-        with ClientSession() as session:
-            response = await session.get(PROJECTS_URL.format(space_id=CLICKUP_SPACE), headers=HEADERS)
-            result = await response.json()
+        response = await self.bot.http_session.get(
+            PROJECTS_URL.format(space_id=CLICKUP_SPACE), headers=HEADERS
+        )
+        result = await response.json()
 
         if "err" in result:
             embed = Embed(
@@ -278,14 +281,13 @@ class ClickUp:
             embed.description = f"Unknown list: {task_list}"
             return await ctx.send(embed=embed)
 
-        with ClientSession() as session:
-            response = await session.post(
-                CREATE_TASK_URL.format(list_id=task_list), headers=HEADERS, json={
-                    "name": title,
-                    "status": "Open"
-                }
-            )
-            result = await response.json()
+        response = await self.bot.http_session.post(
+            CREATE_TASK_URL.format(list_id=task_list), headers=HEADERS, json={
+                "name": title,
+                "status": "Open"
+            }
+        )
+        result = await response.json()
 
         if "err" in result:
             embed.colour = Colour.red()
@@ -318,11 +320,10 @@ class ClickUp:
             embed.colour = Colour.red()
             embed.description = f"Unknown status: {status}"
         else:
-            with ClientSession() as session:
-                response = await session.put(
-                    EDIT_TASK_URL.format(task_id=task_id), headers=HEADERS, json={"status": status}
-                )
-                result = await response.json()
+            response = await self.bot.http_session.put(
+                EDIT_TASK_URL.format(task_id=task_id), headers=HEADERS, json={"status": status}
+            )
+            result = await response.json()
 
             if "err" in result:
                 embed.description = f"`{result['ECODE']}`: {result['err']}"
diff --git a/bot/cogs/deployment.py b/bot/cogs/deployment.py
index 0e07ad12b..dcc478917 100644
--- a/bot/cogs/deployment.py
+++ b/bot/cogs/deployment.py
@@ -1,6 +1,4 @@
 # coding=utf-8
-from aiohttp import ClientSession
-
 from discord import Colour, Embed
 from discord.ext.commands import AutoShardedBot, Context, command
 
@@ -23,9 +21,8 @@ class Deployment:
         Trigger bot deployment on the server - will only redeploy if there were changes to deploy
         """
 
-        with ClientSession() as session:
-            response = await session.get(DEPLOY_URL, headers={"token": DEPLOY_BOT_KEY})
-            result = response.text()
+        response = await self.bot.http_session.get(DEPLOY_URL, headers={"token": DEPLOY_BOT_KEY})
+        result = response.text()
 
         if result == "True":
             await ctx.send(f"{ctx.author.mention} Bot deployment started.")
@@ -39,9 +36,8 @@ class Deployment:
         Trigger website deployment on the server - will only redeploy if there were changes to deploy
         """
 
-        with ClientSession() as session:
-            response = await session.get(DEPLOY_URL, headers={"token": DEPLOY_SITE_KEY})
-            result = response.text()
+        response = await self.bot.http_session.get(DEPLOY_URL, headers={"token": DEPLOY_SITE_KEY})
+        result = response.text()
 
         if result == "True":
             await ctx.send(f"{ctx.author.mention} Site deployment started.")
@@ -55,9 +51,8 @@ class Deployment:
         Check the various deployment uptimes for each service
         """
 
-        with ClientSession() as session:
-            response = await session.get(STATUS_URL)
-            data = await response.json()
+        response = await self.bot.http_session.get(STATUS_URL)
+        data = await response.json()
 
         embed = Embed(
             title="Service status",
diff --git a/bot/cogs/events.py b/bot/cogs/events.py
index f689fcd92..fb258454f 100644
--- a/bot/cogs/events.py
+++ b/bot/cogs/events.py
@@ -1,6 +1,4 @@
 # coding=utf-8
-from aiohttp import ClientSession
-
 from discord import Embed, Member
 from discord.ext.commands import (
     AutoShardedBot, BadArgument, BotMissingPermissions,
@@ -23,13 +21,13 @@ class Events:
 
     async def send_updated_users(self, *users):
         try:
-            with ClientSession(headers={"X-API-Key": SITE_API_KEY}) as session:
-                response = await session.post(
-                    url=SITE_API_USER_URL,
-                    json=list(users)
-                )
+            response = await self.bot.http_session.post(
+                url=SITE_API_USER_URL,
+                json=list(users),
+                headers={"X-API-ey": SITE_API_KEY}
+            )
 
-                return await response.json()
+            return await response.json()
         except Exception as e:
             print(f"Failed to send role updates: {e}")
             return {}
diff --git a/bot/cogs/math.py b/bot/cogs/math.py
index 0ef045748..649cf4915 100644
--- a/bot/cogs/math.py
+++ b/bot/cogs/math.py
@@ -8,8 +8,6 @@ from io import BytesIO
 from re import search
 from subprocess import PIPE, Popen, STDOUT, TimeoutExpired  # noqa: B404
 
-from aiohttp import ClientSession
-
 from discord import File
 from discord.ext.commands import command
 
@@ -78,14 +76,13 @@ class Math:
                 "color": 808080
             }
 
-            async with ClientSession() as session:
-                async with session.post(LATEX_URL, data=data) as resp:
-                    html = await resp.text()
+            async with self.bot.http_session.post(LATEX_URL, data=data) as resp:
+                html = await resp.text()
 
-                name = search(r'hist\.request\.basename = "(?P<url>[^"]+)"', html).group('url')
+            name = search(r'hist\.request\.basename = "(?P<url>[^"]+)"', html).group('url')
 
-                async with session.get(f"{LATEX_URL}/output/{name}.png") as resp:
-                    bytes_img = await resp.read()
+            async with self.bot.http_session.get(f"{LATEX_URL}/output/{name}.png") as resp:
+                bytes_img = await resp.read()
 
             file = File(fp=BytesIO(bytes_img), filename="latex.png")
 
@@ -119,14 +116,13 @@ class Math:
                 "color": 808080
             }
 
-            async with ClientSession() as session:
-                async with session.post(LATEX_URL, data=data) as resp:
-                    html = await resp.text()
+            async with self.bot.http_session.post(LATEX_URL, data=data) as resp:
+                html = await resp.text()
 
-                name = search(r'hist\.request\.basename = "(?P<url>[^"]+)"', html).group('url')
+            name = search(r'hist\.request\.basename = "(?P<url>[^"]+)"', html).group('url')
 
-                async with session.get(f"{LATEX_URL}/output/{name}.png") as resp:
-                    bytes_img = await resp.read()
+            async with self.bot.http_session.get(f"{LATEX_URL}/output/{name}.png") as resp:
+                bytes_img = await resp.read()
 
             file = File(fp=BytesIO(bytes_img), filename="latex.png")
 
diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py
index be57ed483..fc24cc47f 100644
--- a/bot/cogs/tags.py
+++ b/bot/cogs/tags.py
@@ -1,7 +1,5 @@
 import time
 
-from aiohttp import ClientSession
-
 from discord import Colour, Embed
 from discord.ext.commands import AutoShardedBot, Context, command
 
@@ -36,9 +34,8 @@ class Tags:
         if tag_name:
             params['tag_name'] = tag_name
 
-        async with ClientSession() as session:
-            response = await session.get(SITE_API_TAGS_URL, headers=self.headers, params=params)
-            tag_data = await response.json()
+        response = await self.bot.http_session.get(SITE_API_TAGS_URL, headers=self.headers, params=params)
+        tag_data = await response.json()
 
         return tag_data
 
@@ -59,9 +56,8 @@ class Tags:
             'tag_content': tag_content
         }
 
-        async with ClientSession() as session:
-            response = await session.post(SITE_API_TAGS_URL, headers=self.headers, json=params)
-            tag_data = await response.json()
+        response = await self.bot.http_session.post(SITE_API_TAGS_URL, headers=self.headers, json=params)
+        tag_data = await response.json()
 
         return tag_data
 
@@ -81,9 +77,8 @@ class Tags:
         if tag_name:
             params['tag_name'] = tag_name
 
-        async with ClientSession() as session:
-            response = await session.delete(SITE_API_TAGS_URL, headers=self.headers, json=params)
-            tag_data = await response.json()
+        response = await self.bot.http_session.delete(SITE_API_TAGS_URL, headers=self.headers, json=params)
+        tag_data = await response.json()
 
         return tag_data
 
diff --git a/requirements.txt b/requirements.txt
index 941d7f2b6..e0159eded 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,4 @@ https://github.com/Rapptz/discord.py/archive/rewrite.zip#egg=discord.py[voice]
 dulwich
 multidict
 sympy
+aiodns