aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorGravatar S. Co1 <[email protected]>2019-09-24 15:14:07 -0400
committerGravatar GitHub <[email protected]>2019-09-24 15:14:07 -0400
commit54d4f7105833860723fe5a7aea7e38e7418e82e0 (patch)
treeca4fbdca201e915791adcadb4648f98494754048
parentRemove repeat logic for off-topic-name api call (diff)
parentMerge pull request #451 from python-discord/all-the-shields (diff)
Merge branch 'master' into ot-fix
-rw-r--r--.gitignore1
-rw-r--r--Dockerfile31
-rw-r--r--Pipfile38
-rw-r--r--Pipfile.lock2
-rw-r--r--README.md10
-rw-r--r--azure-pipelines.yml122
-rw-r--r--bot/cogs/cogs.py19
-rw-r--r--bot/cogs/filtering.py48
-rw-r--r--bot/cogs/reminders.py4
-rw-r--r--docker-compose.yml30
-rw-r--r--docker/ci.Dockerfile20
-rw-r--r--scripts/deploy-azure.sh12
12 files changed, 166 insertions, 171 deletions
diff --git a/.gitignore b/.gitignore
index cda3aeb9f..261fa179f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,7 +20,6 @@ lib64/
parts/
sdist/
var/
-wheels/
*.egg-info/
.installed.cfg
*.egg
diff --git a/Dockerfile b/Dockerfile
index aa6333380..271c25050 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,27 +1,20 @@
-FROM python:3.7-alpine3.7
+FROM python:3.7-slim
-RUN apk add --no-cache \
- build-base \
- freetype-dev \
- git \
- jpeg-dev \
- libffi-dev \
- libxml2 \
- libxml2-dev \
- libxslt-dev \
- tini \
- zlib \
- zlib-dev
-
-ENV \
- LIBRARY_PATH=/lib:/usr/lib
+# Set pip to have cleaner logs and no saved cache
+ENV PIP_NO_CACHE_DIR=false \
+ PIPENV_HIDE_EMOJIS=1 \
+ PIPENV_IGNORE_VIRTUALENVS=1 \
+ PIPENV_NOSPIN=1
+# Install pipenv
RUN pip install -U pipenv
+# Copy project files into working directory
WORKDIR /bot
COPY . .
-RUN pipenv install --deploy --system
+# Install project dependencies
+RUN pipenv install --system --deploy
-ENTRYPOINT ["/sbin/tini", "--"]
-CMD ["python3", "-m", "bot"]
+ENTRYPOINT ["python3"]
+CMD ["-m", "bot"]
diff --git a/Pipfile b/Pipfile
index 33f44e9a6..da46a536d 100644
--- a/Pipfile
+++ b/Pipfile
@@ -5,18 +5,18 @@ name = "pypi"
[packages]
discord-py = "~=1.2"
-aiodns = "*"
-logmatic-python = "*"
-aiohttp = "*"
-sphinx = "*"
-markdownify = "*"
-lxml = "*"
-pyyaml = "*"
-fuzzywuzzy = "*"
-aio-pika = "*"
-python-dateutil = "*"
-deepdiff = "*"
-requests = "*"
+aiodns = "~=2.0"
+logmatic-python = "~=0.1"
+aiohttp = "~=3.5"
+sphinx = "~=2.2"
+markdownify = "~=0.4"
+lxml = "~=4.4"
+pyyaml = "~=5.1"
+fuzzywuzzy = "~=0.17"
+aio-pika = "~=6.1"
+python-dateutil = "~=2.8"
+deepdiff = "~=4.0"
+requests = "~=2.22"
more_itertools = "~=7.2"
urllib3 = ">=1.24.2,<1.25"
@@ -30,10 +30,10 @@ flake8-string-format = "~=0.2"
flake8-tidy-imports = "~=2.0"
flake8-todo = "~=0.7"
pre-commit = "~=1.18"
-safety = "*"
-dodgy = "*"
-pytest = "*"
-pytest-cov = "*"
+safety = "~=1.8"
+dodgy = "~=0.1"
+pytest = "~=5.1"
+pytest-cov = "~=2.7"
[requires]
python_version = "3.7"
@@ -42,9 +42,5 @@ python_version = "3.7"
start = "python -m bot"
lint = "python -m flake8"
precommit = "pre-commit install"
-build = "docker build -t pythondiscord/bot:latest -f docker/bot.Dockerfile ."
+build = "docker build -t pythondiscord/bot:latest -f Dockerfile ."
push = "docker push pythondiscord/bot:latest"
-buildbase = "docker build -t pythondiscord/bot-base:latest -f docker/base.Dockerfile ."
-pushbase = "docker push pythondiscord/bot-base:latest"
-buildci = "docker build -t pythondiscord/bot-ci:latest -f docker/ci.Dockerfile ."
-pushci = "docker push pythondiscord/bot-ci:latest"
diff --git a/Pipfile.lock b/Pipfile.lock
index 7674acb26..58489c60e 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "d582b1e226b1ce675817161d9059352d8f303c1bc1646034a9e73673f6581d12"
+ "sha256": "6c2d9ea980f1dbe954237de6d173ffa9ba480aa5cf0a03c4d7840b0739d4e2fa"
},
"pipfile-spec": 6,
"requires": {
diff --git a/README.md b/README.md
index 1c9e52b71..ae3288dbf 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,11 @@
-# Python Utility Bot
+# Python Utility Bot
-[![Build Status](https://dev.azure.com/python-discord/Python%20Discord/_apis/build/status/Bot%20(Mainline))](https://dev.azure.com/python-discord/Python%20Discord/_build/latest?definitionId=1)
-[![Discord](https://discordapp.com/api/guilds/267624335836053506/embed.png)](https://discord.gg/2B963hn)
+[![Discord](https://img.shields.io/discord/267624335836053506?color=%237289DA&label=Python%20Discord&logo=discord&logoColor=white)](https://discord.gg/2B963hn)
+[![Build Status](https://dev.azure.com/python-discord/Python%20Discord/_apis/build/status/Bot?branchName=master)](https://dev.azure.com/python-discord/Python%20Discord/_build/latest?definitionId=1&branchName=master)
+[![Tests](https://img.shields.io/azure-devops/tests/python-discord/Python%20Discord/1?compact_message)](https://dev.azure.com/python-discord/Python%20Discord/_apis/build/status/Bot?branchName=master)
+[![Coverage](https://img.shields.io/azure-devops/coverage/python-discord/Python%20Discord/1/master)](https://dev.azure.com/python-discord/Python%20Discord/_apis/build/status/Bot?branchName=master)
+[![License](https://img.shields.io/github/license/python-discord/bot)](LICENSE)
+[![Website](https://img.shields.io/badge/website-visit-brightgreen)](https://pythondiscord.com)
This project is a Discord bot specifically for use with the Python Discord server. It provides numerous utilities
and other tools to help keep the server running like a well-oiled machine.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 4dcad685c..b5ecab83c 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -6,69 +6,59 @@ variables:
PIPENV_NOSPIN: 1
jobs:
-- job: test
- displayName: 'Lint & Test'
-
- pool:
- vmImage: ubuntu-16.04
-
- variables:
- PIPENV_CACHE_DIR: ".cache/pipenv"
- PIP_CACHE_DIR: ".cache/pip"
- PIP_SRC: ".cache/src"
-
- steps:
- - script: |
- sudo apt-get update
- sudo apt-get install build-essential curl docker libffi-dev libfreetype6-dev libxml2 libxml2-dev libxslt1-dev zlib1g zlib1g-dev
- displayName: 'Install base dependencies'
-
- - task: UsePythonVersion@0
- displayName: 'Set Python version'
- inputs:
- versionSpec: '3.7.x'
- addToPath: true
-
- - script: sudo pip install pipenv
- displayName: 'Install pipenv'
-
- - script: pipenv install --dev --deploy --system
- displayName: 'Install project using pipenv'
-
- - script: python -m flake8
- displayName: 'Run linter'
-
- - script: BOT_API_KEY=foo BOT_TOKEN=bar WOLFRAM_API_KEY=baz python -m pytest --junitxml=junit.xml --cov=bot --cov-branch --cov-report=term --cov-report=xml tests
- displayName: Run tests
-
- - task: PublishCodeCoverageResults@1
- displayName: 'Publish Coverage Results'
- condition: succeededOrFailed()
- inputs:
- codeCoverageTool: Cobertura
- summaryFileLocation: coverage.xml
-
- - task: PublishTestResults@2
- displayName: 'Publish Test Results'
- condition: succeededOrFailed()
- inputs:
- testResultsFiles: junit.xml
- testRunTitle: 'Bot Test results'
-
-- job: build
- displayName: 'Build Containers'
- dependsOn: 'test'
-
- steps:
- - task: Docker@1
- displayName: 'Login: Docker Hub'
-
- inputs:
- containerregistrytype: 'Container Registry'
- dockerRegistryEndpoint: 'DockerHub'
- command: 'login'
-
- - task: ShellScript@2
- displayName: 'Build and deploy containers'
- inputs:
- scriptPath: scripts/deploy-azure.sh
+ - job: test
+ displayName: 'Lint & Test'
+ pool:
+ vmImage: ubuntu-16.04
+
+ variables:
+ PIP_CACHE_DIR: ".cache/pip"
+
+ steps:
+ - task: UsePythonVersion@0
+ displayName: 'Set Python version'
+ inputs:
+ versionSpec: '3.7.x'
+ addToPath: true
+
+ - script: pip install pipenv
+ displayName: 'Install pipenv'
+
+ - script: pipenv install --dev --deploy --system
+ displayName: 'Install project using pipenv'
+
+ - script: python -m flake8
+ displayName: 'Run linter'
+
+ - script: BOT_API_KEY=foo BOT_TOKEN=bar WOLFRAM_API_KEY=baz python -m pytest --junitxml=junit.xml --cov=bot --cov-branch --cov-report=term --cov-report=xml tests
+ displayName: Run tests
+
+ - task: PublishCodeCoverageResults@1
+ displayName: 'Publish Coverage Results'
+ condition: succeededOrFailed()
+ inputs:
+ codeCoverageTool: Cobertura
+ summaryFileLocation: coverage.xml
+
+ - task: PublishTestResults@2
+ displayName: 'Publish Test Results'
+ condition: succeededOrFailed()
+ inputs:
+ testResultsFiles: junit.xml
+ testRunTitle: 'Bot Test results'
+
+ - job: build
+ displayName: 'Build & Push Container'
+ dependsOn: 'test'
+ condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+
+ steps:
+ - task: Docker@2
+ displayName: 'Build & Push Container'
+ inputs:
+ containerRegistry: 'DockerHub'
+ repository: 'pythondiscord/bot'
+ command: 'buildAndPush'
+ Dockerfile: 'Dockerfile'
+ buildContext: '.'
+ tags: 'latest'
diff --git a/bot/cogs/cogs.py b/bot/cogs/cogs.py
index 117c77d4b..1f6ccd09c 100644
--- a/bot/cogs/cogs.py
+++ b/bot/cogs/cogs.py
@@ -74,13 +74,12 @@ class Cogs(Cog):
try:
self.bot.load_extension(full_cog)
except ImportError:
- log.error(f"{ctx.author} requested we load the '{cog}' cog, "
- f"but the cog module {full_cog} could not be found!")
+ log.exception(f"{ctx.author} requested we load the '{cog}' cog, "
+ f"but the cog module {full_cog} could not be found!")
embed.description = f"Invalid cog: {cog}\n\nCould not find cog module {full_cog}"
except Exception as e:
- log.error(f"{ctx.author} requested we load the '{cog}' cog, "
- "but the loading failed with the following error: \n"
- f"**{e.__class__.__name__}: {e}**")
+ log.exception(f"{ctx.author} requested we load the '{cog}' cog, "
+ "but the loading failed")
embed.description = f"Failed to load cog: {cog}\n\n{e.__class__.__name__}: {e}"
else:
log.debug(f"{ctx.author} requested we load the '{cog}' cog. Cog loaded!")
@@ -129,9 +128,8 @@ class Cogs(Cog):
try:
self.bot.unload_extension(full_cog)
except Exception as e:
- log.error(f"{ctx.author} requested we unload the '{cog}' cog, "
- "but the unloading failed with the following error: \n"
- f"{e}")
+ log.exception(f"{ctx.author} requested we unload the '{cog}' cog, "
+ "but the unloading failed")
embed.description = f"Failed to unload cog: {cog}\n\n```{e}```"
else:
log.debug(f"{ctx.author} requested we unload the '{cog}' cog. Cog unloaded!")
@@ -234,9 +232,8 @@ class Cogs(Cog):
self.bot.unload_extension(full_cog)
self.bot.load_extension(full_cog)
except Exception as e:
- log.error(f"{ctx.author} requested we reload the '{cog}' cog, "
- "but the unloading failed with the following error: \n"
- f"{e}")
+ log.exception(f"{ctx.author} requested we reload the '{cog}' cog, "
+ "but the unloading failed")
embed.description = f"Failed to reload cog: {cog}\n\n```{e}```"
else:
log.debug(f"{ctx.author} requested we reload the '{cog}' cog. Cog reloaded!")
diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py
index 9cd1b7203..bd8c6ed67 100644
--- a/bot/cogs/filtering.py
+++ b/bot/cogs/filtering.py
@@ -15,18 +15,26 @@ from bot.constants import (
log = logging.getLogger(__name__)
-INVITE_RE = (
+INVITE_RE = re.compile(
r"(?:discord(?:[\.,]|dot)gg|" # Could be discord.gg/
r"discord(?:[\.,]|dot)com(?:\/|slash)invite|" # or discord.com/invite/
r"discordapp(?:[\.,]|dot)com(?:\/|slash)invite|" # or discordapp.com/invite/
r"discord(?:[\.,]|dot)me|" # or discord.me
r"discord(?:[\.,]|dot)io" # or discord.io.
r")(?:[\/]|slash)" # / or 'slash'
- r"([a-zA-Z0-9]+)" # the invite code itself
+ r"([a-zA-Z0-9]+)", # the invite code itself
+ flags=re.IGNORECASE
)
-URL_RE = r"(https?://[^\s]+)"
-ZALGO_RE = r"[\u0300-\u036F\u0489]"
+URL_RE = re.compile(r"(https?://[^\s]+)", flags=re.IGNORECASE)
+ZALGO_RE = re.compile(r"[\u0300-\u036F\u0489]")
+
+WORD_WATCHLIST_PATTERNS = [
+ re.compile(fr'\b{expression}\b', flags=re.IGNORECASE) for expression in Filter.word_watchlist
+]
+TOKEN_WATCHLIST_PATTERNS = [
+ re.compile(fr'{expression}', flags=re.IGNORECASE) for expression in Filter.token_watchlist
+]
class Filtering(Cog):
@@ -228,8 +236,8 @@ class Filtering(Cog):
Only matches words with boundaries before and after the expression.
"""
- for expression in Filter.word_watchlist:
- if re.search(fr"\b{expression}\b", text, re.IGNORECASE):
+ for regex_pattern in WORD_WATCHLIST_PATTERNS:
+ if regex_pattern.search(text):
return True
return False
@@ -241,11 +249,11 @@ class Filtering(Cog):
This will match the expression even if it does not have boundaries before and after.
"""
- for expression in Filter.token_watchlist:
- if re.search(fr"{expression}", text, re.IGNORECASE):
+ for regex_pattern in TOKEN_WATCHLIST_PATTERNS:
+ if regex_pattern.search(text):
# Make sure it's not a URL
- if not re.search(URL_RE, text, re.IGNORECASE):
+ if not URL_RE.search(text):
return True
return False
@@ -253,7 +261,7 @@ class Filtering(Cog):
@staticmethod
async def _has_urls(text: str) -> bool:
"""Returns True if the text contains one of the blacklisted URLs from the config file."""
- if not re.search(URL_RE, text, re.IGNORECASE):
+ if not URL_RE.search(text):
return False
text = text.lower()
@@ -271,7 +279,7 @@ class Filtering(Cog):
Zalgo range is \u0300 – \u036F and \u0489.
"""
- return bool(re.search(ZALGO_RE, text))
+ return bool(ZALGO_RE.search(text))
async def _has_invites(self, text: str) -> Union[dict, bool]:
"""
@@ -286,7 +294,7 @@ class Filtering(Cog):
# discord\.gg/gdudes-pony-farm
text = text.replace("\\", "")
- invites = re.findall(INVITE_RE, text, re.IGNORECASE)
+ invites = INVITE_RE.findall(text)
invite_data = dict()
for invite in invites:
if invite in invite_data:
@@ -323,11 +331,21 @@ class Filtering(Cog):
@staticmethod
async def _has_rich_embed(msg: Message) -> bool:
- """Returns True if any of the embeds in the message are of type 'rich', but are not twitter embeds."""
+ """Determines if `msg` contains any rich embeds not auto-generated from a URL."""
if msg.embeds:
for embed in msg.embeds:
- if embed.type == "rich" and (not embed.url or "twitter.com" not in embed.url):
- return True
+ if embed.type == "rich":
+ urls = URL_RE.findall(msg.content)
+ if not embed.url or embed.url not in urls:
+ # If `embed.url` does not exist or if `embed.url` is not part of the content
+ # of the message, it's unlikely to be an auto-generated embed by Discord.
+ return True
+ else:
+ log.trace(
+ "Found a rich embed sent by a regular user account, "
+ "but it was likely just an automatic URL embed."
+ )
+ return False
return False
async def notify_member(self, filtered_member: Member, reason: str, channel: TextChannel) -> None:
diff --git a/bot/cogs/reminders.py b/bot/cogs/reminders.py
index c37abf21e..6e91d2c06 100644
--- a/bot/cogs/reminders.py
+++ b/bot/cogs/reminders.py
@@ -146,7 +146,7 @@ class Reminders(Scheduler, Cog):
active_reminders = await self.bot.api_client.get(
'bot/reminders',
params={
- 'user__id': str(ctx.author.id)
+ 'author__id': str(ctx.author.id)
}
)
@@ -184,7 +184,7 @@ class Reminders(Scheduler, Cog):
# Get all the user's reminders from the database.
data = await self.bot.api_client.get(
'bot/reminders',
- params={'user__id': str(ctx.author.id)}
+ params={'author__id': str(ctx.author.id)}
)
now = datetime.utcnow()
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 000000000..4b0dcff35
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,30 @@
+# This docker compose is used for quick setups of the site and database which
+# the bot project relies on for testing. Use it if you haven't got a
+# ready-to-use site environment already setup.
+
+version: "3.7"
+
+services:
+ postgres:
+ image: postgres:11-alpine
+ ports:
+ - "127.0.0.1:7777:5432"
+ environment:
+ POSTGRES_DB: pysite
+ POSTGRES_PASSWORD: pysite
+ POSTGRES_USER: pysite
+
+ web:
+ image: pythondiscord/site:latest
+ command: >
+ bash -c "echo \"from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser('admin', 'admin', 'admin') if not User.objects.filter(username='admin').exists() else print('Admin user already exists')\" | python manage.py shell
+ && ./manage.py runserver 0.0.0.0:8000"
+ ports:
+ - "127.0.0.1:8000:8000"
+ depends_on:
+ - postgres
+ environment:
+ DATABASE_URL: postgres://pysite:pysite@postgres:5432/pysite
+ DEBUG: "true"
+ SECRET_KEY: suitable-for-development-only
+ STATIC_ROOT: /var/www/static
diff --git a/docker/ci.Dockerfile b/docker/ci.Dockerfile
deleted file mode 100644
index fd7e25239..000000000
--- a/docker/ci.Dockerfile
+++ /dev/null
@@ -1,20 +0,0 @@
-FROM python:3.6-alpine3.7
-
-RUN apk add --update docker \
- curl \
- tini \
- build-base \
- libffi-dev \
- zlib \
- jpeg-dev \
- libxml2 libxml2-dev libxslt-dev \
- zlib-dev \
- freetype-dev
-
-RUN pip install pipenv
-
-ENV LIBRARY_PATH=/lib:/usr/lib
-ENV PIPENV_VENV_IN_PROJECT=1
-ENV PIPENV_IGNORE_VIRTUALENVS=1
-ENV PIPENV_NOSPIN=1
-ENV PIPENV_HIDE_EMOJIS=1
diff --git a/scripts/deploy-azure.sh b/scripts/deploy-azure.sh
deleted file mode 100644
index ed4b719e2..000000000
--- a/scripts/deploy-azure.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-cd ..
-
-# Build and deploy on master branch, only if not a pull request
-if [[ ($BUILD_SOURCEBRANCHNAME == 'master') && ($SYSTEM_PULLREQUEST_PULLREQUESTID == '') ]]; then
- echo "Building image"
- docker build -t pythondiscord/bot:latest .
-
- echo "Pushing image"
- docker push pythondiscord/bot:latest
-fi