diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Dockerfile | 31 | ||||
| -rw-r--r-- | Pipfile | 38 | ||||
| -rw-r--r-- | Pipfile.lock | 2 | ||||
| -rw-r--r-- | README.md | 10 | ||||
| -rw-r--r-- | azure-pipelines.yml | 122 | ||||
| -rw-r--r-- | bot/cogs/cogs.py | 19 | ||||
| -rw-r--r-- | bot/cogs/filtering.py | 48 | ||||
| -rw-r--r-- | bot/cogs/reminders.py | 4 | ||||
| -rw-r--r-- | docker-compose.yml | 30 | ||||
| -rw-r--r-- | docker/ci.Dockerfile | 20 | ||||
| -rw-r--r-- | scripts/deploy-azure.sh | 12 |
12 files changed, 166 insertions, 171 deletions
diff --git a/.gitignore b/.gitignore index cda3aeb9f..261fa179f 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,6 @@ lib64/ parts/ sdist/ var/ -wheels/ *.egg-info/ .installed.cfg *.egg diff --git a/Dockerfile b/Dockerfile index aa6333380..271c25050 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,20 @@ -FROM python:3.7-alpine3.7 +FROM python:3.7-slim -RUN apk add --no-cache \ - build-base \ - freetype-dev \ - git \ - jpeg-dev \ - libffi-dev \ - libxml2 \ - libxml2-dev \ - libxslt-dev \ - tini \ - zlib \ - zlib-dev - -ENV \ - LIBRARY_PATH=/lib:/usr/lib +# Set pip to have cleaner logs and no saved cache +ENV PIP_NO_CACHE_DIR=false \ + PIPENV_HIDE_EMOJIS=1 \ + PIPENV_IGNORE_VIRTUALENVS=1 \ + PIPENV_NOSPIN=1 +# Install pipenv RUN pip install -U pipenv +# Copy project files into working directory WORKDIR /bot COPY . . -RUN pipenv install --deploy --system +# Install project dependencies +RUN pipenv install --system --deploy -ENTRYPOINT ["/sbin/tini", "--"] -CMD ["python3", "-m", "bot"] +ENTRYPOINT ["python3"] +CMD ["-m", "bot"] @@ -5,18 +5,18 @@ name = "pypi" [packages] discord-py = "~=1.2" -aiodns = "*" -logmatic-python = "*" -aiohttp = "*" -sphinx = "*" -markdownify = "*" -lxml = "*" -pyyaml = "*" -fuzzywuzzy = "*" -aio-pika = "*" -python-dateutil = "*" -deepdiff = "*" -requests = "*" +aiodns = "~=2.0" +logmatic-python = "~=0.1" +aiohttp = "~=3.5" +sphinx = "~=2.2" +markdownify = "~=0.4" +lxml = "~=4.4" +pyyaml = "~=5.1" +fuzzywuzzy = "~=0.17" +aio-pika = "~=6.1" +python-dateutil = "~=2.8" +deepdiff = "~=4.0" +requests = "~=2.22" more_itertools = "~=7.2" urllib3 = ">=1.24.2,<1.25" @@ -30,10 +30,10 @@ flake8-string-format = "~=0.2" flake8-tidy-imports = "~=2.0" flake8-todo = "~=0.7" pre-commit = "~=1.18" -safety = "*" -dodgy = "*" -pytest = "*" -pytest-cov = "*" +safety = "~=1.8" +dodgy = "~=0.1" +pytest = "~=5.1" +pytest-cov = "~=2.7" [requires] python_version = "3.7" @@ -42,9 +42,5 @@ python_version = "3.7" start = "python -m bot" lint = "python -m flake8" precommit = "pre-commit install" -build = "docker build -t pythondiscord/bot:latest -f docker/bot.Dockerfile ." +build = "docker build -t pythondiscord/bot:latest -f Dockerfile ." push = "docker push pythondiscord/bot:latest" -buildbase = "docker build -t pythondiscord/bot-base:latest -f docker/base.Dockerfile ." -pushbase = "docker push pythondiscord/bot-base:latest" -buildci = "docker build -t pythondiscord/bot-ci:latest -f docker/ci.Dockerfile ." -pushci = "docker push pythondiscord/bot-ci:latest" diff --git a/Pipfile.lock b/Pipfile.lock index 7674acb26..58489c60e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "d582b1e226b1ce675817161d9059352d8f303c1bc1646034a9e73673f6581d12" + "sha256": "6c2d9ea980f1dbe954237de6d173ffa9ba480aa5cf0a03c4d7840b0739d4e2fa" }, "pipfile-spec": 6, "requires": { @@ -1,7 +1,11 @@ -# Python Utility Bot +# Python Utility Bot -[)](https://dev.azure.com/python-discord/Python%20Discord/_build/latest?definitionId=1) -[](https://discord.gg/2B963hn) +[](https://discord.gg/2B963hn) +[](https://dev.azure.com/python-discord/Python%20Discord/_build/latest?definitionId=1&branchName=master) +[](https://dev.azure.com/python-discord/Python%20Discord/_apis/build/status/Bot?branchName=master) +[](https://dev.azure.com/python-discord/Python%20Discord/_apis/build/status/Bot?branchName=master) +[](LICENSE) +[](https://pythondiscord.com) This project is a Discord bot specifically for use with the Python Discord server. It provides numerous utilities and other tools to help keep the server running like a well-oiled machine. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4dcad685c..b5ecab83c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -6,69 +6,59 @@ variables: PIPENV_NOSPIN: 1 jobs: -- job: test - displayName: 'Lint & Test' - - pool: - vmImage: ubuntu-16.04 - - variables: - PIPENV_CACHE_DIR: ".cache/pipenv" - PIP_CACHE_DIR: ".cache/pip" - PIP_SRC: ".cache/src" - - steps: - - script: | - sudo apt-get update - sudo apt-get install build-essential curl docker libffi-dev libfreetype6-dev libxml2 libxml2-dev libxslt1-dev zlib1g zlib1g-dev - displayName: 'Install base dependencies' - - - task: UsePythonVersion@0 - displayName: 'Set Python version' - inputs: - versionSpec: '3.7.x' - addToPath: true - - - script: sudo pip install pipenv - displayName: 'Install pipenv' - - - script: pipenv install --dev --deploy --system - displayName: 'Install project using pipenv' - - - script: python -m flake8 - displayName: 'Run linter' - - - script: BOT_API_KEY=foo BOT_TOKEN=bar WOLFRAM_API_KEY=baz python -m pytest --junitxml=junit.xml --cov=bot --cov-branch --cov-report=term --cov-report=xml tests - displayName: Run tests - - - task: PublishCodeCoverageResults@1 - displayName: 'Publish Coverage Results' - condition: succeededOrFailed() - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: coverage.xml - - - task: PublishTestResults@2 - displayName: 'Publish Test Results' - condition: succeededOrFailed() - inputs: - testResultsFiles: junit.xml - testRunTitle: 'Bot Test results' - -- job: build - displayName: 'Build Containers' - dependsOn: 'test' - - steps: - - task: Docker@1 - displayName: 'Login: Docker Hub' - - inputs: - containerregistrytype: 'Container Registry' - dockerRegistryEndpoint: 'DockerHub' - command: 'login' - - - task: ShellScript@2 - displayName: 'Build and deploy containers' - inputs: - scriptPath: scripts/deploy-azure.sh + - job: test + displayName: 'Lint & Test' + pool: + vmImage: ubuntu-16.04 + + variables: + PIP_CACHE_DIR: ".cache/pip" + + steps: + - task: UsePythonVersion@0 + displayName: 'Set Python version' + inputs: + versionSpec: '3.7.x' + addToPath: true + + - script: pip install pipenv + displayName: 'Install pipenv' + + - script: pipenv install --dev --deploy --system + displayName: 'Install project using pipenv' + + - script: python -m flake8 + displayName: 'Run linter' + + - script: BOT_API_KEY=foo BOT_TOKEN=bar WOLFRAM_API_KEY=baz python -m pytest --junitxml=junit.xml --cov=bot --cov-branch --cov-report=term --cov-report=xml tests + displayName: Run tests + + - task: PublishCodeCoverageResults@1 + displayName: 'Publish Coverage Results' + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: coverage.xml + + - task: PublishTestResults@2 + displayName: 'Publish Test Results' + condition: succeededOrFailed() + inputs: + testResultsFiles: junit.xml + testRunTitle: 'Bot Test results' + + - job: build + displayName: 'Build & Push Container' + dependsOn: 'test' + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + + steps: + - task: Docker@2 + displayName: 'Build & Push Container' + inputs: + containerRegistry: 'DockerHub' + repository: 'pythondiscord/bot' + command: 'buildAndPush' + Dockerfile: 'Dockerfile' + buildContext: '.' + tags: 'latest' diff --git a/bot/cogs/cogs.py b/bot/cogs/cogs.py index 117c77d4b..1f6ccd09c 100644 --- a/bot/cogs/cogs.py +++ b/bot/cogs/cogs.py @@ -74,13 +74,12 @@ class Cogs(Cog): try: self.bot.load_extension(full_cog) except ImportError: - log.error(f"{ctx.author} requested we load the '{cog}' cog, " - f"but the cog module {full_cog} could not be found!") + log.exception(f"{ctx.author} requested we load the '{cog}' cog, " + f"but the cog module {full_cog} could not be found!") embed.description = f"Invalid cog: {cog}\n\nCould not find cog module {full_cog}" except Exception as e: - log.error(f"{ctx.author} requested we load the '{cog}' cog, " - "but the loading failed with the following error: \n" - f"**{e.__class__.__name__}: {e}**") + log.exception(f"{ctx.author} requested we load the '{cog}' cog, " + "but the loading failed") embed.description = f"Failed to load cog: {cog}\n\n{e.__class__.__name__}: {e}" else: log.debug(f"{ctx.author} requested we load the '{cog}' cog. Cog loaded!") @@ -129,9 +128,8 @@ class Cogs(Cog): try: self.bot.unload_extension(full_cog) except Exception as e: - log.error(f"{ctx.author} requested we unload the '{cog}' cog, " - "but the unloading failed with the following error: \n" - f"{e}") + log.exception(f"{ctx.author} requested we unload the '{cog}' cog, " + "but the unloading failed") embed.description = f"Failed to unload cog: {cog}\n\n```{e}```" else: log.debug(f"{ctx.author} requested we unload the '{cog}' cog. Cog unloaded!") @@ -234,9 +232,8 @@ class Cogs(Cog): self.bot.unload_extension(full_cog) self.bot.load_extension(full_cog) except Exception as e: - log.error(f"{ctx.author} requested we reload the '{cog}' cog, " - "but the unloading failed with the following error: \n" - f"{e}") + log.exception(f"{ctx.author} requested we reload the '{cog}' cog, " + "but the unloading failed") embed.description = f"Failed to reload cog: {cog}\n\n```{e}```" else: log.debug(f"{ctx.author} requested we reload the '{cog}' cog. Cog reloaded!") diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 9cd1b7203..bd8c6ed67 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -15,18 +15,26 @@ from bot.constants import ( log = logging.getLogger(__name__) -INVITE_RE = ( +INVITE_RE = re.compile( r"(?:discord(?:[\.,]|dot)gg|" # Could be discord.gg/ r"discord(?:[\.,]|dot)com(?:\/|slash)invite|" # or discord.com/invite/ r"discordapp(?:[\.,]|dot)com(?:\/|slash)invite|" # or discordapp.com/invite/ r"discord(?:[\.,]|dot)me|" # or discord.me r"discord(?:[\.,]|dot)io" # or discord.io. r")(?:[\/]|slash)" # / or 'slash' - r"([a-zA-Z0-9]+)" # the invite code itself + r"([a-zA-Z0-9]+)", # the invite code itself + flags=re.IGNORECASE ) -URL_RE = r"(https?://[^\s]+)" -ZALGO_RE = r"[\u0300-\u036F\u0489]" +URL_RE = re.compile(r"(https?://[^\s]+)", flags=re.IGNORECASE) +ZALGO_RE = re.compile(r"[\u0300-\u036F\u0489]") + +WORD_WATCHLIST_PATTERNS = [ + re.compile(fr'\b{expression}\b', flags=re.IGNORECASE) for expression in Filter.word_watchlist +] +TOKEN_WATCHLIST_PATTERNS = [ + re.compile(fr'{expression}', flags=re.IGNORECASE) for expression in Filter.token_watchlist +] class Filtering(Cog): @@ -228,8 +236,8 @@ class Filtering(Cog): Only matches words with boundaries before and after the expression. """ - for expression in Filter.word_watchlist: - if re.search(fr"\b{expression}\b", text, re.IGNORECASE): + for regex_pattern in WORD_WATCHLIST_PATTERNS: + if regex_pattern.search(text): return True return False @@ -241,11 +249,11 @@ class Filtering(Cog): This will match the expression even if it does not have boundaries before and after. """ - for expression in Filter.token_watchlist: - if re.search(fr"{expression}", text, re.IGNORECASE): + for regex_pattern in TOKEN_WATCHLIST_PATTERNS: + if regex_pattern.search(text): # Make sure it's not a URL - if not re.search(URL_RE, text, re.IGNORECASE): + if not URL_RE.search(text): return True return False @@ -253,7 +261,7 @@ class Filtering(Cog): @staticmethod async def _has_urls(text: str) -> bool: """Returns True if the text contains one of the blacklisted URLs from the config file.""" - if not re.search(URL_RE, text, re.IGNORECASE): + if not URL_RE.search(text): return False text = text.lower() @@ -271,7 +279,7 @@ class Filtering(Cog): Zalgo range is \u0300 – \u036F and \u0489. """ - return bool(re.search(ZALGO_RE, text)) + return bool(ZALGO_RE.search(text)) async def _has_invites(self, text: str) -> Union[dict, bool]: """ @@ -286,7 +294,7 @@ class Filtering(Cog): # discord\.gg/gdudes-pony-farm text = text.replace("\\", "") - invites = re.findall(INVITE_RE, text, re.IGNORECASE) + invites = INVITE_RE.findall(text) invite_data = dict() for invite in invites: if invite in invite_data: @@ -323,11 +331,21 @@ class Filtering(Cog): @staticmethod async def _has_rich_embed(msg: Message) -> bool: - """Returns True if any of the embeds in the message are of type 'rich', but are not twitter embeds.""" + """Determines if `msg` contains any rich embeds not auto-generated from a URL.""" if msg.embeds: for embed in msg.embeds: - if embed.type == "rich" and (not embed.url or "twitter.com" not in embed.url): - return True + if embed.type == "rich": + urls = URL_RE.findall(msg.content) + if not embed.url or embed.url not in urls: + # If `embed.url` does not exist or if `embed.url` is not part of the content + # of the message, it's unlikely to be an auto-generated embed by Discord. + return True + else: + log.trace( + "Found a rich embed sent by a regular user account, " + "but it was likely just an automatic URL embed." + ) + return False return False async def notify_member(self, filtered_member: Member, reason: str, channel: TextChannel) -> None: diff --git a/bot/cogs/reminders.py b/bot/cogs/reminders.py index c37abf21e..6e91d2c06 100644 --- a/bot/cogs/reminders.py +++ b/bot/cogs/reminders.py @@ -146,7 +146,7 @@ class Reminders(Scheduler, Cog): active_reminders = await self.bot.api_client.get( 'bot/reminders', params={ - 'user__id': str(ctx.author.id) + 'author__id': str(ctx.author.id) } ) @@ -184,7 +184,7 @@ class Reminders(Scheduler, Cog): # Get all the user's reminders from the database. data = await self.bot.api_client.get( 'bot/reminders', - params={'user__id': str(ctx.author.id)} + params={'author__id': str(ctx.author.id)} ) now = datetime.utcnow() diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..4b0dcff35 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +# This docker compose is used for quick setups of the site and database which +# the bot project relies on for testing. Use it if you haven't got a +# ready-to-use site environment already setup. + +version: "3.7" + +services: + postgres: + image: postgres:11-alpine + ports: + - "127.0.0.1:7777:5432" + environment: + POSTGRES_DB: pysite + POSTGRES_PASSWORD: pysite + POSTGRES_USER: pysite + + web: + image: pythondiscord/site:latest + command: > + bash -c "echo \"from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser('admin', 'admin', 'admin') if not User.objects.filter(username='admin').exists() else print('Admin user already exists')\" | python manage.py shell + && ./manage.py runserver 0.0.0.0:8000" + ports: + - "127.0.0.1:8000:8000" + depends_on: + - postgres + environment: + DATABASE_URL: postgres://pysite:pysite@postgres:5432/pysite + DEBUG: "true" + SECRET_KEY: suitable-for-development-only + STATIC_ROOT: /var/www/static diff --git a/docker/ci.Dockerfile b/docker/ci.Dockerfile deleted file mode 100644 index fd7e25239..000000000 --- a/docker/ci.Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM python:3.6-alpine3.7 - -RUN apk add --update docker \ - curl \ - tini \ - build-base \ - libffi-dev \ - zlib \ - jpeg-dev \ - libxml2 libxml2-dev libxslt-dev \ - zlib-dev \ - freetype-dev - -RUN pip install pipenv - -ENV LIBRARY_PATH=/lib:/usr/lib -ENV PIPENV_VENV_IN_PROJECT=1 -ENV PIPENV_IGNORE_VIRTUALENVS=1 -ENV PIPENV_NOSPIN=1 -ENV PIPENV_HIDE_EMOJIS=1 diff --git a/scripts/deploy-azure.sh b/scripts/deploy-azure.sh deleted file mode 100644 index ed4b719e2..000000000 --- a/scripts/deploy-azure.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -cd .. - -# Build and deploy on master branch, only if not a pull request -if [[ ($BUILD_SOURCEBRANCHNAME == 'master') && ($SYSTEM_PULLREQUEST_PULLREQUESTID == '') ]]; then - echo "Building image" - docker build -t pythondiscord/bot:latest . - - echo "Pushing image" - docker push pythondiscord/bot:latest -fi |