From 3a9b8943d1d4203a7a6f41af5437d2c9dc90e828 Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff Date: Thu, 19 Nov 2020 00:37:14 +0100 Subject: Ensure that flake8 runs on PR changes Unfortunately, the way we previously set up our workflow caused flake8 to run on code already committed to master, not the changes made in a PR, because it ran in the context of the target branch. This is obviously useless when it comes to protecting our codebase from linting errors. I've now set up flake8 in a different way, using Workflow Commands to create error annotions. I've also split up the workflow into two separate workflows. --- .github/workflows/build-deploy.yaml | 70 +++++++++++++ .github/workflows/lint-test-deploy.yaml | 172 -------------------------------- .github/workflows/lint-test.yaml | 118 ++++++++++++++++++++++ 3 files changed, 188 insertions(+), 172 deletions(-) create mode 100644 .github/workflows/build-deploy.yaml delete mode 100644 .github/workflows/lint-test-deploy.yaml create mode 100644 .github/workflows/lint-test.yaml diff --git a/.github/workflows/build-deploy.yaml b/.github/workflows/build-deploy.yaml new file mode 100644 index 00000000..668927e0 --- /dev/null +++ b/.github/workflows/build-deploy.yaml @@ -0,0 +1,70 @@ +name: Build & Deploy + +on: + workflow_run: + workflows: ["Lint & Test"] + branches: + - master + types: + - completed + + build-and-deploy: + name: Build and Deploy to Kubernetes + needs: lint-test + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + + steps: + # Create a commit SHA-based tag for the container repositories + - name: Create SHA Container Tag + id: sha_tag + run: | + tag=$(cut -c 1-7 <<< $GITHUB_SHA) + echo "::set-output name=tag::$tag" + + - name: Checkout code + uses: actions/checkout@v2 + + # The current version (v2) of Docker's build-push action uses + # buildx, which comes with BuildKit features that help us speed + # up our builds using additional cache features. Buildx also + # has a lot of other features that are not as relevant to us. + # + # See https://github.com/docker/build-push-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Github Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USER }} + password: ${{ secrets.GHCR_TOKEN }} + + # Build the container, including an inline cache manifest to + # allow us to use the registry as a cache source. + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + push: true + cache-from: type=registry,ref=ghcr.io/python-discord/site:latest + cache-to: type=inline + tags: | + ghcr.io/python-discord/site:latest + ghcr.io/python-discord/site:${{ steps.sha_tag.outputs.tag }} + + - name: Authenticate with Kubernetes + uses: azure/k8s-set-context@v1 + with: + method: kubeconfig + kubeconfig: ${{ secrets.KUBECONFIG }} + + - name: Deploy to Kubernetes + uses: Azure/k8s-deploy@v1 + with: + manifests: | + deployment.yaml + images: 'ghcr.io/python-discord/site:${{ steps.sha_tag.outputs.tag }}' + kubectl-version: 'latest' diff --git a/.github/workflows/lint-test-deploy.yaml b/.github/workflows/lint-test-deploy.yaml deleted file mode 100644 index 7369a3b8..00000000 --- a/.github/workflows/lint-test-deploy.yaml +++ /dev/null @@ -1,172 +0,0 @@ -name: Lint, Test & Deploy - -on: - push: - branches: - - master - # We use pull_request_target as we get PRs from - # forks, but need to be able to add annotations - # for our flake8 step. - pull_request_target: - - -jobs: - lint-test: - runs-on: ubuntu-latest - env: - # Configure pip to cache dependencies and do a user install - PIP_NO_CACHE_DIR: false - PIP_USER: 1 - - # Hide the graphical elements from pipenv's output - PIPENV_HIDE_EMOJIS: 1 - PIPENV_NOSPIN: 1 - - # Make sure pipenv does not try reuse an environment it's running in - PIPENV_IGNORE_VIRTUALENVS: 1 - - # Specify explicit paths for python dependencies and the pre-commit - # environment so we know which directories to cache - PYTHONUSERBASE: ${{ github.workspace }}/.cache/py-user-base - PRE_COMMIT_HOME: ${{ github.workspace }}/.cache/pre-commit-cache - - steps: - - name: Add custom PYTHONUSERBASE to PATH - run: echo '${{ env.PYTHONUSERBASE }}/bin/' >> $GITHUB_PATH - - # We don't want to persist credentials, as our GitHub Action - # may be run when a PR is made from a fork. - - name: Checkout repository - uses: actions/checkout@v2 - with: - persist-credentials: false - - - name: Setup python - id: python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - # This step caches our Python dependencies. To make sure we - # only restore a cache when the dependencies, the python version, - # the runner operating system, and the dependency location haven't - # changed, we create a cache key that is a composite of those states. - # - # Only when the context is exactly the same, we will restore the cache. - - name: Python Dependency Caching - uses: actions/cache@v2 - id: python_cache - with: - path: ${{ env.PYTHONUSERBASE }} - key: "python-0-${{ runner.os }}-${{ env.PYTHONUSERBASE }}-\ - ${{ steps.python.outputs.python-version }}-\ - ${{ hashFiles('./Pipfile', './Pipfile.lock') }}" - - # Install our dependencies if we did not restore a dependency cache - - name: Install dependencies using pipenv - if: steps.python_cache.outputs.cache-hit != 'true' - run: | - pip install pipenv - pipenv install --dev --deploy --system - - # This step caches our pre-commit environment. To make sure we - # do create a new environment when our pre-commit setup changes, - # we create a cache key based on relevant factors. - - name: Pre-commit Environment Caching - uses: actions/cache@v2 - with: - path: ${{ env.PRE_COMMIT_HOME }} - key: "precommit-0-${{ runner.os }}-${{ env.PRE_COMMIT_HOME }}-\ - ${{ steps.python.outputs.python-version }}-\ - ${{ hashFiles('./.pre-commit-config.yaml') }}" - - # We will not run `flake8` here, as we will use a separate flake8 - # action. As pre-commit does not support user installs, we set - # PIP_USER=0 to not do a user install. - - name: Run pre-commit hooks - run: export PIP_USER=0; SKIP=flake8 pre-commit run --all-files - - # This step requires `pull_request_target`, as adding annotations - # requires "write" permissions to the repo. - - name: Run flake8 - uses: julianwachholz/flake8-action@v1 - with: - checkName: lint-test - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Run database using docker-compose - run: docker-compose run -d -p 7777:5432 --name pydis_web postgres - - - name: Migrations and run tests with coverage.py - run: | - python manage.py makemigrations --check - python manage.py migrate - coverage run manage.py test --no-input - coverage report -m - env: - CI: True - DATABASE_URL: postgres://pysite:pysite@localhost:7777/pysite - METRICITY_DB_URL: postgres://pysite:pysite@localhost:7777/metricity - - # This step will publish the coverage reports coveralls.io and - # print a "job" link in the output of the GitHub Action - - name: Publish coverage report to coveralls.io - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: coveralls - - - name: Tear down docker-compose containers - run: docker-compose stop - if: ${{ always() }} - - build-and-deploy: - name: Build and Deploy to Kubernetes - needs: lint-test - if: github.event_name != 'pull_request_target' && github.ref == 'refs/heads/master' - runs-on: ubuntu-latest - - steps: - # Create a commit SHA-based tag for the container repositories - - name: Create SHA Container Tag - id: sha_tag - run: | - tag=$(cut -c 1-7 <<< $GITHUB_SHA) - echo "::set-output name=tag::$tag" - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to Github Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ secrets.GHCR_USER }} - password: ${{ secrets.GHCR_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile - push: true - cache-from: type=registry,ref=ghcr.io/python-discord/site:latest - tags: | - ghcr.io/python-discord/site:latest - ghcr.io/python-discord/site:${{ steps.sha_tag.outputs.tag }} - - - name: Authenticate with Kubernetes - uses: azure/k8s-set-context@v1 - with: - method: kubeconfig - kubeconfig: ${{ secrets.KUBECONFIG }} - - - name: Deploy to Kubernetes - uses: Azure/k8s-deploy@v1 - with: - manifests: | - deployment.yaml - images: 'ghcr.io/python-discord/site:${{ steps.sha_tag.outputs.tag }}' - kubectl-version: 'latest' diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml new file mode 100644 index 00000000..80305322 --- /dev/null +++ b/.github/workflows/lint-test.yaml @@ -0,0 +1,118 @@ +name: Lint & Test + +on: + push: + branches: + - master + pull_request: + + +jobs: + lint-test: + runs-on: ubuntu-latest + env: + # Configure pip to cache dependencies and do a user install + PIP_NO_CACHE_DIR: false + PIP_USER: 1 + + # Hide the graphical elements from pipenv's output + PIPENV_HIDE_EMOJIS: 1 + PIPENV_NOSPIN: 1 + + # Make sure pipenv does not try reuse an environment it's running in + PIPENV_IGNORE_VIRTUALENVS: 1 + + # Specify explicit paths for python dependencies and the pre-commit + # environment so we know which directories to cache + PYTHONUSERBASE: ${{ github.workspace }}/.cache/py-user-base + PRE_COMMIT_HOME: ${{ github.workspace }}/.cache/pre-commit-cache + + steps: + - name: Add custom PYTHONUSERBASE to PATH + run: echo '${{ env.PYTHONUSERBASE }}/bin/' >> $GITHUB_PATH + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Setup python + id: python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + + # This step caches our Python dependencies. To make sure we + # only restore a cache when the dependencies, the python version, + # the runner operating system, and the dependency location haven't + # changed, we create a cache key that is a composite of those states. + # + # Only when the context is exactly the same, we will restore the cache. + - name: Python Dependency Caching + uses: actions/cache@v2 + id: python_cache + with: + path: ${{ env.PYTHONUSERBASE }} + key: "python-0-${{ runner.os }}-${{ env.PYTHONUSERBASE }}-\ + ${{ steps.python.outputs.python-version }}-\ + ${{ hashFiles('./Pipfile', './Pipfile.lock') }}" + + # Install our dependencies if we did not restore a dependency cache + - name: Install dependencies using pipenv + if: steps.python_cache.outputs.cache-hit != 'true' + run: | + pip install pipenv + pipenv install --dev --deploy --system + + # This step caches our pre-commit environment. To make sure we + # do create a new environment when our pre-commit setup changes, + # we create a cache key based on relevant factors. + - name: Pre-commit Environment Caching + uses: actions/cache@v2 + with: + path: ${{ env.PRE_COMMIT_HOME }} + key: "precommit-0-${{ runner.os }}-${{ env.PRE_COMMIT_HOME }}-\ + ${{ steps.python.outputs.python-version }}-\ + ${{ hashFiles('./.pre-commit-config.yaml') }}" + + # We will not run `flake8` here, as we will use a separate flake8 + # action. As pre-commit does not support user installs, we set + # PIP_USER=0 to not do a user install. + - name: Run pre-commit hooks + run: export PIP_USER=0; SKIP=flake8 pre-commit run --all-files + + # Run flake8 and have it format the linting errors in the format of + # the GitHub Workflow command to register error annotations. This + # means that our flake8 output is automatically added as an error + # annotation to both the run result and in the "Files" tab of a + # pull request. + # + # Format used: + # ::error file={filename},line={line},col={col}::{message} + - name: Run flake8 + run: "flake8 \ + --format='::error file=%(path)s,line=%(row)d,col=%(col)d::\ + [flake8] %(code)s: %(text)s'" + + - name: Run database using docker-compose + run: docker-compose run -d -p 7777:5432 --name pydis_web postgres + + - name: Migrations and run tests with coverage.py + run: | + python manage.py makemigrations --check + python manage.py migrate + coverage run manage.py test --no-input + coverage report -m + env: + CI: True + DATABASE_URL: postgres://pysite:pysite@localhost:7777/pysite + METRICITY_DB_URL: postgres://pysite:pysite@localhost:7777/metricity + + # This step will publish the coverage reports coveralls.io and + # print a "job" link in the output of the GitHub Action + - name: Publish coverage report to coveralls.io + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: coveralls + + - name: Tear down docker-compose containers + run: docker-compose stop + if: ${{ always() }} -- cgit v1.2.3 From 3ce92eaf0d24018c2e099345dd6704009be83622 Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff Date: Thu, 19 Nov 2020 00:45:49 +0100 Subject: Update badges in README to reflect new workflows I've also restructured it a bit by using reference links instead of inline links. --- README.md | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b449d338..daaa041d 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,24 @@ # Python Discord: Site [![Discord](https://img.shields.io/static/v1?label=Python%20Discord&logo=discord&message=%3E100k%20members&color=%237289DA&logoColor=white)](https://discord.gg/2B963hn) -![Lint, Test & Deploy](https://github.com/python-discord/site/workflows/Lint,%20Test%20&%20Deploy/badge.svg?branch=master) -[![Coverage Status](https://coveralls.io/repos/github/python-discord/site/badge.svg?branch=master)](https://coveralls.io/github/python-discord/site?branch=master) +[![Lint & Test][1]][2] +[![Build & Deploy][3]][4] +[![Coverage Status][5]][6] [![License](https://img.shields.io/github/license/python-discord/site)](LICENSE) -[![Status](https://img.shields.io/website?url=https%3A%2F%2Fpythondiscord.com)][1] +[![Status](https://img.shields.io/website?url=https%3A%2F%2Fpythondiscord.com)][7] -This is all of the code that is responsible for maintaining [our website][1] and all of its subdomains. +This is all of the code that is responsible for maintaining [our website][7] and all of its subdomains. The website is built on Django and should be simple to set up and get started with. If you happen to run into issues with setup, please don't hesitate to open an issue! -If you're looking to contribute or play around with the code, take a look at [the wiki][2] or the [`docs` directory](docs). If you're looking for things to do, check out [our issues][3]. +If you're looking to contribute or play around with the code, take a look at [the wiki][8] or the [`docs` directory](docs). If you're looking for things to do, check out [our issues][9]. -[1]: https://pythondiscord.com -[2]: https://pythondiscord.com/pages/contributing/site/ -[3]: https://github.com/python-discord/site/issues +[1]: https://github.com/python-discord/site/workflows/Lint%20&%20Test/badge.svg?branch=master +[2]: https://github.com/python-discord/site/actions?query=workflow%3A%22Lint+%26+Test%22+branch%3Amaster +[3]: https://github.com/python-discord/site/workflows/Build%20&%20Deploy/badge.svg?branch=master +[4]: https://github.com/python-discord/site/actions?query=workflow%3A%22Build+%26+Deploy%22+branch%3Amaster +[5]: https://coveralls.io/repos/github/python-discord/site/badge.svg?branch=master +[6]: https://coveralls.io/github/python-discord/site?branch=master +[7]: https://pythondiscord.com +[8]: https://pythondiscord.com/pages/contributing/site/ +[9]: https://github.com/python-discord/site/issues -- cgit v1.2.3