diff options
| -rw-r--r-- | .coveragerc | 1 | ||||
| -rw-r--r-- | .github/workflows/build.yaml | 96 | ||||
| -rw-r--r-- | .github/workflows/deploy.yaml | 96 | ||||
| -rw-r--r-- | .github/workflows/lint-test-build-push.yaml | 242 | ||||
| -rw-r--r-- | .github/workflows/lint.yaml | 73 | ||||
| -rw-r--r-- | .github/workflows/main.yaml | 26 | ||||
| -rw-r--r-- | .github/workflows/test.yaml | 97 |
7 files changed, 389 insertions, 242 deletions
diff --git a/.coveragerc b/.coveragerc index cc2a148..ce475d7 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,6 @@ [run] branch = true +data_file = ${COVERAGE_DATAFILE-.coverage} include = snekbox/* omit = snekbox/api/app.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..28e5b69 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,96 @@ +on: + workflow_call: + outputs: + artifact: + description: The name of the uploaded image aretfact. + value: ${{ jobs.build.outputs.artifact }} + tag: + description: The tag used for the built image. + value: ${{ jobs.build.outputs.tag }} + +jobs: + build: + name: Build snekbox-venv image + runs-on: ubuntu-latest + outputs: + artifact: ${{ env.artifact }} + tag: ${{ steps.sha_tag.outputs.tag }} + env: + artifact: image_artifact_snekbox-venv + + steps: + # Create a short SHA with which to tag built images. + - name: Create SHA Container Tag + id: sha_tag + run: | + tag=$(cut -c 1-7 <<< $GITHUB_SHA) + echo "::set-output name=tag::$tag" + + - name: Checkout code + uses: actions/checkout@v2 + + # The current version (v2) of Docker's build-push action uses buildx, + # which comes with BuildKit. It has cache features which can speed up + # the builds. See https://github.com/docker/build-push-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + # The image built for PRs may start to deviate from the "latest" image + # currently in GHCR. Configure the subsequent build step to cache the + # layers in GitHub Actions for PRs. + # See https://github.com/moby/buildkit#github-actions-cache-experimental + # + # Because the cache is scoped to the branch, it will not be available + # on the main branch when the PR is merged. Furthermore, using this cache + # on main is redundant since the previous build's images are already + # cached on GHCR. Thus, this step is only used for PRs. + - name: Configure cache + id: cache_config + run: | + set -eu + if [ "$GITHUB_EVENT_NAME" = 'pull_request' ]; then + cache_from="type=gha,scope=buildkit-${GITHUB_REF}" + cache_to="${cache_from},mode=max" + fi + echo "::set-output name=cache_from::${cache_from:-}" + echo "::set-output name=cache_to::${cache_to:-}" + + # Build the "DEV" version of the image, which targets the `venv` stage + # and includes development dependencies. + # + # Include an inline cache manifest in the image to support caching from + # GHCR. This enables subsequent builds to pull reusable layers from GHCR. + # If configured by the cache_config step, also cache the layers in + # GitHub Actions. + - name: Build image for linting and testing + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + push: false + target: venv + build-args: DEV=1 + outputs: type=docker,dest=${{ env.artifact }}.tar + cache-from: | + ${{ steps.cache_config.outputs.cache_from }} + ghcr.io/python-discord/snekbox-base:latest + ghcr.io/python-discord/snekbox-venv:latest + cache-to: ${{ steps.cache_config.outputs.cache_to }} + tags: ghcr.io/python-discord/snekbox-venv:${{ steps.sha_tag.outputs.tag }} + + # Make the image available as an artifact so other jobs will be able to + # download it. + - name: Upload image archive as an artifact + uses: actions/upload-artifact@v2 + with: + name: ${{ env.artifact }} + path: ${{ env.artifact }}.tar + retention-days: 1 # It's only needed for the duration of the workflow. + if-no-files-found: error diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 0000000..c4020b9 --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,96 @@ +on: + workflow_call: + inputs: + artifact: + required: true + type: string + tag: + required: true + type: string + +jobs: + deploy: + name: Build, push, & deploy + runs-on: ubuntu-latest + + steps: + - name: Download image artifact + uses: actions/download-artifact@v2 + with: + name: ${{ inputs.artifact }} + + # Load the image to make use of common layers during the final build. + - name: Load image from archive + run: docker load -i ${{ inputs.artifact }}.tar + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + # The Dockerfile will be needed. + - name: Checkout code + uses: actions/checkout@v2 + + # Build the final production image and push it to GHCR. + # Tag it with both the short commit SHA and 'latest'. + - name: Build final image + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + push: true + cache-from: | + ghcr.io/python-discord/snekbox-base:latest + ghcr.io/python-discord/snekbox-venv:latest + ghcr.io/python-discord/snekbox:latest + cache-to: type=inline + tags: | + ghcr.io/python-discord/snekbox:latest + ghcr.io/python-discord/snekbox:${{ inputs.tag }} + build-args: git_sha=${{ github.sha }} + + # Deploy to Kubernetes. + - name: Authenticate with Kubernetes + uses: azure/k8s-set-context@v1 + with: + method: kubeconfig + kubeconfig: ${{ secrets.KUBECONFIG }} + + - name: Deploy to Kubernetes + uses: Azure/k8s-deploy@v1 + with: + manifests: deployment.yaml + images: 'ghcr.io/python-discord/snekbox:${{ inputs.tag }}' + kubectl-version: 'latest' + + # Push the base image to GHCR, with an inline cache manifest. + - name: Push base image + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + target: base + push: true + cache-from: ghcr.io/python-discord/snekbox-base:latest + cache-to: type=inline + tags: ghcr.io/python-discord/snekbox-base:latest + + # Push the venv image to GHCR, with an inline cache manifest. + - name: Push venv image + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + target: venv + push: true + cache-from: | + ghcr.io/python-discord/snekbox-base:latest + ghcr.io/python-discord/snekbox-venv:latest + cache-to: type=inline + tags: ghcr.io/python-discord/snekbox-venv:latest diff --git a/.github/workflows/lint-test-build-push.yaml b/.github/workflows/lint-test-build-push.yaml deleted file mode 100644 index 0df0b32..0000000 --- a/.github/workflows/lint-test-build-push.yaml +++ /dev/null @@ -1,242 +0,0 @@ -name: Lint, Test, Build, Push - -on: - push: - branches: - - main - pull_request: - - -jobs: - lint-test-build-push: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-20.04, self-hosted] - include: - - os: ubuntu-20.04 - full: true - - os: self-hosted - full: false # Only run tests. - - env: - # Determine whether or not we should build the - # final production image and push it to GHCR. - production_build: ${{ github.event_name != 'pull_request' && - github.ref == 'refs/heads/main' }} - - steps: - # Create a short SHA-tag to tag built images - - name: Create SHA Container Tag - id: sha_tag - run: | - tag=$(cut -c 1-7 <<< $GITHUB_SHA) - echo "::set-output name=tag::$tag" - - - name: Checkout code - uses: actions/checkout@v2 - - # The current version (v2) of Docker's build-push action uses - # buildx, which comes with BuildKit features that help us speed - # up our builds using additional cache features. Buildx also - # has a lot of other features that are not as relevant to us. - # - # See https://github.com/docker/build-push-action - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to Github Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Create a local cache directory for PR builds, as the image - # we build for PRs may start to deviate from the "latest" image - # currently registered in the GHCR. For main, the best we can - # do is use the previous main build, which can be cached from - # the GHCR. - - name: Cache Image Layers - if: github.event_name == 'pull_request' - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-v0-buildx-${{ github.ref }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-v0-buildx-${{ github.ref }}- - - # Build the image we need for linting and testing using the - # `venv` target stage within our Dockerfile. We load the image - # into the runner's Docker image collection so we can run it - # later. - # - # The image includes an inline cache manifest to support caching - # from the GHCR, which means that a build can pull the layers it - # can reuse instead of building them from scratch. - - name: Build image for linting and testing - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile - push: false - load: true - target: venv - build-args: DEV=1 - cache-from: | - type=local,src=/tmp/.buildx-cache - ghcr.io/python-discord/snekbox-base:latest - ghcr.io/python-discord/snekbox-venv:latest - cache-to: type=local,dest=/tmp/.buildx-cache,mode=max - tags: ghcr.io/python-discord/snekbox-venv:${{ steps.sha_tag.outputs.tag }} - - - name: Start Container - run: | - export IMAGE_SUFFIX='-venv:${{ steps.sha_tag.outputs.tag }}' - docker-compose up --no-build -d - - # Required by pre-commit. - - name: Install git - if: matrix.full - run: >- - docker exec snekbox_dev /bin/bash -c - 'apt-get -y update && apt-get install -y git=1:2.20.*' - - # pre-commit's venv doesn't work with user installs. - # Skip the flake8 hook because the following step will run it. - - name: Run pre-commit hooks - id: run-pre-commit-hooks - if: matrix.full - run: >- - docker exec snekbox_dev /bin/bash -c - 'PIP_USER=0 SKIP=flake8 pre-commit run --all-files' - - - name: Show pre-commit logs - if: matrix.full && always() && steps.run-pre-commit-hooks.outcome != 'success' - run: >- - docker exec snekbox_dev /bin/bash -c - 'cat /root/.cache/pre-commit/pre-commit.log' - - # This runs `flake8` in the container and asks `flake8` to output - # linting errors in the format of the command for registering workflow - # error messages/annotations. This means that Github Actions will pick - # up on this output to generate nice annotations to indicate what went - # wrong where. - - name: Run linter - if: matrix.full - run: >- - docker exec snekbox_dev /bin/bash -c - 'flake8 --format - "::error file=%(path)s,line=%(row)d,col=%(col)d::[flake8] %(code)s: %(text)s"' - - # Memory limit tests would fail if this isn't disabled. - - name: Disable swap memory - run: sudo swapoff -a - - # Run unittests and generate coverage report in the container - - name: Run unit tests - id: run_tests - run: | - echo '::set-output name=started::true' - docker exec snekbox_dev /bin/bash -c 'coverage run -m unittest' - - - name: Generate coverage report - if: always() && steps.run_tests.outputs.started == 'true' - run: docker exec snekbox_dev /bin/bash -c 'coverage report -m' - - # Set-up a Python version to process the coverage reports - # Note: This step runs even if the test step failed to make - # sure we process the coverage reports. - - name: Setup python - if: matrix.os != 'self-hosted' && always() && steps.run_tests.outputs.started == 'true' - id: python - uses: actions/setup-python@v2 - with: - python-version: '3.10' - - # We'll only ever need a single dependency in this python - # environment and we'll only use it in the CI, so let's - # install it directly here and run it. - # - # This step will publish the coverage results to coveralls.io - # print a job link in the output. It will also register a - # step in the check suite visible in the PR with a link to - # the job. - - name: Publish coverage report to coveralls.io - if: always() && steps.run_tests.outputs.started == 'true' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - pip install coveralls~=2.1 - coveralls - - # Final build stage. This is run in the same job with conditions - # in order to use the local build cache generated by buildx while - # building the `venv` image in the lint/test phase. - - # Build the final production image and push it to GHCR, tagging it - # both with the short commit SHA and 'latest'. This step should use - # the local build cache of the current run. - - name: Build final image - if: matrix.full && env.production_build == 'true' - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile - push: true - cache-from: | - ghcr.io/python-discord/snekbox-base:latest - ghcr.io/python-discord/snekbox-venv:latest - ghcr.io/python-discord/snekbox:latest - cache-to: type=inline - tags: | - ghcr.io/python-discord/snekbox:latest - ghcr.io/python-discord/snekbox:${{ steps.sha_tag.outputs.tag }} - build-args: | - git_sha=${{ github.sha }} - - # Deploy to Kubernetes - - name: Authenticate with Kubernetes - if: matrix.full && env.production_build == 'true' - uses: azure/k8s-set-context@v1 - with: - method: kubeconfig - kubeconfig: ${{ secrets.KUBECONFIG }} - - - name: Deploy to Kubernetes - if: matrix.full && env.production_build == 'true' - uses: Azure/k8s-deploy@v1 - with: - manifests: | - deployment.yaml - images: 'ghcr.io/python-discord/snekbox:${{ steps.sha_tag.outputs.tag }}' - kubectl-version: 'latest' - - # Push the base image to GHCR, with an inline cache manifest - - name: Push base image - if: matrix.full && env.production_build == 'true' - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile - target: base - push: true - cache-from: | - ghcr.io/python-discord/snekbox-base:latest - cache-to: type=inline - tags: ghcr.io/python-discord/snekbox-base:latest - - # Push the venv image to GHCR, with an inline cache manifest - - name: Push venv image - if: matrix.full && env.production_build == 'true' - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile - target: venv - push: true - cache-from: | - ghcr.io/python-discord/snekbox-base:latest - ghcr.io/python-discord/snekbox-venv:latest - cache-to: type=inline - tags: ghcr.io/python-discord/snekbox-venv:latest diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 0000000..8f530ab --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,73 @@ +on: + workflow_call: + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + PIP_NO_CACHE_DIR: false + PIP_USER: 1 # Make dependencies install into PYTHONUSERBASE. + + PIPENV_DONT_USE_PYENV: 1 + PIPENV_HIDE_EMOJIS: 1 + PIPENV_NOSPIN: 1 + + PYTHONUSERBASE: ${{ github.workspace }}/.cache/py-user-base + PRE_COMMIT_HOME: ${{ github.workspace }}/.cache/pre-commit-cache + + steps: + - name: Add custom PYTHONUSERBASE to PATH + run: echo '${{ env.PYTHONUSERBASE }}/bin/' >> $GITHUB_PATH + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python + id: python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + + - name: Python dependency cache + uses: actions/cache@v2 + id: python_cache + with: + path: ${{ env.PYTHONUSERBASE }} + key: "python-0-${{ runner.os }}-${{ env.PYTHONUSERBASE }}-\ + ${{ steps.python.outputs.python-version }}-\ + ${{ hashFiles('./Pipfile', './Pipfile.lock') }}" + + # Install dependencies if there was a cache miss. + - name: Install Python dependencies + if: steps.python_cache.outputs.cache-hit != 'true' + run: >- + pip install pipenv==2021.11.23 + && pipenv install --deploy --system --dev + + - name: Pre-commit environment cache + uses: actions/cache@v2 + with: + path: ${{ env.PRE_COMMIT_HOME }} + key: "precommit-0-${{ runner.os }}-${{ env.PRE_COMMIT_HOME }}-\ + ${{ steps.python.outputs.python-version }}-\ + ${{ hashFiles('./.pre-commit-config.yaml') }}" + + # pre-commit's venv doesn't work with user installs. + # Skip the flake8 hook because the following step will run it. + - name: Run pre-commit hooks + id: run-pre-commit-hooks + run: PIP_USER=0 SKIP=flake8 pre-commit run --all-files + + # Show the log to debug failures. + - name: Show pre-commit log + if: always() && steps.run-pre-commit-hooks.outcome == 'failure' + run: cat "${PRE_COMMIT_HOME}/pre-commit.log" + + # Output linting errors in the format GitHub Actions recognises for + # annotations. + - name: Run flake8 + run: >- + flake8 --format "::error + file=%(path)s,line=%(row)d,col=%(col)d::[flake8] %(code)s: %(text)s" diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 0000000..516a1d6 --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,26 @@ +name: main + +on: + push: + branches: + - main + pull_request: + +jobs: + build: + uses: ./.github/workflows/build.yaml + lint: + uses: ./.github/workflows/lint.yaml + test: + uses: ./.github/workflows/test.yaml + needs: build + with: + artifact: ${{ needs.build.outputs.artifact }} + tag: ${{ needs.build.outputs.tag }} + deploy: + uses: ./.github/workflows/deploy.yaml + if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/main' }} + needs: [build, lint, test] + with: + artifact: ${{ needs.build.outputs.artifact }} + tag: ${{ needs.build.outputs.tag }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..77ef1fc --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,97 @@ +on: + workflow_call: + inputs: + artifact: + required: true + type: string + tag: + required: true + type: string + +jobs: + test: + name: Test with coverage + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04, self-hosted] + + steps: + - name: Download image artifact + uses: actions/download-artifact@v2 + with: + name: ${{ inputs.artifact }} + + - name: Load image from archive + run: docker load -i ${{ inputs.artifact }}.tar + + # Needed for the Docker Compose file. + - name: Checkout code + uses: actions/checkout@v2 + + # Memory limit tests would fail if this isn't disabled. + - name: Disable swap memory + run: sudo swapoff -a + + # Run tests with coverage within the container. + # Suffix the generated coverage datafile with the name of the runner's OS. + - name: Run tests + id: run_tests + run: | + export IMAGE_SUFFIX='-venv:${{ inputs.tag }}' + docker-compose run \ + --rm -T -e COVERAGE_DATAFILE=.coverage.${{ matrix.os }} \ + snekbox \ + coverage run -m unittest + + # Upload it so the coverage from all matrix jobs can be combined later. + - name: Upload coverage data + uses: actions/upload-artifact@v2 + with: + name: coverage + path: .coverage.* + retention-days: 1 + + # Self-hosted runner needs containers, images, networks, volumes, etc. + # removed because they persist across runs and may interfere. + - name: Docker cleanup + if: matrix.os == 'self-hosted' && always() + run: | + export IMAGE_SUFFIX='-venv:${{ inputs.tag }}' + docker-compose down --rmi all --remove-orphans -v -t 0 + + report: + name: Report coverage + runs-on: ubuntu-20.04 + needs: test + + steps: + # Needed for the .coveragerc file. + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + + - name: Install dependencies + run: pip install coverage~=6.0 coveralls~=3.3 + + - name: Download coverage data + uses: actions/download-artifact@v2 + with: + name: coverage + + - name: Combine coverage data + run: coverage combine .coverage.* + + - name: Display coverage report + run: coverage report -m + + # Comment on the PR with the coverage results and register a GitHub check + # which links to the coveralls.io job. + - name: Publish coverage report to coveralls.io + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: coveralls --service=github |