From 2c843101843b975ece546b8921d53b3dd4e6974d Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 6 Jun 2019 16:54:33 -0700 Subject: Create shell script for building a dev image and running a shell * Put scripts in a new scripts folder --- scripts/dev.sh | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 scripts/dev.sh (limited to 'scripts/dev.sh') diff --git a/scripts/dev.sh b/scripts/dev.sh new file mode 100755 index 0000000..490021f --- /dev/null +++ b/scripts/dev.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env sh + +# Sets up a development environment and runs a shell in a docker container. +# Usage: dev.sh [--build [--clean]] [ash_args ...] + +if [ "$1" = "--build" ]; then + shift + printf "Building pythondiscord/snekbox-venv:dev..." + + docker build \ + -t pythondiscord/snekbox-venv:dev \ + -f docker/venv.Dockerfile \ + --build-arg DEV=1 \ + -q \ + . \ + >/dev/null \ + && printf " done!\n" || exit "$?" + + if [ "$1" = "--clean" ]; then + shift + dangling_imgs=$(docker images -f "dangling=true" -q) + + if [ -n "${dangling_imgs}" ]; then + printf "Removing dangling images..." + + docker rmi $dangling_imgs >/dev/null \ + && printf " done!\n" || exit "$?" + fi + fi +fi + +docker run \ + -it \ + --rm \ + --privileged \ + --network host \ + -h pdsnk-dev \ + -e PYTHONDONTWRITEBYTECODE=1 \ + -e PIPENV_PIPFILE="/snekbox/Pipfile" \ + -e ENV="/snekbox-local/scripts/.profile" \ + -v "${PWD}":/snekbox-local \ + -w "/snekbox-local" \ + --entrypoint /bin/ash \ + pythondiscord/snekbox-venv:dev \ + "$@" -- cgit v1.2.3 From c1a6440899ced2f3f787352cd1d3ea1f49e520ee Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 20 Jun 2019 16:25:28 -0700 Subject: Fix ownership of coverage file When coverage runs in a container, it is ran under root so the resulting coverage file is owned by root. chown is used to change ownership to be the same as the folder it is in. --- scripts/dev.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'scripts/dev.sh') diff --git a/scripts/dev.sh b/scripts/dev.sh index 490021f..6ebae71 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -31,7 +31,7 @@ fi docker run \ -it \ - --rm \ + --name snekbox_test \ --privileged \ --network host \ -h pdsnk-dev \ @@ -43,3 +43,12 @@ docker run \ --entrypoint /bin/ash \ pythondiscord/snekbox-venv:dev \ "$@" + +# Fix ownership of coverage file +docker start snekbox_test >/dev/null +docker exec \ + -it \ + snekbox_test \ + /bin/ash \ + -c 'chown "$(stat -c "%u:%g" "/snekbox-local")" /snekbox-local/.coverage' +docker rm -f snekbox_test >/dev/null -- cgit v1.2.3 From 495baa3045c63040f460538e94eaaed6a6499fba Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Fri, 21 Jun 2019 19:35:57 -0700 Subject: Fix coverage not finding sources * Mount volume to the same path as the source directory on the host * Keep the container up in the background so it doesn't have to be restarted or the ownership fix --- scripts/dev.sh | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) (limited to 'scripts/dev.sh') diff --git a/scripts/dev.sh b/scripts/dev.sh index 6ebae71..097690b 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -29,26 +29,35 @@ if [ "$1" = "--build" ]; then fi fi +# Keep the container up in the background so it doesn't have to be restarted +# for the ownership fix. +# The volume is mounted to same the path in the container as the source +# directory on the host to ensure coverage can find the source files. docker run \ - -it \ + -td \ --name snekbox_test \ --privileged \ --network host \ -h pdsnk-dev \ -e PYTHONDONTWRITEBYTECODE=1 \ -e PIPENV_PIPFILE="/snekbox/Pipfile" \ - -e ENV="/snekbox-local/scripts/.profile" \ - -v "${PWD}":/snekbox-local \ - -w "/snekbox-local" \ + -e ENV="${PWD}/scripts/.profile" \ + -v "${PWD}":"${PWD}" \ + -w "${PWD}"\ --entrypoint /bin/ash \ pythondiscord/snekbox-venv:dev \ - "$@" + >/dev/null \ + +# Execute the given command(s) +docker exec -it snekbox_test /bin/ash "$@" # Fix ownership of coverage file -docker start snekbox_test >/dev/null +# BusyBox doesn't support --reference for chown docker exec \ -it \ + -e CWD="${PWD}" \ snekbox_test \ /bin/ash \ - -c 'chown "$(stat -c "%u:%g" "/snekbox-local")" /snekbox-local/.coverage' -docker rm -f snekbox_test >/dev/null + -c 'chown "$(stat -c "%u:%g" "${CWD}")" "${CWD}/.coverage"' + +docker rm -f snekbox_test >/dev/null # Stop and remove the container -- cgit v1.2.3 From 2f864a14a3f4d49d11801af45946dbd81e3e343f Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 30 Jul 2019 15:31:19 -0700 Subject: Add comments to Azure Pipelines YAML * Replace some shorthand Docker command options with their full names for clarity --- azure-pipelines.yml | 29 ++++++++++++++++++++++++----- scripts/dev.sh | 9 +++++---- 2 files changed, 29 insertions(+), 9 deletions(-) (limited to 'scripts/dev.sh') diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3b7c1dc..f7b8eb7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,13 +14,16 @@ jobs: inputs: scriptPath: scripts/check_dockerfiles.sh - # Without a login the following Docker build tasks won't add image tags + # Without a login the following Docker build tasks won't add image tags. - task: Docker@2 displayName: 'Log into Docker Hub' inputs: command: login containerRegistry: DockerHubV2 + # The venv image depends on this image. Build it if it can't be pulled + # from Docker Hub, which will be the case if the base Dockerfile has had + # changes. - task: Docker@2 displayName: 'Build Base Image' condition: and(succeeded(), ne(variables['check.BASE_PULL'], True)) @@ -31,6 +34,7 @@ jobs: Dockerfile: docker/base.Dockerfile buildContext: . + # The dev image is never pushed and therefore is always built. - task: Docker@2 displayName: 'Build Development Image' inputs: @@ -41,18 +45,20 @@ jobs: buildContext: . arguments: --build-arg DEV=1 + # The linter and all tests run inside this container. - script: | docker run \ - -td \ + --tty \ + --detach \ --name snekbox_test \ --privileged \ --network host \ - -h pdsnk-dev \ + --hostname pdsnk-dev \ -e PYTHONDONTWRITEBYTECODE=1 \ -e PIPENV_PIPFILE="/snekbox/Pipfile" \ -e ENV="${PWD}/scripts/.profile" \ - -v "${PWD}":"${PWD}" \ - -w "${PWD}"\ + --volume "${PWD}":"${PWD}" \ + --workdir "${PWD}"\ --entrypoint /bin/ash \ pythondiscord/snekbox-venv:dev displayName: 'Start Container' @@ -69,6 +75,7 @@ jobs: testResultsFiles: '**/test-lint.xml' testRunTitle: 'Lint Results' + # Memory limit tests would fail if this isn't disabled. - script: sudo swapoff -a displayName: 'Disable Swap Memory' @@ -96,6 +103,9 @@ jobs: codeCoverageTool: Cobertura summaryFileLocation: '**/coverage.xml' + # When a pull request, only perform this job if images need to be built. + # It's always performed for non-PRs because the final image will always need + # to be built. - job: build displayName: 'Build' condition: > @@ -109,6 +119,7 @@ jobs: ) dependsOn: test + # coalesce() gives variables default values if they are null (i.e. unset). variables: BASE_CHANGED: $[ coalesce(dependencies.test.outputs['check.BASE_CHANGED'], True) ] VENV_CHANGED: $[ coalesce(dependencies.test.outputs['check.VENV_CHANGED'], True) ] @@ -121,6 +132,9 @@ jobs: command: login containerRegistry: DockerHubV2 + # Because this is the base image for the venv image, if the venv needs to + # be built, this base image must also be present. Build it if it has + # changed or can't be pulled from Docker Hub. - task: Docker@2 displayName: 'Build Base Image' condition: > @@ -139,6 +153,7 @@ jobs: Dockerfile: docker/base.Dockerfile buildContext: . + # Also build this image if base has changed - even if this image hasn't. - task: Docker@2 displayName: 'Build Virtual Environment Image' condition: > @@ -156,6 +171,7 @@ jobs: Dockerfile: docker/venv.Dockerfile buildContext: . + # Always build this image unless it's for a pull request. - task: Docker@2 displayName: 'Build Final Image' condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) @@ -166,6 +182,9 @@ jobs: Dockerfile: docker/Dockerfile buildContext: . + # Push images only after they've all successfully been built. + # These have the same conditions as the build tasks. However, for safety, + # a condition for not being a pull request is added. - task: Docker@2 displayName: 'Push Base Image' condition: > diff --git a/scripts/dev.sh b/scripts/dev.sh index 097690b..8f5b24f 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -34,16 +34,17 @@ fi # The volume is mounted to same the path in the container as the source # directory on the host to ensure coverage can find the source files. docker run \ - -td \ + --tty \ + --detach \ --name snekbox_test \ --privileged \ --network host \ - -h pdsnk-dev \ + --hostname pdsnk-dev \ -e PYTHONDONTWRITEBYTECODE=1 \ -e PIPENV_PIPFILE="/snekbox/Pipfile" \ -e ENV="${PWD}/scripts/.profile" \ - -v "${PWD}":"${PWD}" \ - -w "${PWD}"\ + --volume "${PWD}":"${PWD}" \ + --workdir "${PWD}"\ --entrypoint /bin/ash \ pythondiscord/snekbox-venv:dev \ >/dev/null \ -- cgit v1.2.3