diff options
-rw-r--r-- | .github/workflows/lint-test-build-push.yaml (renamed from .github/workflows/lint-test.yaml) | 2 | ||||
-rw-r--r-- | Pipfile | 15 | ||||
-rw-r--r-- | README.md | 16 | ||||
-rw-r--r-- | azure-pipelines.yml | 20 | ||||
-rw-r--r-- | ci/build.yml | 51 | ||||
-rw-r--r-- | ci/lint-test.yml | 41 | ||||
-rw-r--r-- | ci/push.yml | 39 | ||||
-rw-r--r-- | ci/setup.yml | 23 | ||||
-rw-r--r-- | docker-compose.yml | 2 | ||||
-rw-r--r-- | docker/Dockerfile | 2 | ||||
-rw-r--r-- | docker/base.Dockerfile | 55 | ||||
-rw-r--r-- | docker/venv.Dockerfile | 20 | ||||
-rwxr-xr-x | scripts/check_dockerfiles.sh | 127 | ||||
-rwxr-xr-x | scripts/dev.sh | 9 |
14 files changed, 26 insertions, 396 deletions
diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test-build-push.yaml index a668905..a2ebbb5 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test-build-push.yaml @@ -1,4 +1,4 @@ -name: Build, Lint, Test +name: Lint, Test, Build, Push on: push: @@ -67,20 +67,23 @@ build = """ """ buildbase = """ docker build \ - -t pythondiscord/snekbox-base:latest \ - -f docker/base.Dockerfile \ + -t ghcr.io/python-discord/snekbox-base:latest \ + -f docker/Dockerfile \ + --target base \ . """ buildvenv = """ docker build \ - -t pythondiscord/snekbox-venv:latest \ - -f docker/venv.Dockerfile \ + -t ghcr.io/python-discord/snekbox-venv:latest \ + -f docker/Dockerfile \ + --target venv \ . """ builddev = """ docker build \ - -t pythondiscord/snekbox-venv:dev \ - -f docker/venv.Dockerfile \ + -t ghcr.io/python-discord/snekbox-venv:dev \ + -f docker/Dockerfile \ + --target venv \ --build-arg DEV=1 \ . """ @@ -1,4 +1,4 @@ -[![Build Status][1]][2] +[![Build Status][1]][2] [![Coverage Status][3]][4] # snekbox @@ -33,10 +33,10 @@ See [`snekapi.py`] and [`resources`] for API documentation. ## Running snekbox -A Docker image is available on [Docker Hub]. A container can be started with the following command, which will also pull the image if it doesn't currently exist locally: +A Docker image is available in the [GitHub Container Registry]. A container can be started with the following command, which will also pull the image if it doesn't currently exist locally: ``` -docker run --ipc=none --privileged -p 8060:8060 pythondiscord/snekbox +docker run --ipc=none --privileged -p 8060:8060 ghcr.io/python-discord/snekbox ``` To run it in the background, use the `-d` option. See the documentation on [`docker run`] for more information. @@ -47,7 +47,7 @@ The above command will make the API accessible on the host via `http://localhost ### Initial Setup -A Python 3.8 interpreter and the [pipenv] package are required. Once those requirements are satisfied, install the project's dependencies: +A Python 3.9 interpreter and the [pipenv] package are required. Once those requirements are satisfied, install the project's dependencies: ``` pipenv sync --dev @@ -142,8 +142,10 @@ nsjpy "print('hello world!')" The alias can be found in `./scripts/.profile`, which is automatically added when the shell is launched in the container. -[1]: https://dev.azure.com/python-discord/Python%20Discord/_apis/build/status/Snekbox?branchName=master -[2]: https://dev.azure.com/python-discord/Python%20Discord/_build/latest?definitionId=13&branchName=master +[1]: https://github.com/python-discord/snekbox/workflows/Lint,%20Test,%20Build,%20Push/badge.svg?branch=master +[2]: https://github.com/python-discord/snekbox/actions?query=workflow%3A%22Lint%2C+Test%2C+Build%2C+Push%22+branch%3Amaster +[3]: https://coveralls.io/repos/github/python-discord/snekbox/badge.svg?branch=master +[4]: https://coveralls.io/github/python-discord/snekbox?branch=master [`snekbox.cfg`]: config/snekbox.cfg [`snekapi.py`]: snekbox/api/snekapi.py [`resources`]: snekbox/api/resources @@ -151,5 +153,5 @@ The alias can be found in `./scripts/.profile`, which is automatically added whe [nsjail]: https://github.com/google/nsjail [falcon]: https://falconframework.org/ [gunicorn]: https://gunicorn.org/ -[docker hub]: https://hub.docker.com/r/pythondiscord/snekbox +[GitHub Container Registry]: https://github.com/orgs/python-discord/packages/container/package/snekbox [pipenv]: https://docs.pipenv.org/en/latest/ diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 573e3cc..0000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,20 +0,0 @@ -# https://aka.ms/yaml - -jobs: - - job: build_lint_test_push - displayName: 'Build, Lint, Test, & Push' - - pool: - vmImage: 'ubuntu-18.04' - - variables: - BASE_CHANGED: 'True' - VENV_CHANGED: 'True' - BASE_PULL: 'False' - VENV_PULL: 'False' - - steps: - - template: ci/build.yml - - template: ci/setup.yml - - template: ci/lint-test.yml - - template: ci/push.yml diff --git a/ci/build.yml b/ci/build.yml deleted file mode 100644 index 7d51709..0000000 --- a/ci/build.yml +++ /dev/null @@ -1,51 +0,0 @@ -steps: - - task: ShellScript@2 - displayName: 'Check If Images Need to Be Built' - inputs: - scriptPath: scripts/check_dockerfiles.sh - disableAutoCwd: true - - # Without a login, the following Docker build steps wouldn't add image tags. - - task: Docker@1 - displayName: 'Log into Docker Hub' - inputs: - command: login - containerregistrytype: 'Container Registry' - dockerRegistryEndpoint: 'DockerHub' - - # Building the venv depends on this base image. Build the base if it can't - # pulled from Docker Hub, which will be the case if the base Dockerfile has - # has had changes. - - script: | - docker build \ - -f docker/base.Dockerfile \ - -t pythondiscord/snekbox-base:latest \ - . - displayName: 'Build Base Image' - condition: > - and( - succeeded(), - or( - eq(variables.BASE_CHANGED, 'True'), - and( - eq(variables.VENV_CHANGED, 'True'), - eq(variables.BASE_PULL, 'False') - ) - ) - ) - - # Build the venv image if it's had changes or it can't be pulled. - - script: | - docker build \ - -f docker/venv.Dockerfile \ - -t pythondiscord/snekbox-venv:latest \ - . - displayName: 'Build Virtual Environment Image' - condition: > - and( - succeeded(), - or( - eq(variables.VENV_CHANGED, 'True'), - eq(variables.VENV_PULL, 'False') - ) - ) diff --git a/ci/lint-test.yml b/ci/lint-test.yml deleted file mode 100644 index 2d70f6e..0000000 --- a/ci/lint-test.yml +++ /dev/null @@ -1,41 +0,0 @@ -steps: - - script: | - docker exec snekbox_test /bin/bash -c \ - 'flake8; flake8 --format junit-xml --output-file test-lint.xml' - displayName: 'Run Linter' - - - task: PublishTestResults@2 - displayName: 'Publish Lint Results' - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-lint.xml' - testRunTitle: 'Lint Results' - - # Memory limit tests would fail if this isn't disabled. - - script: sudo swapoff -a - displayName: 'Disable Swap Memory' - - - script: | - docker exec snekbox_test /bin/bash -c \ - 'coverage run -m xmlrunner' - displayName: 'Run Unit Tests' - - - task: PublishTestResults@2 - displayName: 'Publish Test Results' - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/TEST-*.xml' - testRunTitle: 'Test Results' - - # Run report too because the XML report doesn't output to stdout. - - script: | - docker exec snekbox_test /bin/bash -c \ - 'coverage report && coverage xml' - displayName: 'Generate Coverage Report' - - - task: PublishCodeCoverageResults@1 - displayName: 'Publish Coverage Results' - condition: succeededOrFailed() - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '**/coverage.xml' diff --git a/ci/push.yml b/ci/push.yml deleted file mode 100644 index 9449df0..0000000 --- a/ci/push.yml +++ /dev/null @@ -1,39 +0,0 @@ -steps: - # Always build this image unless it's for a pull request. - - script: | - docker build \ - -f docker/Dockerfile \ - -t pythondiscord/snekbox:latest \ - . - displayName: 'Build Final Image' - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - - # Push images only after they've all successfully been built. - - script: docker push pythondiscord/snekbox-base:latest - displayName: 'Push Base Image' - condition: > - and( - succeeded(), - ne(variables['Build.Reason'], 'PullRequest'), - ne(variables.BASE_PULL, 'True'), - or( - eq(variables.BASE_CHANGED, 'True'), - eq(variables.VENV_CHANGED, 'True') - ) - ) - - - script: docker push pythondiscord/snekbox-venv:latest - displayName: 'Push Virtual Environment Image' - condition: > - and( - succeeded(), - ne(variables['Build.Reason'], 'PullRequest'), - or( - eq(variables.BASE_CHANGED, 'True'), - eq(variables.VENV_CHANGED, 'True') - ) - ) - - - script: docker push pythondiscord/snekbox:latest - displayName: 'Push Final Image' - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) diff --git a/ci/setup.yml b/ci/setup.yml deleted file mode 100644 index 1c66aa7..0000000 --- a/ci/setup.yml +++ /dev/null @@ -1,23 +0,0 @@ -steps: - # The linter and all tests run inside this container. - # The venv image will be pulled if it doesn't exist locally. - - script: | - docker run \ - --tty \ - --detach \ - --name snekbox_test \ - --privileged \ - --hostname pdsnk-dev \ - -e PYTHONDONTWRITEBYTECODE=1 \ - -e PIPENV_PIPFILE="/snekbox/Pipfile" \ - -e ENV="${PWD}/scripts/.profile" \ - --volume "${PWD}":"${PWD}" \ - --workdir "${PWD}"\ - --entrypoint /bin/bash \ - pythondiscord/snekbox-venv:latest - displayName: 'Start Container' - - - script: | - docker exec snekbox_test /bin/bash -c \ - 'pipenv install --system --deploy --dev' - displayName: 'Install Development Dependencies' diff --git a/docker-compose.yml b/docker-compose.yml index aec17d3..066f38b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: snekbox: container_name: snekbox privileged: true - image: pythondiscord/snekbox:latest + image: ghcr.io/python-discord/snekbox:latest ports: - 8060:8060 init: true diff --git a/docker/Dockerfile b/docker/Dockerfile index 693da6e..ea05c5c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -34,8 +34,8 @@ COPY --from=builder /nsjail/nsjail /usr/sbin/ RUN chmod +x /usr/sbin/nsjail FROM base as venv - ARG DEV + ENV PIP_NO_CACHE_DIR=false \ PIPENV_DONT_USE_PYENV=1 \ PIPENV_HIDE_EMOJIS=1 \ diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile deleted file mode 100644 index 1655aa1..0000000 --- a/docker/base.Dockerfile +++ /dev/null @@ -1,55 +0,0 @@ -FROM python:3.9-slim-buster as builder -RUN apt-get -y update \ - && apt-get install -y \ - bison=2:3.3.* \ - flex=2.6.* \ - g++=4:8.3.* \ - gcc=4:8.3.* \ - git=1:2.20.* \ - libprotobuf-dev=3.6.* \ - libnl-route-3-dev=3.4.* \ - make=4.2.* \ - pkg-config=0.29-6 \ - protobuf-compiler=3.6.* -RUN git clone \ - -b '2.9' \ - --single-branch \ - --depth 1 \ - https://github.com/google/nsjail.git /nsjail -WORKDIR /nsjail -RUN make - -FROM python:3.9-slim-buster as venv -ENV PIP_NO_CACHE_DIR=false - -RUN apt-get -y update \ - && apt-get install -y \ - gcc=4:8.3.* \ - libnl-route-3-200=3.4.* \ - libprotobuf17=3.6.* \ - && rm -rf /var/lib/apt/lists/* -RUN pip install pipenv==2020.11.4 - -COPY --from=builder /nsjail/nsjail /usr/sbin/ -RUN chmod +x /usr/sbin/nsjail - -FROM venv - -ARG DEV -ENV PIP_NO_CACHE_DIR=false \ - PIPENV_DONT_USE_PYENV=1 \ - PIPENV_HIDE_EMOJIS=1 \ - PIPENV_NOSPIN=1 - -COPY Pipfile Pipfile.lock /snekbox/ -WORKDIR /snekbox - -RUN if [ -n "${DEV}" ]; \ - then \ - pipenv install --deploy --system --dev; \ - else \ - pipenv install --deploy --system; \ - fi - -# At the end to avoid re-installing dependencies when only a config changes. -COPY config/ /snekbox/config diff --git a/docker/venv.Dockerfile b/docker/venv.Dockerfile deleted file mode 100644 index 27932cd..0000000 --- a/docker/venv.Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM ghcr.io/python-discord/snekbox-base:latest - -ARG DEV -ENV PIP_NO_CACHE_DIR=false \ - PIPENV_DONT_USE_PYENV=1 \ - PIPENV_HIDE_EMOJIS=1 \ - PIPENV_NOSPIN=1 - -COPY Pipfile Pipfile.lock /snekbox/ -WORKDIR /snekbox - -RUN if [ -n "${DEV}" ]; \ - then \ - pipenv install --deploy --system --dev; \ - else \ - pipenv install --deploy --system; \ - fi - -# At the end to avoid re-installing dependencies when only a config changes. -COPY config/ /snekbox/config diff --git a/scripts/check_dockerfiles.sh b/scripts/check_dockerfiles.sh deleted file mode 100755 index 88cb7cc..0000000 --- a/scripts/check_dockerfiles.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -shopt -s inherit_errexit -exec 3>&1 # New file descriptor to stdout - -BASE_URL="https://dev.azure.com/\ -python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?\ -queryOrder=finishTimeDescending&\ -resultFilter=succeeded&\ -\$top=1&\ -repositoryType=${BUILD_REPOSITORY_PROVIDER}&\ -repositoryId=${BUILD_REPOSITORY_NAME}&\ -api-version=5.0" - -declare -A build_cache - -get_build() { - local branch="${1:?"get_build: argument 1 'branch' is unset"}" - - # Attempt to use cached value - if [[ -v build_cache["${branch}"] ]]; then - printf '%s\n' "Retrieving build for ${branch} from cache." >&3 - printf '%s' "${build_cache[$branch]}" - return 0 - fi - - local url="${BASE_URL}&branchName=${branch}" - - printf '%s\n' "Retrieving the latest successful build using ${url}" >&3 - - local response - response="$(curl -sSL "${url}")" - - if [[ -z "${response}" ]] \ - || ! count="$(printf '%s' "${response}" | jq -re '.count')" \ - || (( "${count}" < 1 )) - then - return 1 - else - # Cache the response - build_cache["${branch}"]="${response}" - printf '%s' "${response}" - fi -} - -can_pull() { - local image="${1:?"can_pull: argument 1 'image' is unset"}" - - local master_commit - if master_commit="$( - get_build "refs/heads/master" \ - | jq -re '.value[0].sourceVersion' - )" \ - && git diff --quiet "${master_commit}" -- "${@:2}" - then - printf \ - '%s\n' \ - "Can pull ${image} image from Docker Hub; no changes since master." - - printf '%s\n' "##vso[task.setvariable variable=${image^^}_PULL]True" - else - printf \ - '%s\n' \ - "Cannot pull ${image} image from Docker Hub due to detected " \ - "changes; the ${image} image will be built." - - return 1 - fi -} - -# Get the previous commit -if [[ "${BUILD_REASON}" = "PullRequest" ]]; then - if ! prev_commit="$( - get_build "${BUILD_SOURCEBRANCH}" \ - | jq -re '.value[0].triggerInfo."pr.sourceSha"' - )" - then - echo \ - "Could not retrieve the previous build's commit." \ - "Falling back to the head of the target branch." - - prev_commit="origin/${SYSTEM_PULLREQUEST_TARGETBRANCH}" - fi -elif ! prev_commit="$( - get_build "${BUILD_SOURCEBRANCH}" \ - | jq -re '.value[0].sourceVersion' - )" -then - echo \ - "No previous build was found." \ - "Either the previous build is too old and was deleted" \ - "or the branch was empty before this build." \ - "All images will be built." - exit 0 -fi - -# Compare diffs -head="$(git rev-parse HEAD)" -printf '%s\n' "Comparing HEAD (${head}) against ${prev_commit}." - -if git diff --quiet "${prev_commit}" -- docker/base.Dockerfile; then - echo "No changes detected in docker/base.Dockerfile." - echo "##vso[task.setvariable variable=BASE_CHANGED]False" -else - # Always rebuild the venv if the base changes. - echo "Changes detected in docker/base.Dockerfile; all images will be built." - exit 0 -fi - -if git diff --quiet "${prev_commit}" -- docker/venv.Dockerfile Pipfile*; then - echo "No changes detected in docker/venv.Dockerfile or the Pipfiles." - echo "##vso[task.setvariable variable=VENV_CHANGED]False" - - if ! can_pull venv docker/venv.Dockerfile Pipfile*; then - # Venv image can't be pulled so it needs to be built. - # Therefore, the base image is needed too. - can_pull base docker/base.Dockerfile || true - fi -else - echo \ - "Changes detected in docker/venv.Dockerfile or the Pipfiles;" \ - "the venv image will be built." - - # Though base image hasn't changed, it's still needed to build the venv. - can_pull base docker/base.Dockerfile || true -fi diff --git a/scripts/dev.sh b/scripts/dev.sh index 408ce2e..f9fc016 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -5,12 +5,13 @@ if [ "$1" = "--build" ]; then shift - printf "Building pythondiscord/snekbox-venv:dev..." + printf "Building ghcr.io/python-discord/snekbox-venv:dev..." docker build \ - -t pythondiscord/snekbox-venv:dev \ - -f docker/venv.Dockerfile \ + -t ghcr.io/python-discord/snekbox-venv:dev \ + -f docker/Dockerfile \ --build-arg DEV=1 \ + --target venv \ -q \ . \ >/dev/null \ @@ -46,7 +47,7 @@ docker run \ --volume "${PWD}":"${PWD}" \ --workdir "${PWD}"\ --entrypoint /bin/bash \ - pythondiscord/snekbox-venv:dev \ + ghcr.io/python-discord/snekbox-venv:dev \ >/dev/null \ # Execute the given command(s) |