diff options
author | 2020-11-19 00:49:40 +0100 | |
---|---|---|
committer | 2020-11-19 00:49:40 +0100 | |
commit | 2c17c3d6e752494573b77c71abf1cd8b461a7a81 (patch) | |
tree | ad9f2aad5cda66e7cc36f8e219f4a929c66510f4 | |
parent | Merge pull request #78 - Update to Python 3.9 (diff) | |
parent | Fix Dockerfile paths & revert error in Dockerfile (diff) |
Merge pull request #79 from python-discord/sebastiaan/backend/migrate-ci-to-github-actions
Migrate to GitHub Actions and GitHub Container Registry
-rw-r--r-- | .github/workflows/lint-test-build-push.yaml | 201 | ||||
-rw-r--r-- | Dockerfile (renamed from docker/base.Dockerfile) | 31 | ||||
-rw-r--r-- | Pipfile | 22 | ||||
-rw-r--r-- | Pipfile.lock | 96 | ||||
-rw-r--r-- | README.md | 20 | ||||
-rw-r--r-- | azure-pipelines.yml | 20 | ||||
-rw-r--r-- | ci/build.yml | 51 | ||||
-rw-r--r-- | ci/lint-test.yml | 41 | ||||
-rw-r--r-- | ci/push.yml | 39 | ||||
-rw-r--r-- | ci/setup.yml | 23 | ||||
-rw-r--r-- | docker-compose.yml | 4 | ||||
-rw-r--r-- | docker/Dockerfile | 7 | ||||
-rw-r--r-- | docker/venv.Dockerfile | 20 | ||||
-rwxr-xr-x | scripts/check_dockerfiles.sh | 127 | ||||
-rwxr-xr-x | scripts/dev.sh | 9 |
15 files changed, 299 insertions, 412 deletions
diff --git a/.github/workflows/lint-test-build-push.yaml b/.github/workflows/lint-test-build-push.yaml new file mode 100644 index 0000000..62691ab --- /dev/null +++ b/.github/workflows/lint-test-build-push.yaml @@ -0,0 +1,201 @@ +name: Lint, Test, Build, Push + +on: + push: + branches: + - master + pull_request: + + +jobs: + lint-test-build-push: + runs-on: ubuntu-latest + env: + # Determine whether or not we should build the + # final production image and push it to GHCR. + production_build: ${{ github.event_name != 'pull_request' && + github.ref == 'refs/heads/master' }} + + steps: + # Create a short SHA-tag to tag built images + - name: Create SHA Container Tag + id: sha_tag + run: | + tag=$(cut -c 1-7 <<< $GITHUB_SHA) + echo "::set-output name=tag::$tag" + + - name: Checkout code + uses: actions/checkout@v2 + + # The current version (v2) of Docker's build-push action uses + # buildx, which comes with BuildKit features that help us speed + # up our builds using additional cache features. Buildx also + # has a lot of other features that are not as relevant to us. + # + # See https://github.com/docker/build-push-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Github Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GHCR_TOKEN }} + + # Create a local cache directory for PR builds, as the image + # we build for PRs may start to deviate from the "latest" image + # currently registered in the GHCR. For master, the best we can + # do is use the previous master build, which can be cached from + # the GHCR. + - name: Cache Image Layers + if: github.event_name == 'pull_request' + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-v0-buildx-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-v0-buildx-${{ github.ref }}- + + # Build the image we need for linting and testing using the + # `venv` target stage within our Dockerfile. We load the image + # into the runner's Docker image collection so we can run it + # later. + # + # The image includes an inline cache manifest to support caching + # from the GHCR, which means that a build can pull the layers it + # can reuse instead of building them from scratch. + - name: Build image for linting and testing + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + push: false + load: true + target: venv + cache-from: | + type=local,src=/tmp/.buildx-cache + ghcr.io/python-discord/snekbox-base:latest + ghcr.io/python-discord/snekbox-venv:latest + cache-to: type=local,dest=/tmp/.buildx-cache,mode=max + tags: ghcr.io/python-discord/snekbox-venv:${{ steps.sha_tag.outputs.tag }} + + - name: Start Container + run: >- + docker run + --tty + --detach + --name snekbox_test + --privileged + --hostname pdsnk-dev + -e PYTHONDONTWRITEBYTECODE=1 + -e PIPENV_PIPFILE='/snekbox/Pipfile' + -e ENV="${PWD}/scripts/.profile" + --volume "${PWD}":"${PWD}" + --workdir "${PWD}" + --entrypoint /bin/bash + ghcr.io/python-discord/snekbox-venv:${{ steps.sha_tag.outputs.tag }} + + - name: Install dependencies + run: "docker exec snekbox_test /bin/bash -c \ + 'pipenv install --system --deploy --dev'" + + # This runs `flake8` in the container and asks `flake8` to output + # linting errors in the format of the command for registering workflow + # error messages/annotations. This means that Github Actions will pick + # up on this output to generate nice annotations to indicate what went + # wrong where. + - name: Run linter + run: "docker exec snekbox_test /bin/bash -c 'flake8 \ + --format \"::error file=%(path)s,line=%(row)d,col=%(col)d::\ + [flake8] %(code)s: %(text)s\"'" + + # Memory limit tests would fail if this isn't disabled. + - name: Disable swap memory + run: sudo swapoff -a + + # Run unittests and generate coverage report in the container + - name: Run unit tests and generate coverage report + id: run_tests + run: | + echo '::set-output name=started::true' + cmd='coverage run -m unittest; coverage report -m' + docker exec snekbox_test /bin/bash -c "${cmd}" + + # Set-up a Python version to process the coverage reports + # Note: This step runs even if the test step failed to make + # sure we process the coverage reports. + - name: Setup python + if: always() && steps.run_tests.outputs.started == 'true' + id: python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + + # We'll only ever need a single dependency in this python + # environment and we'll only use it in the CI, so let's + # install it directly here and run it. + # + # This step will publish the coverage results to coveralls.io + # print a job link in the output. It will also register a + # step in the check suite visible in the PR with a link to + # the job. + - name: Publish coverage report to coveralls.io + if: always() && steps.run_tests.outputs.started == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pip install coveralls~=2.1 + coveralls + + # Final build stage. This is run in the same job with conditions + # in order to use the local build cache generated by buildx while + # building the `venv` image in the lint/test phase. + + # Build the final production image and push it to GHCR, tagging it + # both with the short commit SHA and 'latest'. This step should use + # the local build cache of the current run. + - name: Build final image + if: env.production_build == 'true' + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + push: true + cache-from: | + ghcr.io/python-discord/snekbox-base:latest + ghcr.io/python-discord/snekbox-venv:latest + ghcr.io/python-discord/snekbox:latest + cache-to: type=inline + tags: | + ghcr.io/python-discord/snekbox:latest + ghcr.io/python-discord/snekbox:${{ steps.sha_tag.outputs.tag }} + + # Push the base image to GHCR, with an inline cache manifest + - name: Push base image + if: env.production_build == 'true' + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + target: base + push: true + cache-from: | + ghcr.io/python-discord/snekbox-base:latest + cache-to: type=inline + tags: ghcr.io/python-discord/snekbox-base:latest + + # Push the venv image to GHCR, with an inline cache manifest + - name: Push venv image + if: env.production_build == 'true' + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + target: venv + push: true + cache-from: | + ghcr.io/python-discord/snekbox-base:latest + ghcr.io/python-discord/snekbox-venv:latest + cache-to: type=inline + tags: ghcr.io/python-discord/snekbox-venv:latest diff --git a/docker/base.Dockerfile b/Dockerfile index a4a5ad1..ea05c5c 100644 --- a/docker/base.Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ RUN git clone \ WORKDIR /nsjail RUN make -FROM python:3.9-slim-buster +FROM python:3.9-slim-buster as base ENV PIP_NO_CACHE_DIR=false RUN apt-get -y update \ @@ -32,3 +32,32 @@ RUN pip install pipenv==2020.11.4 COPY --from=builder /nsjail/nsjail /usr/sbin/ RUN chmod +x /usr/sbin/nsjail + +FROM base as venv +ARG DEV + +ENV PIP_NO_CACHE_DIR=false \ + PIPENV_DONT_USE_PYENV=1 \ + PIPENV_HIDE_EMOJIS=1 \ + PIPENV_NOSPIN=1 + +COPY Pipfile Pipfile.lock /snekbox/ +WORKDIR /snekbox + +RUN if [ -n "${DEV}" ]; \ + then \ + pipenv install --deploy --system --dev; \ + else \ + pipenv install --deploy --system; \ + fi + +# At the end to avoid re-installing dependencies when only a config changes. +COPY config/ /snekbox/config + +FROM venv + +ENTRYPOINT ["gunicorn"] +CMD ["-c", "config/gunicorn.conf.py", "snekbox.api.app"] + +COPY . /snekbox +WORKDIR /snekbox @@ -41,7 +41,6 @@ flake8-tidy-imports = "~= 4.1" flake8-todo = "~= 0.7" pre-commit = "~= 2.8.2" pydocstyle = "~= 5.1" -unittest-xml-reporting = "~= 3.0" [requires] python_version = "3.9" @@ -61,26 +60,15 @@ snekbox = "gunicorn -c config/gunicorn.conf.py snekbox.api.app" devsh = "sh scripts/dev.sh" build = """ docker build \ - -t pythondiscord/snekbox:latest \ - -f docker/Dockerfile \ - . -""" -buildbase = """ - docker build \ - -t pythondiscord/snekbox-base:latest \ - -f docker/base.Dockerfile \ - . -""" -buildvenv = """ - docker build \ - -t pythondiscord/snekbox-venv:latest \ - -f docker/venv.Dockerfile \ + -t ghcr.io/python-discord/snekbox:latest \ + -f Dockerfile \ . """ builddev = """ docker build \ - -t pythondiscord/snekbox-venv:dev \ - -f docker/venv.Dockerfile \ + -t ghcr.io/python-discord/snekbox-venv:dev \ + -f Dockerfile \ + --target venv \ --build-arg DEV=1 \ . """ diff --git a/Pipfile.lock b/Pipfile.lock index ff4323a..7c49075 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "43d68017476a707dec8891ad40e2328390122567e338dd44686e1f48063d655d" + "sha256": "55406a3c3f1c64403f7f1efdc2cc215056c914a10c7da823e11ef707b53d3fff" }, "pipfile-spec": 6, "requires": { @@ -116,11 +116,11 @@ }, "lark": { "hashes": [ - "sha256:98f2c6f8e41fe601fd103476eb759ac1ad4d3dc8094633133a16cef5a32b0f65", - "sha256:eb919a7326a76af36e2e2cb5ac6c1be6723bba83945a0a33e6d533070cbaccc7" + "sha256:d755f63b8d361110dce780ca6806833b14ddb0b831f3dbcce2ddec187374ff37", + "sha256:f2c6ed79ae128a89714bbaa4a6ecb61b6eec84d1b5d63b9195ad461762f96298" ], "index": "pypi", - "version": "==0.10.1" + "version": "==0.11.1" }, "more-itertools": { "hashes": [ @@ -385,7 +385,7 @@ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.15.0" }, "soupsieve": { @@ -422,42 +422,46 @@ }, "yarl": { "hashes": [ - "sha256:03b7a44384ad60be1b7be93c2a24dc74895f8d767ea0bce15b2f6fc7695a3843", - "sha256:076157404db9db4bb3fa9db22db319bbb36d075eeab19ba018ce20ae0cacf037", - "sha256:1c05ae3d5ea4287470046a2c2754f0a4c171b84ea72c8a691f776eb1753dfb91", - "sha256:2467baf8233f7c64048df37e11879c553943ffe7f373e689711ec2807ea13805", - "sha256:2bb2e21cf062dfbe985c3cd4618bae9f25271efcad9e7be1277861247eee9839", - "sha256:311effab3b3828ab34f0e661bb57ff422f67d5c33056298bda4c12195251f8dd", - "sha256:3526cb5905907f0e42bee7ef57ae4a5f02bc27dcac27859269e2bba0caa4c2b6", - "sha256:39b1e586f34b1d2512c9b39aa3cf24c870c972d525e36edc9ee19065db4737bb", - "sha256:4bed5cd7c8e69551eb19df15295ba90e62b9a6a1149c76eb4a9bab194402a156", - "sha256:51c6d3cf7a1f1fbe134bb92f33b7affd94d6de24cd64b466eb12de52120fb8c6", - "sha256:59f78b5da34ddcffb663b772f7619e296518712e022e57fc5d9f921818e2ab7c", - "sha256:6f29115b0c330da25a04f48612d75333bca04521181a666ca0b8761005a99150", - "sha256:73d4e1e1ef5e52d526c92f07d16329e1678612c6a81dd8101fdcae11a72de15c", - "sha256:9b48d31f8d881713fd461abfe7acbb4dcfeb47cec3056aa83f2fbcd2244577f7", - "sha256:a1fd575dd058e10ad4c35065e7c3007cc74d142f622b14e168d8a273a2fa8713", - "sha256:b3dd1052afd436ba737e61f5d3bed1f43a7f9a33fc58fbe4226eb919a7006019", - "sha256:b99c25ed5c355b35d1e6dae87ac7297a4844a57dc5766b173b88b6163a36eb0d", - "sha256:c056e86bff5a0b566e0d9fab4f67e83b12ae9cbcd250d334cbe2005bbe8c96f2", - "sha256:c45b49b59a5724869899798e1bbd447ac486215269511d3b76b4c235a1b766b6", - "sha256:cd623170c729a865037828e3f99f8ebdb22a467177a539680dfc5670b74c84e2", - "sha256:d25d3311794e6c71b608d7c47651c8f65eea5ab15358a27f29330b3475e8f8e5", - "sha256:d695439c201ed340745250f9eb4dfe8d32bf1e680c16477107b8f3ce4bff4fdb", - "sha256:d77f6c9133d2aabb290a7846aaa74ec14d7b5ab35b01591fac5a70c4a8c959a2", - "sha256:d894a2442d2cd20a3b0b0dce5a353d316c57d25a2b445e03f7eac90eee27b8af", - "sha256:db643ce2b58a4bd11a82348225c53c76ecdd82bb37cf4c085e6df1b676f4038c", - "sha256:e3a0c43a26dfed955b2a06fdc4d51d2c51bc2200aff8ce8faf14e676ea8c8862", - "sha256:e77bf79ad1ccae672eab22453838382fe9029fc27c8029e84913855512a587d8", - "sha256:f2f0174cb15435957d3b751093f89aede77df59a499ab7516bbb633b77ead13a", - "sha256:f3031c78edf10315abe232254e6a36b65afe65fded41ee54ed7976d0b2cdf0da", - "sha256:f4c007156732866aa4507d619fe6f8f2748caabed4f66b276ccd97c82572620c", - "sha256:f4f27ff3dd80bc7c402def211a47291ea123d59a23f59fe18fc0e81e3e71f385", - "sha256:f57744fc61e118b5d114ae8077d8eb9df4d2d2c11e2af194e21f0c11ed9dcf6c", - "sha256:f835015a825980b65356e9520979a1564c56efea7da7d4b68a14d4a07a3a7336" - ], - "index": "pypi", - "version": "==1.6.2" + "sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e", + "sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434", + "sha256:15263c3b0b47968c1d90daa89f21fcc889bb4b1aac5555580d74565de6836366", + "sha256:2ce4c621d21326a4a5500c25031e102af589edb50c09b321049e388b3934eec3", + "sha256:31ede6e8c4329fb81c86706ba8f6bf661a924b53ba191b27aa5fcee5714d18ec", + "sha256:324ba3d3c6fee56e2e0b0d09bf5c73824b9f08234339d2b788af65e60040c959", + "sha256:329412812ecfc94a57cd37c9d547579510a9e83c516bc069470db5f75684629e", + "sha256:4736eaee5626db8d9cda9eb5282028cc834e2aeb194e0d8b50217d707e98bb5c", + "sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6", + "sha256:4c5bcfc3ed226bf6419f7a33982fb4b8ec2e45785a0561eb99274ebbf09fdd6a", + "sha256:547f7665ad50fa8563150ed079f8e805e63dd85def6674c97efd78eed6c224a6", + "sha256:5b883e458058f8d6099e4420f0cc2567989032b5f34b271c0827de9f1079a424", + "sha256:63f90b20ca654b3ecc7a8d62c03ffa46999595f0167d6450fa8383bab252987e", + "sha256:68dc568889b1c13f1e4745c96b931cc94fdd0defe92a72c2b8ce01091b22e35f", + "sha256:69ee97c71fee1f63d04c945f56d5d726483c4762845400a6795a3b75d56b6c50", + "sha256:6d6283d8e0631b617edf0fd726353cb76630b83a089a40933043894e7f6721e2", + "sha256:72a660bdd24497e3e84f5519e57a9ee9220b6f3ac4d45056961bf22838ce20cc", + "sha256:73494d5b71099ae8cb8754f1df131c11d433b387efab7b51849e7e1e851f07a4", + "sha256:7356644cbed76119d0b6bd32ffba704d30d747e0c217109d7979a7bc36c4d970", + "sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10", + "sha256:8aa3decd5e0e852dc68335abf5478a518b41bf2ab2f330fe44916399efedfae0", + "sha256:97b5bdc450d63c3ba30a127d018b866ea94e65655efaf889ebeabc20f7d12406", + "sha256:9ede61b0854e267fd565e7527e2f2eb3ef8858b301319be0604177690e1a3896", + "sha256:b2e9a456c121e26d13c29251f8267541bd75e6a1ccf9e859179701c36a078643", + "sha256:b5dfc9a40c198334f4f3f55880ecf910adebdcb2a0b9a9c23c9345faa9185721", + "sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478", + "sha256:c49ff66d479d38ab863c50f7bb27dee97c6627c5fe60697de15529da9c3de724", + "sha256:ce3beb46a72d9f2190f9e1027886bfc513702d748047b548b05dab7dfb584d2e", + "sha256:d26608cf178efb8faa5ff0f2d2e77c208f471c5a3709e577a7b3fd0445703ac8", + "sha256:d597767fcd2c3dc49d6eea360c458b65643d1e4dbed91361cf5e36e53c1f8c96", + "sha256:d5c32c82990e4ac4d8150fd7652b972216b204de4e83a122546dce571c1bdf25", + "sha256:d8d07d102f17b68966e2de0e07bfd6e139c7c02ef06d3a0f8d2f0f055e13bb76", + "sha256:e46fba844f4895b36f4c398c5af062a9808d1f26b2999c58909517384d5deda2", + "sha256:e6b5460dc5ad42ad2b36cca524491dfcaffbfd9c8df50508bddc354e787b8dc2", + "sha256:f040bcc6725c821a4c0665f3aa96a4d0805a7aaf2caf266d256b8ed71b9f041c", + "sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a", + "sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71" + ], + "index": "pypi", + "version": "==1.6.3" } }, "develop": { @@ -698,7 +702,7 @@ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.15.0" }, "snowballstemmer": { @@ -716,14 +720,6 @@ "index": "pypi", "version": "==0.10.2" }, - "unittest-xml-reporting": { - "hashes": [ - "sha256:7bf515ea8cb244255a25100cd29db611a73f8d3d0aaf672ed3266307e14cc1ca", - "sha256:984cebba69e889401bfe3adb9088ca376b3a1f923f0590d005126c1bffd1a695" - ], - "index": "pypi", - "version": "==3.0.4" - }, "virtualenv": { "hashes": [ "sha256:b0011228208944ce71052987437d3843e05690b2f23d1c7da4263fde104c97a2", @@ -1,4 +1,4 @@ -[![Build Status][1]][2] +[![Build Status][1]][2] [![Coverage Status][3]][4] # snekbox @@ -33,10 +33,10 @@ See [`snekapi.py`] and [`resources`] for API documentation. ## Running snekbox -A Docker image is available on [Docker Hub]. A container can be started with the following command, which will also pull the image if it doesn't currently exist locally: +A Docker image is available in the [GitHub Container Registry]. A container can be started with the following command, which will also pull the image if it doesn't currently exist locally: ``` -docker run --ipc=none --privileged -p 8060:8060 pythondiscord/snekbox +docker run --ipc=none --privileged -p 8060:8060 ghcr.io/python-discord/snekbox ``` To run it in the background, use the `-d` option. See the documentation on [`docker run`] for more information. @@ -47,7 +47,7 @@ The above command will make the API accessible on the host via `http://localhost ### Initial Setup -A Python 3.8 interpreter and the [pipenv] package are required. Once those requirements are satisfied, install the project's dependencies: +A Python 3.9 interpreter and the [pipenv] package are required. Once those requirements are satisfied, install the project's dependencies: ``` pipenv sync --dev @@ -67,11 +67,9 @@ pipenv run lint ### Running snekbox -The Docker images can be built with: +The Docker image can be built with: ``` -pipenv run buildbase -pipenv run buildvenv pipenv run build ``` @@ -142,8 +140,10 @@ nsjpy "print('hello world!')" The alias can be found in `./scripts/.profile`, which is automatically added when the shell is launched in the container. -[1]: https://dev.azure.com/python-discord/Python%20Discord/_apis/build/status/Snekbox?branchName=master -[2]: https://dev.azure.com/python-discord/Python%20Discord/_build/latest?definitionId=13&branchName=master +[1]: https://github.com/python-discord/snekbox/workflows/Lint,%20Test,%20Build,%20Push/badge.svg?branch=master +[2]: https://github.com/python-discord/snekbox/actions?query=workflow%3A%22Lint%2C+Test%2C+Build%2C+Push%22+branch%3Amaster +[3]: https://coveralls.io/repos/github/python-discord/snekbox/badge.svg?branch=master +[4]: https://coveralls.io/github/python-discord/snekbox?branch=master [`snekbox.cfg`]: config/snekbox.cfg [`snekapi.py`]: snekbox/api/snekapi.py [`resources`]: snekbox/api/resources @@ -151,5 +151,5 @@ The alias can be found in `./scripts/.profile`, which is automatically added whe [nsjail]: https://github.com/google/nsjail [falcon]: https://falconframework.org/ [gunicorn]: https://gunicorn.org/ -[docker hub]: https://hub.docker.com/r/pythondiscord/snekbox +[GitHub Container Registry]: https://github.com/orgs/python-discord/packages/container/package/snekbox [pipenv]: https://docs.pipenv.org/en/latest/ diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 573e3cc..0000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,20 +0,0 @@ -# https://aka.ms/yaml - -jobs: - - job: build_lint_test_push - displayName: 'Build, Lint, Test, & Push' - - pool: - vmImage: 'ubuntu-18.04' - - variables: - BASE_CHANGED: 'True' - VENV_CHANGED: 'True' - BASE_PULL: 'False' - VENV_PULL: 'False' - - steps: - - template: ci/build.yml - - template: ci/setup.yml - - template: ci/lint-test.yml - - template: ci/push.yml diff --git a/ci/build.yml b/ci/build.yml deleted file mode 100644 index 7d51709..0000000 --- a/ci/build.yml +++ /dev/null @@ -1,51 +0,0 @@ -steps: - - task: ShellScript@2 - displayName: 'Check If Images Need to Be Built' - inputs: - scriptPath: scripts/check_dockerfiles.sh - disableAutoCwd: true - - # Without a login, the following Docker build steps wouldn't add image tags. - - task: Docker@1 - displayName: 'Log into Docker Hub' - inputs: - command: login - containerregistrytype: 'Container Registry' - dockerRegistryEndpoint: 'DockerHub' - - # Building the venv depends on this base image. Build the base if it can't - # pulled from Docker Hub, which will be the case if the base Dockerfile has - # has had changes. - - script: | - docker build \ - -f docker/base.Dockerfile \ - -t pythondiscord/snekbox-base:latest \ - . - displayName: 'Build Base Image' - condition: > - and( - succeeded(), - or( - eq(variables.BASE_CHANGED, 'True'), - and( - eq(variables.VENV_CHANGED, 'True'), - eq(variables.BASE_PULL, 'False') - ) - ) - ) - - # Build the venv image if it's had changes or it can't be pulled. - - script: | - docker build \ - -f docker/venv.Dockerfile \ - -t pythondiscord/snekbox-venv:latest \ - . - displayName: 'Build Virtual Environment Image' - condition: > - and( - succeeded(), - or( - eq(variables.VENV_CHANGED, 'True'), - eq(variables.VENV_PULL, 'False') - ) - ) diff --git a/ci/lint-test.yml b/ci/lint-test.yml deleted file mode 100644 index 2d70f6e..0000000 --- a/ci/lint-test.yml +++ /dev/null @@ -1,41 +0,0 @@ -steps: - - script: | - docker exec snekbox_test /bin/bash -c \ - 'flake8; flake8 --format junit-xml --output-file test-lint.xml' - displayName: 'Run Linter' - - - task: PublishTestResults@2 - displayName: 'Publish Lint Results' - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-lint.xml' - testRunTitle: 'Lint Results' - - # Memory limit tests would fail if this isn't disabled. - - script: sudo swapoff -a - displayName: 'Disable Swap Memory' - - - script: | - docker exec snekbox_test /bin/bash -c \ - 'coverage run -m xmlrunner' - displayName: 'Run Unit Tests' - - - task: PublishTestResults@2 - displayName: 'Publish Test Results' - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/TEST-*.xml' - testRunTitle: 'Test Results' - - # Run report too because the XML report doesn't output to stdout. - - script: | - docker exec snekbox_test /bin/bash -c \ - 'coverage report && coverage xml' - displayName: 'Generate Coverage Report' - - - task: PublishCodeCoverageResults@1 - displayName: 'Publish Coverage Results' - condition: succeededOrFailed() - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '**/coverage.xml' diff --git a/ci/push.yml b/ci/push.yml deleted file mode 100644 index 9449df0..0000000 --- a/ci/push.yml +++ /dev/null @@ -1,39 +0,0 @@ -steps: - # Always build this image unless it's for a pull request. - - script: | - docker build \ - -f docker/Dockerfile \ - -t pythondiscord/snekbox:latest \ - . - displayName: 'Build Final Image' - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - - # Push images only after they've all successfully been built. - - script: docker push pythondiscord/snekbox-base:latest - displayName: 'Push Base Image' - condition: > - and( - succeeded(), - ne(variables['Build.Reason'], 'PullRequest'), - ne(variables.BASE_PULL, 'True'), - or( - eq(variables.BASE_CHANGED, 'True'), - eq(variables.VENV_CHANGED, 'True') - ) - ) - - - script: docker push pythondiscord/snekbox-venv:latest - displayName: 'Push Virtual Environment Image' - condition: > - and( - succeeded(), - ne(variables['Build.Reason'], 'PullRequest'), - or( - eq(variables.BASE_CHANGED, 'True'), - eq(variables.VENV_CHANGED, 'True') - ) - ) - - - script: docker push pythondiscord/snekbox:latest - displayName: 'Push Final Image' - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) diff --git a/ci/setup.yml b/ci/setup.yml deleted file mode 100644 index 1c66aa7..0000000 --- a/ci/setup.yml +++ /dev/null @@ -1,23 +0,0 @@ -steps: - # The linter and all tests run inside this container. - # The venv image will be pulled if it doesn't exist locally. - - script: | - docker run \ - --tty \ - --detach \ - --name snekbox_test \ - --privileged \ - --hostname pdsnk-dev \ - -e PYTHONDONTWRITEBYTECODE=1 \ - -e PIPENV_PIPFILE="/snekbox/Pipfile" \ - -e ENV="${PWD}/scripts/.profile" \ - --volume "${PWD}":"${PWD}" \ - --workdir "${PWD}"\ - --entrypoint /bin/bash \ - pythondiscord/snekbox-venv:latest - displayName: 'Start Container' - - - script: | - docker exec snekbox_test /bin/bash -c \ - 'pipenv install --system --deploy --dev' - displayName: 'Install Development Dependencies' diff --git a/docker-compose.yml b/docker-compose.yml index aec17d3..3062af3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,11 +3,11 @@ services: snekbox: container_name: snekbox privileged: true - image: pythondiscord/snekbox:latest + image: ghcr.io/python-discord/snekbox:latest ports: - 8060:8060 init: true ipc: none build: context: . - dockerfile: docker/Dockerfile + dockerfile: Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index b6c2f77..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM pythondiscord/snekbox-venv:latest - -ENTRYPOINT ["gunicorn"] -CMD ["-c", "config/gunicorn.conf.py", "snekbox.api.app"] - -COPY . /snekbox -WORKDIR /snekbox diff --git a/docker/venv.Dockerfile b/docker/venv.Dockerfile deleted file mode 100644 index 5c0fcfc..0000000 --- a/docker/venv.Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM pythondiscord/snekbox-base:latest - -ARG DEV -ENV PIP_NO_CACHE_DIR=false \ - PIPENV_DONT_USE_PYENV=1 \ - PIPENV_HIDE_EMOJIS=1 \ - PIPENV_NOSPIN=1 - -COPY Pipfile Pipfile.lock /snekbox/ -WORKDIR /snekbox - -RUN if [ -n "${DEV}" ]; \ - then \ - pipenv install --deploy --system --dev; \ - else \ - pipenv install --deploy --system; \ - fi - -# At the end to avoid re-installing dependencies when only a config changes. -COPY config/ /snekbox/config diff --git a/scripts/check_dockerfiles.sh b/scripts/check_dockerfiles.sh deleted file mode 100755 index 88cb7cc..0000000 --- a/scripts/check_dockerfiles.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -shopt -s inherit_errexit -exec 3>&1 # New file descriptor to stdout - -BASE_URL="https://dev.azure.com/\ -python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?\ -queryOrder=finishTimeDescending&\ -resultFilter=succeeded&\ -\$top=1&\ -repositoryType=${BUILD_REPOSITORY_PROVIDER}&\ -repositoryId=${BUILD_REPOSITORY_NAME}&\ -api-version=5.0" - -declare -A build_cache - -get_build() { - local branch="${1:?"get_build: argument 1 'branch' is unset"}" - - # Attempt to use cached value - if [[ -v build_cache["${branch}"] ]]; then - printf '%s\n' "Retrieving build for ${branch} from cache." >&3 - printf '%s' "${build_cache[$branch]}" - return 0 - fi - - local url="${BASE_URL}&branchName=${branch}" - - printf '%s\n' "Retrieving the latest successful build using ${url}" >&3 - - local response - response="$(curl -sSL "${url}")" - - if [[ -z "${response}" ]] \ - || ! count="$(printf '%s' "${response}" | jq -re '.count')" \ - || (( "${count}" < 1 )) - then - return 1 - else - # Cache the response - build_cache["${branch}"]="${response}" - printf '%s' "${response}" - fi -} - -can_pull() { - local image="${1:?"can_pull: argument 1 'image' is unset"}" - - local master_commit - if master_commit="$( - get_build "refs/heads/master" \ - | jq -re '.value[0].sourceVersion' - )" \ - && git diff --quiet "${master_commit}" -- "${@:2}" - then - printf \ - '%s\n' \ - "Can pull ${image} image from Docker Hub; no changes since master." - - printf '%s\n' "##vso[task.setvariable variable=${image^^}_PULL]True" - else - printf \ - '%s\n' \ - "Cannot pull ${image} image from Docker Hub due to detected " \ - "changes; the ${image} image will be built." - - return 1 - fi -} - -# Get the previous commit -if [[ "${BUILD_REASON}" = "PullRequest" ]]; then - if ! prev_commit="$( - get_build "${BUILD_SOURCEBRANCH}" \ - | jq -re '.value[0].triggerInfo."pr.sourceSha"' - )" - then - echo \ - "Could not retrieve the previous build's commit." \ - "Falling back to the head of the target branch." - - prev_commit="origin/${SYSTEM_PULLREQUEST_TARGETBRANCH}" - fi -elif ! prev_commit="$( - get_build "${BUILD_SOURCEBRANCH}" \ - | jq -re '.value[0].sourceVersion' - )" -then - echo \ - "No previous build was found." \ - "Either the previous build is too old and was deleted" \ - "or the branch was empty before this build." \ - "All images will be built." - exit 0 -fi - -# Compare diffs -head="$(git rev-parse HEAD)" -printf '%s\n' "Comparing HEAD (${head}) against ${prev_commit}." - -if git diff --quiet "${prev_commit}" -- docker/base.Dockerfile; then - echo "No changes detected in docker/base.Dockerfile." - echo "##vso[task.setvariable variable=BASE_CHANGED]False" -else - # Always rebuild the venv if the base changes. - echo "Changes detected in docker/base.Dockerfile; all images will be built." - exit 0 -fi - -if git diff --quiet "${prev_commit}" -- docker/venv.Dockerfile Pipfile*; then - echo "No changes detected in docker/venv.Dockerfile or the Pipfiles." - echo "##vso[task.setvariable variable=VENV_CHANGED]False" - - if ! can_pull venv docker/venv.Dockerfile Pipfile*; then - # Venv image can't be pulled so it needs to be built. - # Therefore, the base image is needed too. - can_pull base docker/base.Dockerfile || true - fi -else - echo \ - "Changes detected in docker/venv.Dockerfile or the Pipfiles;" \ - "the venv image will be built." - - # Though base image hasn't changed, it's still needed to build the venv. - can_pull base docker/base.Dockerfile || true -fi diff --git a/scripts/dev.sh b/scripts/dev.sh index 408ce2e..f9cd28e 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -5,12 +5,13 @@ if [ "$1" = "--build" ]; then shift - printf "Building pythondiscord/snekbox-venv:dev..." + printf "Building ghcr.io/python-discord/snekbox-venv:dev..." docker build \ - -t pythondiscord/snekbox-venv:dev \ - -f docker/venv.Dockerfile \ + -t ghcr.io/python-discord/snekbox-venv:dev \ + -f Dockerfile \ --build-arg DEV=1 \ + --target venv \ -q \ . \ >/dev/null \ @@ -46,7 +47,7 @@ docker run \ --volume "${PWD}":"${PWD}" \ --workdir "${PWD}"\ --entrypoint /bin/bash \ - pythondiscord/snekbox-venv:dev \ + ghcr.io/python-discord/snekbox-venv:dev \ >/dev/null \ # Execute the given command(s) |