diff options
author | 2019-09-20 22:39:28 -0700 | |
---|---|---|
committer | 2019-09-20 22:39:28 -0700 | |
commit | 00b7c8e52b7fdf55d886dbf2fff4e6144882640b (patch) | |
tree | c3a76bbaeb4c69f35b6d32522dee1bd55ad3e452 | |
parent | Login with v1 API (diff) | |
parent | CI: use DockerHub container registry & remove login tasks (diff) |
Merge branch 'ci' into research
-rw-r--r-- | .flake8 | 5 | ||||
-rw-r--r-- | .github/.github/FUNDING.yml | 2 | ||||
-rw-r--r-- | .github/FUNDING.yml | 2 | ||||
-rw-r--r-- | .pre-commit-config.yaml | 25 | ||||
-rw-r--r-- | CONTRIBUTING.md | 3 | ||||
-rw-r--r-- | Pipfile | 72 | ||||
-rw-r--r-- | Pipfile.lock | 190 | ||||
-rw-r--r-- | README.md | 157 | ||||
-rw-r--r-- | azure-pipelines.yml | 224 | ||||
-rw-r--r-- | docker/base.Dockerfile | 29 | ||||
-rw-r--r-- | scripts/.profile | 4 | ||||
-rwxr-xr-x | scripts/check_dockerfiles.sh | 10 | ||||
-rwxr-xr-x | scripts/dev.sh | 9 |
13 files changed, 473 insertions, 259 deletions
@@ -1,16 +1,17 @@ [flake8] max-line-length=100 application_import_names=snekbox +docstring-convention=all ignore= P102,B311,W503,E226,S311, # Missing Docstrings - D100,D104,D107, + D100,D104,D105,D107, # Docstring Whitespace D203,D212,D214,D215, # Docstring Quotes D301,D302, # Docstring Content - D400,D401,D402,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414 + D400,D401,D402,D404,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414,D416,D417 exclude= __pycache__,.cache, venv,.venv diff --git a/.github/.github/FUNDING.yml b/.github/.github/FUNDING.yml new file mode 100644 index 0000000..6d9919e --- /dev/null +++ b/.github/.github/FUNDING.yml @@ -0,0 +1,2 @@ +patreon: python_discord +custom: https://www.redbubble.com/people/pythondiscord diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..6d9919e --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,2 @@ +patreon: python_discord +custom: https://www.redbubble.com/people/pythondiscord diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d2737fe..b5f3715 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,15 @@ repos: -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.2.3 hooks: - - id: flake8 - additional_dependencies: [ - flake8-docstrings, - flake8-bugbear, - flake8-import-order, - flake8-tidy-imports, - flake8-todo, - flake8-string-format, - flake8-formatter-junit-xml, - flake8-quotes - ] - + - id: flake8 + additional_dependencies: [ + pydocstyle ~= 4.0, + "flake8-docstrings >= 1.3.1, == 1.*", + flake8-bugbear ~= 19.3, + flake8-import-order ~= 0.18.1, + flake8-tidy-imports ~= 2.0, + flake8-todo ~= 0.7, + flake8-string-format ~= 0.2.3, + flake8-quotes ~= 2.1 + ] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 429fbf1..73ec91b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,6 +9,7 @@ Note that contributions may be rejected on the basis of a contributor failing to 1. **No force-pushes** or modifying the Git history in any way. 2. If you have direct access to the repository, **create a branch for your changes** and create a pull request for that branch. If not, create a branch on a fork of the repository and create a pull request from there. * It's common practice for a repository to reject direct pushes to `master`, so make branching a habit! + * If PRing from your own fork, **ensure that "Allow edits from maintainers" is checked**. This gives permission for maintainers to commit changes directly to your fork, speeding up the review process. 3. **Adhere to the prevailing code style**, which we enforce using [flake8](http://flake8.pycqa.org/en/latest/index.html). * Run `flake8` against your code **before** you push it. Your commit will be rejected by the build server if it fails to lint. * [Git Hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) are a powerful tool that can be a daunting to set up. Fortunately, [`pre-commit`](https://github.com/pre-commit/pre-commit) abstracts this process away from you and is provided as a dev dependency for this project. Run `pipenv run precommit` when setting up the project and you'll never have to worry about breaking the build for linting errors. @@ -100,6 +101,8 @@ Github [has introduced a new PR feature](https://github.blog/2019-02-14-introduc This feature should be utilized in place of the traditional method of prepending `[WIP]` to the PR title. +As stated earlier, **ensure that "Allow edits from maintainers" is checked**. This gives permission for maintainers to commit changes directly to your fork, speeding up the review process. + ## Footnotes This document was inspired by the [Glowstone contribution guidelines](https://github.com/GlowstoneMC/Glowstone/blob/dev/docs/CONTRIBUTING.md). @@ -4,23 +4,24 @@ verify_ssl = true name = "pypi" [packages] -falcon = "*" -gunicorn = "*" -jsonschema = "*" +falcon = "~= 2.0.0" +gunicorn = "~= 19.9" +jsonschema = "~= 3.0" [dev-packages] -coverage = "*" -pre-commit = "*" -flake8 = "*" -flake8-docstrings = "*" -flake8-bugbear = "*" -flake8-import-order = "*" -flake8-tidy-imports = "*" -flake8-todo = "*" -flake8-string-format = "*" -flake8-formatter-junit-xml = "*" -flake8-quotes = "*" -unittest-xml-reporting = "*" +coverage = ">= 4.4.2, == 4.*" +pre-commit = "~= 1.18" +pydocstyle = "~= 4.0" +flake8 = "~= 3.7.8" +flake8-docstrings = "~=1.4" +flake8-bugbear = "~= 19.3" +flake8-import-order = "~= 0.18.1" +flake8-tidy-imports = "~= 2.0" +flake8-todo = "~= 0.7" +flake8-string-format = "~= 0.2.3" +flake8-formatter-junit-xml = "~= 0.0.6" +flake8-quotes = "~= 2.1" +unittest-xml-reporting = ">= 2.5.1, == 2.*" [requires] python_version = "3.7" @@ -28,8 +29,14 @@ python_version = "3.7" [scripts] lint = "flake8" precommit = "pre-commit install" -test = "scripts/dev.sh -c 'pipenv run coverage run -m unittest'" -report = "coverage html" +test = "sh scripts/dev.sh -c 'pipenv run coverage run -m unittest'" +testb = """ + sh scripts/dev.sh \ + --build \ + --clean \ + -c 'pipenv run coverage run -m unittest' +""" +report = "coverage report" snekbox = """ gunicorn \ -w 2 \ @@ -39,8 +46,29 @@ snekbox = """ --access-logfile - \ snekbox.api.app """ -devsh = "scripts/dev.sh" -buildbox = "docker build -t pythondiscord/snekbox:latest -f docker/Dockerfile ." -pushbox = "docker push pythondiscord/snekbox:latest" -buildboxbase = "docker build -t pythondiscord/snekbox-base:latest -f docker/base.Dockerfile ." -pushboxbase = "docker push pythondiscord/snekbox-base:latest" +devsh = "sh scripts/dev.sh" +build = """ + docker build \ + -t pythondiscord/snekbox:latest \ + -f docker/Dockerfile \ + . +""" +buildbase = """ + docker build \ + -t pythondiscord/snekbox-base:latest \ + -f docker/base.Dockerfile \ + . +""" +buildvenv = """ + docker build \ + -t pythondiscord/snekbox-venv:latest \ + -f docker/venv.Dockerfile \ + . +""" +builddev = """ + docker build \ + -t pythondiscord/snekbox-venv:dev \ + -f docker/venv.Dockerfile \ + --build-arg DEV=1 \ + . +""" diff --git a/Pipfile.lock b/Pipfile.lock index 4f6bef8..b541730 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "421cdae80970f990af48506e38464e4e2b99e20291070943027b86bd7ca29c5b" + "sha256": "fa4ef446ed6cd8618914fd2b509e1d437f71cd70140bba4b5b95a4eaf6e53933" }, "pipfile-spec": 6, "requires": { @@ -53,17 +53,17 @@ }, "jsonschema": { "hashes": [ - "sha256:0c0a81564f181de3212efa2d17de1910f8732fa1b71c42266d983cd74304e20d", - "sha256:a5f6559964a3851f59040d3b961de5e68e70971afb88ba519d27e6a039efff1a" + "sha256:5f9c0a719ca2ce14c5de2fd350a64fd2d13e8539db29836a86adc990bb1a068f", + "sha256:8d4a2b7b6c2237e0199c8ea1a6d3e05bf118e289ae2b9d7ba444182a2959560d" ], "index": "pypi", - "version": "==3.0.1" + "version": "==3.0.2" }, "pyrsistent": { "hashes": [ - "sha256:16692ee739d42cf5e39cef8d27649a8c1fdb7aa99887098f1460057c5eb75c3a" + "sha256:34b47fa169d6006b32e99d4b3c4031f155e6e68ebcc107d6454852e8e0ee6533" ], - "version": "==0.15.2" + "version": "==0.15.4" }, "six": { "hashes": [ @@ -90,47 +90,48 @@ }, "cfgv": { "hashes": [ - "sha256:32edbe09de6f4521224b87822103a8c16a614d31a894735f7a5b3bcf0eb3c37e", - "sha256:3bd31385cd2bebddbba8012200aaf15aa208539f1b33973759b4d02fc2148da5" + "sha256:edb387943b665bf9c434f717bf630fa78aecd53d5900d2e05da6ad6048553144", + "sha256:fbd93c9ab0a523bf7daec408f3be2ed99a980e20b2d19b50fc184ca6b820d289" ], - "version": "==2.0.0" + "version": "==2.0.1" }, "coverage": { "hashes": [ - "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", - "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", - "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", - "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", - "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", - "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", - "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", - "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", - "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", - "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", - "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", - "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", - "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", - "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", - "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", - "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", - "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", - "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", - "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", - "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", - "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", - "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", - "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", - "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", - "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", - "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", - "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", - "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", - "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", - "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", - "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" + "sha256:08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6", + "sha256:0be0f1ed45fc0c185cfd4ecc19a1d6532d72f86a2bac9de7e24541febad72650", + "sha256:141f08ed3c4b1847015e2cd62ec06d35e67a3ac185c26f7635f4406b90afa9c5", + "sha256:19e4df788a0581238e9390c85a7a09af39c7b539b29f25c89209e6c3e371270d", + "sha256:23cc09ed395b03424d1ae30dcc292615c1372bfba7141eb85e11e50efaa6b351", + "sha256:245388cda02af78276b479f299bbf3783ef0a6a6273037d7c60dc73b8d8d7755", + "sha256:331cb5115673a20fb131dadd22f5bcaf7677ef758741312bee4937d71a14b2ef", + "sha256:386e2e4090f0bc5df274e720105c342263423e77ee8826002dcffe0c9533dbca", + "sha256:3a794ce50daee01c74a494919d5ebdc23d58873747fa0e288318728533a3e1ca", + "sha256:60851187677b24c6085248f0a0b9b98d49cba7ecc7ec60ba6b9d2e5574ac1ee9", + "sha256:63a9a5fc43b58735f65ed63d2cf43508f462dc49857da70b8980ad78d41d52fc", + "sha256:6b62544bb68106e3f00b21c8930e83e584fdca005d4fffd29bb39fb3ffa03cb5", + "sha256:6ba744056423ef8d450cf627289166da65903885272055fb4b5e113137cfa14f", + "sha256:7494b0b0274c5072bddbfd5b4a6c6f18fbbe1ab1d22a41e99cd2d00c8f96ecfe", + "sha256:826f32b9547c8091679ff292a82aca9c7b9650f9fda3e2ca6bf2ac905b7ce888", + "sha256:93715dffbcd0678057f947f496484e906bf9509f5c1c38fc9ba3922893cda5f5", + "sha256:9a334d6c83dfeadae576b4d633a71620d40d1c379129d587faa42ee3e2a85cce", + "sha256:af7ed8a8aa6957aac47b4268631fa1df984643f07ef00acd374e456364b373f5", + "sha256:bf0a7aed7f5521c7ca67febd57db473af4762b9622254291fbcbb8cd0ba5e33e", + "sha256:bf1ef9eb901113a9805287e090452c05547578eaab1b62e4ad456fcc049a9b7e", + "sha256:c0afd27bc0e307a1ffc04ca5ec010a290e49e3afbe841c5cafc5c5a80ecd81c9", + "sha256:dd579709a87092c6dbee09d1b7cfa81831040705ffa12a1b248935274aee0437", + "sha256:df6712284b2e44a065097846488f66840445eb987eb81b3cc6e4149e7b6982e1", + "sha256:e07d9f1a23e9e93ab5c62902833bf3e4b1f65502927379148b6622686223125c", + "sha256:e2ede7c1d45e65e209d6093b762e98e8318ddeff95317d07a27a2140b80cfd24", + "sha256:e4ef9c164eb55123c62411f5936b5c2e521b12356037b6e1c2617cef45523d47", + "sha256:eca2b7343524e7ba246cab8ff00cab47a2d6d54ada3b02772e908a45675722e2", + "sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28", + "sha256:ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c", + "sha256:efc89291bd5a08855829a3c522df16d856455297cf35ae827a37edac45f466a7", + "sha256:fa964bae817babece5aa2e8c1af841bebb6d0b9add8e637548809d040443fee0", + "sha256:ff37757e068ae606659c28c3bd0d923f9d29a85de79bf25b2b34b148473b5025" ], "index": "pypi", - "version": "==4.5.3" + "version": "==4.5.4" }, "entrypoints": { "hashes": [ @@ -141,27 +142,27 @@ }, "flake8": { "hashes": [ - "sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661", - "sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8" + "sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548", + "sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696" ], "index": "pypi", - "version": "==3.7.7" + "version": "==3.7.8" }, "flake8-bugbear": { "hashes": [ - "sha256:5070774b668be92c4312e5ca82748ddf4ecaa7a24ff062662681bb745c7896eb", - "sha256:fef9c9826d14ec23187ae1edeb3c6513c4e46bf0e70d86bac38f7d9aabae113d" + "sha256:d8c466ea79d5020cb20bf9f11cf349026e09517a42264f313d3f6fddb83e0571", + "sha256:ded4d282778969b5ab5530ceba7aa1a9f1b86fa7618fc96a19a1d512331640f8" ], "index": "pypi", - "version": "==19.3.0" + "version": "==19.8.0" }, "flake8-docstrings": { "hashes": [ - "sha256:4e0ce1476b64e6291520e5570cf12b05016dd4e8ae454b8a8a9a48bc5f84e1cd", - "sha256:8436396b5ecad51a122a2c99ba26e5b4e623bf6e913b0fea0cb6c2c4050f91eb" + "sha256:1666dd069c9c457ee57e80af3c1a6b37b00cc1801c6fde88e455131bb2e186cd", + "sha256:9c0db5a79a1affd70fdf53b8765c8a26bf968e59e0252d7f2fc546b41c0cda06" ], "index": "pypi", - "version": "==1.3.0" + "version": "==1.4.0" }, "flake8-formatter-junit-xml": { "hashes": [ @@ -179,19 +180,12 @@ "index": "pypi", "version": "==0.18.1" }, - "flake8-polyfill": { - "hashes": [ - "sha256:12be6a34ee3ab795b19ca73505e7b55826d5f6ad7230d31b18e106400169b9e9", - "sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda" - ], - "version": "==1.0.2" - }, "flake8-quotes": { "hashes": [ - "sha256:10c9af6b472d4302a8e721c5260856c3f985c5c082b04841aefd2f808ac02038" + "sha256:5dbaf668887873f28346fb87943d6da2e4b9f77ce9f2169cff21764a0a4934ed" ], "index": "pypi", - "version": "==2.0.1" + "version": "==2.1.0" }, "flake8-string-format": { "hashes": [ @@ -218,17 +212,17 @@ }, "identify": { "hashes": [ - "sha256:0a11379b46d06529795442742a043dc2fa14cd8c995ae81d1febbc5f1c014c87", - "sha256:43a5d24ffdb07bc7e21faf68b08e9f526a1f41f0056073f480291539ef961dfd" + "sha256:4f1fe9a59df4e80fcb0213086fcf502bc1765a01ea4fe8be48da3b65afd2a017", + "sha256:d8919589bd2a5f99c66302fec0ef9027b12ae150b0b0213999ad3f695fc7296e" ], - "version": "==1.4.5" + "version": "==1.4.7" }, "importlib-metadata": { "hashes": [ - "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7", - "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db" + "sha256:9ff1b1c5a354142de080b8a4e9803e5d0d59283c93aed808617c787d16768375", + "sha256:b7143592e374e50584564794fcb8aaf00a23025f9db866627f89a21491847a8d" ], - "version": "==0.18" + "version": "==0.20" }, "junit-xml": { "hashes": [ @@ -243,6 +237,13 @@ ], "version": "==0.6.1" }, + "more-itertools": { + "hashes": [ + "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", + "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" + ], + "version": "==7.2.0" + }, "nodeenv": { "hashes": [ "sha256:ad8259494cf1c9034539f6cced78a1da4840a4b157e23640bc4a0c0546b0cb7a" @@ -251,11 +252,11 @@ }, "pre-commit": { "hashes": [ - "sha256:92e406d556190503630fd801958379861c94884693a032ba66629d0351fdccd4", - "sha256:cccc39051bc2457b0c0f7152a411f8e05e3ba2fe1a5613e4ee0833c1c1985ce3" + "sha256:1d3c0587bda7c4e537a46c27f2c84aa006acc18facf9970bf947df596ce91f3f", + "sha256:fa78ff96e8e9ac94c748388597693f18b041a181c94a4f039ad20f45287ba44a" ], "index": "pypi", - "version": "==1.17.0" + "version": "==1.18.3" }, "pycodestyle": { "hashes": [ @@ -266,11 +267,11 @@ }, "pydocstyle": { "hashes": [ - "sha256:2258f9b0df68b97bf3a6c29003edc5238ff8879f1efb6f1999988d934e432bd8", - "sha256:5741c85e408f9e0ddf873611085e819b809fca90b619f5fd7f34bd4959da3dd4", - "sha256:ed79d4ec5e92655eccc21eb0c6cf512e69512b4a97d215ace46d17e4990f2039" + "sha256:04c84e034ebb56eb6396c820442b8c4499ac5eb94a3bda88951ac3dc519b6058", + "sha256:66aff87ffe34b1e49bff2dd03a88ce6843be2f3346b0c9814410d34987fbab59" ], - "version": "==3.0.0" + "index": "pypi", + "version": "==4.0.1" }, "pyflakes": { "hashes": [ @@ -281,19 +282,21 @@ }, "pyyaml": { "hashes": [ - "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3", - "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043", - "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7", - "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265", - "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391", - "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778", - "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225", - "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955", - "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e", - "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190", - "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd" - ], - "version": "==5.1.1" + "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", + "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", + "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", + "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", + "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", + "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", + "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", + "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", + "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", + "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", + "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", + "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" + ], + "version": "==5.1.2" }, "six": { "hashes": [ @@ -304,10 +307,9 @@ }, "snowballstemmer": { "hashes": [ - "sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128", - "sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89" + "sha256:713e53b79cbcf97bc5245a06080a33d54a77e7cce2f789c835a143bcdb5c033e" ], - "version": "==1.2.1" + "version": "==1.9.1" }, "toml": { "hashes": [ @@ -326,17 +328,17 @@ }, "virtualenv": { "hashes": [ - "sha256:b7335cddd9260a3dd214b73a2521ffc09647bde3e9457fcca31dc3be3999d04a", - "sha256:d28ca64c0f3f125f59cabf13e0a150e1c68e5eea60983cc4395d88c584495783" + "sha256:680af46846662bb38c5504b78bad9ed9e4f3ba2d54f54ba42494fdf94337fe30", + "sha256:f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2" ], - "version": "==16.6.1" + "version": "==16.7.5" }, "zipp": { "hashes": [ - "sha256:8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d", - "sha256:ca943a7e809cc12257001ccfb99e3563da9af99d52f261725e96dfe0f9275bc3" + "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", + "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" ], - "version": "==0.5.1" + "version": "==0.6.0" } } } @@ -2,9 +2,9 @@ # snekbox -Python sandbox runners for executing code in isolation aka snekbox +Python sandbox runners for executing code in isolation aka snekbox. -The user sends a piece of python code to a snekbox, the snekbox executes the code and sends the result back to the users. +A client sends Python code to a snekbox, the snekbox executes the code, and finally the results of the execution are returned to the client. ``` +-------------+ +-----------+ @@ -22,91 +22,124 @@ result <- | |<----------| | <----------+ ``` +The code is executed in a Python process that is launched through [NsJail](https://github.com/google/nsjail), which is responsible for sandboxing the Python process. NsJail is configured as follows: -## Dependencies +* Root directory is mounted as read-only +* Time limit of 2 seconds +* Maximum of 1 PID +* Maximum memory of 52428800 bytes +* Loopback interface is down +* procfs is disabled -| dep | version (or greater) | -|----------------|:---------------------| -| python | 3.6.5 | -| pip | 10.0.1 | -| pipenv | 2018.05.18 | -| docker | 18.03.1-ce | -| docker-compose | 1.21.2 | -| nsjail | 2.5 | -| flask | 1.0.2 | -| gunicorn | 19.9 | +The Python process is configured as follows: -_________________________________________ -## Setup local test +* Version 3.7.4 +* Isolated mode + * Neither the script's directory nor the user's site packages are in `sys.path` + * All `PYTHON*` environment variables are ignored -install python packages -```bash -apt-get install -y libprotobuf-dev #needed by nsjail -pipenv sync --dev +## HTTP REST API + +Communication with snekbox is done over a HTTP REST API. The framework for the HTTP REST API is [Falcon](https://falconframework.org/) and the WSGI being used is [Gunicorn](https://gunicorn.org/). By default, the server is hosted on `0.0.0.0:8060` with two workers. + +See [`snekapi.py`](snekbox/api/snekapi.py) and [`resources`](snekbox/api/resources) for API documentation. + +## Development Environment + +### Initial Setup + +A Python 3.7 interpreter and the [pipenv](https://docs.pipenv.org/en/latest/) package are required. Once those requirements are satisfied, install the project's dependencies: + +``` +pipenv --sync ``` -## NSJail +Follow that up with setting up the pre-commit hook: + +``` +pipenv run precommit +``` -Copy the appropriate binary to an appropriate path +Now Flake8 will run and lint staged changes whenever an attempt to commit the changes is made. Flake8 can still be invoked manually: -```bash -cp binaries/nsjail2.6-ubuntu-x86_64 /usr/bin/nsjail -chmod +x /usr/bin/nsjail +``` +pipenv run lint ``` -give nsjail a test run +### Running snekbox -```bash -# This is a workaround because nsjail can't create the directories automatically -sudo mkdir -p /sys/fs/cgroup/pids/NSJAIL \ - && mkdir -p /sys/fs/cgroup/memory/NSJAIL +The Docker images can be built with: -nsjail -Mo \ ---rlimit_as 700 \ ---chroot / \ --E LANG=en_US.UTF-8 \ --R/usr -R/lib -R/lib64 \ ---user nobody \ ---group nogroup \ ---time_limit 2 \ ---disable_proc \ ---iface_no_lo \ ---cgroup_pids_max=1 \ ---cgroup_mem_max=52428800 \ ---quiet -- \ -python3.6 -ISq -c "print('test')" ``` +pipenv run buildbase +pipenv run buildvenv +pipenv run build +``` + +Use Docker Compose to start snekbox: -> if it fails, try without the `--cgroup_pids_max=1` and `--cgroup_mem_max=52428800` +``` +docker-compose up +``` -## Development environment +### Running Tests -Start the webserver with docker: +Tests are run through coverage.py using unittest. Before tests can run, the dev venv Docker image has to be built: -```bash -docker-compose up -d +``` +pipenv run builddev ``` -Run locally with pipenv: -```bash -pipenv run snekbox # for debugging +Alternatively, the following command will build the image and then run the tests: + +``` +pipenv run testb ``` -Visit: `http://localhost:8060` -________________________________________ -## Unit testing and lint -```bash -pipenv run lint +If the image doesn't need to be built, the tests can be run with: + +``` pipenv run test ``` -________________________________________ -## Build the containers +### Coverage + +To see a coverage report, run + +``` +pipenv run report +``` + +Alternatively, a report can be generated as HTML: + +``` +pipenv run coverage html +``` + +The HTML will output to `./htmlcov/` by default + + +### The `devsh` Helper Script + +This script starts an `ash` shell inside the venv Docker container and attaches to it. Unlike the production image, the venv image that is built by this script contains dev dependencies too. The project directory is mounted inside the container so any filesystem changes made inside the container affect the actual local project. + +#### Usage + +``` +pipenv run devsh [--build [--clean]] [ash_args ...] +``` + +* `--build` Build the venv Docker image +* `--clean` Clean up dangling Docker images (only works if `--build` precedes it) +* `ash_args` Arguments to pass to `/bin/ash` (for example `-c "echo hello"`). An interactive shell is launched if no arguments are given + +#### Invoking NsJail + +A shell alias named `nsjpy` is included and is basically `nsjail python -c <args>` but NsJail is configured as it would be if snekbox invoked it (such as the time and memory limits). It provides an easy way to run Python code inside NsJail without the need to run snekbox with its webserver and send HTTP requests. Example usage: ```bash -# Build -pipenv run buildbox -# Push -pipenv run pushbox +nsjpy "print('hello world!')" ``` + +The alias can be found in `./scripts/.profile`, which is automatically added when the shell is launched in the container. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 175ed16..ffd7e45 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,106 +5,222 @@ jobs: displayName: 'Lint & Test' pool: - vmImage: 'Ubuntu-16.04' + vmImage: 'ubuntu-16.04' steps: - task: ShellScript@2 - inputs: - scriptPath: scripts/check_dockerfiles.sh - name: check2 displayName: 'Check If Images Need to Be Built' - enabled: false - - - script: | - echo "##vso[task.setvariable variable=BASE_CHANGED;isOutput=true]False" - echo "##vso[task.setvariable variable=VENV_CHANGED;isOutput=true]False" - echo "##vso[task.setvariable variable=BASE_PULL;isOutput=true]True" name: check - displayName: 'Set Variables' - enabled: false - - - script: echo $(check.BASE_PULL) - displayName: 'Echo Output Variable in Same Job' - enabled: false - - - script: echo foo - condition: and(succeeded(), eq(variables['check.BASE_PULL'], True)) - displayName: 'Use Output Variable in Condition' - enabled: false - - - task: Docker@2 - displayName: 'Login to Docker Hub' inputs: - command: login - containerRegistry: DockerHub + scriptPath: scripts/check_dockerfiles.sh + # The venv image depends on this image. Build it if it can't be pulled + # from Docker Hub, which will be the case if the base Dockerfile has had + # changes. - task: Docker@2 displayName: 'Build Base Image' + condition: and(succeeded(), ne(variables['check.BASE_PULL'], True)) inputs: command: build + containerRegistry: DockerHub repository: pythondiscord/snekbox-base - tags: foobar + tags: latest Dockerfile: docker/base.Dockerfile buildContext: . - - script: docker images -a - displayName: 'List Docker Images' + # The dev image is never pushed and therefore is always built. + - task: Docker@2 + displayName: 'Build Development Image' + inputs: + command: build + containerRegistry: DockerHub + repository: pythondiscord/snekbox-venv + tags: dev + Dockerfile: docker/venv.Dockerfile + buildContext: . + arguments: --build-arg DEV=1 + # The linter and all tests run inside this container. - script: | docker run \ - -td \ + --tty \ + --detach \ --name snekbox_test \ --privileged \ --network host \ - -h pdsnk-dev \ + --hostname pdsnk-dev \ -e PYTHONDONTWRITEBYTECODE=1 \ -e PIPENV_PIPFILE="/snekbox/Pipfile" \ -e ENV="${PWD}/scripts/.profile" \ - -v "${PWD}":"${PWD}" \ - -w "${PWD}"\ + --volume "${PWD}":"${PWD}" \ + --workdir "${PWD}"\ --entrypoint /bin/ash \ - pythondiscord/snekbox-base:foobar + pythondiscord/snekbox-venv:dev displayName: 'Start Container' + - script: | + docker exec snekbox_test /bin/ash -c \ + 'pipenv run lint --format junit-xml --output-file test-lint.xml' + displayName: 'Run Linter' + + - task: PublishTestResults@2 + displayName: 'Publish Lint Results' + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/test-lint.xml' + testRunTitle: 'Lint Results' + + # Memory limit tests would fail if this isn't disabled. + - script: sudo swapoff -a + displayName: 'Disable Swap Memory' + + - script: | + docker exec snekbox_test /bin/ash -c \ + 'pipenv run coverage run -m xmlrunner' + displayName: 'Run Unit Tests' + + - task: PublishTestResults@2 + displayName: 'Publish Test Results' + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/TEST-*.xml' + testRunTitle: 'Test Results' + + # Run report too because the XML report doesn't output to stdout. + - script: | + docker exec snekbox_test /bin/ash -c \ + 'pipenv run /bin/ash -c "coverage report && coverage xml"' + displayName: 'Generate Coverage Report' + + - task: PublishCodeCoverageResults@1 + displayName: 'Publish Coverage Results' + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '**/coverage.xml' + + # When a pull request, only perform this job if images need to be built. + # It's always performed for non-PRs because the final image will always need + # to be built. - job: build displayName: 'Build' condition: > and( succeeded(), - eq(dependencies.test.outputs['check.BASE_CHANGED'], True) + or( + ne(variables['Build.Reason'], 'PullRequest'), + eq(coalesce(dependencies.test.outputs['check.BASE_CHANGED'], True), True), + eq(coalesce(dependencies.test.outputs['check.VENV_CHANGED'], True), True) + ) ) dependsOn: test + # coalesce() gives variables default values if they are null (i.e. unset). variables: BASE_CHANGED: $[ coalesce(dependencies.test.outputs['check.BASE_CHANGED'], True) ] VENV_CHANGED: $[ coalesce(dependencies.test.outputs['check.VENV_CHANGED'], True) ] BASE_PULL: $[ coalesce(dependencies.test.outputs['check.BASE_PULL'], False) ] steps: - - script: | - echo "${BASE_CHANGED}" - echo "${VENV_CHANGED}" - echo "${BASE_PULL}" - displayName: 'Echo All Variables' + - task: Docker@2 + displayName: 'Log into Docker Hub' + inputs: + command: login + containerRegistry: DockerHub + + # Because this is the base image for the venv image, if the venv needs to + # be built, this base image must also be present. Build it if it has + # changed or can't be pulled from Docker Hub. + - task: Docker@2 + displayName: 'Build Base Image' + condition: > + and( + succeeded(), + ne(variables.BASE_PULL, True), + or( + eq(variables.BASE_CHANGED, True), + eq(variables.VENV_CHANGED, True) + ) + ) + inputs: + command: build + repository: pythondiscord/snekbox-base + tags: latest + Dockerfile: docker/base.Dockerfile + buildContext: . - - script: echo bar + # Also build this image if base has changed - even if this image hasn't. + - task: Docker@2 + displayName: 'Build Virtual Environment Image' condition: > and( succeeded(), - eq(variables.BASE_CHANGED, False), - eq(variables.VENV_CHANGED, False), - eq(variables.BASE_PULL, True) + or( + eq(variables.BASE_CHANGED, True), + eq(variables.VENV_CHANGED, True) + ) ) - displayName: 'Compare String' + inputs: + command: build + repository: pythondiscord/snekbox-venv + tags: latest + Dockerfile: docker/venv.Dockerfile + buildContext: . - - script: echo bar - condition: and(succeeded(), eq(True, variables.BASE_PULL)) - displayName: 'Compare String Reverse' + # Always build this image unless it's for a pull request. + - task: Docker@2 + displayName: 'Build Final Image' + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + inputs: + command: build + repository: pythondiscord/snekbox + tags: latest + Dockerfile: docker/Dockerfile + buildContext: . - - script: echo bar - condition: and(succeeded(), eq('True', variables.BASE_PULL)) - displayName: 'Compare String to String' + # Push images only after they've all successfully been built. + # These have the same conditions as the build tasks. However, for safety, + # a condition for not being a pull request is added. + - task: Docker@2 + displayName: 'Push Base Image' + condition: > + and( + succeeded(), + ne(variables['Build.Reason'], 'PullRequest'), + ne(variables.BASE_PULL, True), + or( + eq(variables.BASE_CHANGED, True), + eq(variables.VENV_CHANGED, True) + ) + ) + inputs: + command: push + containerRegistry: DockerHub + repository: pythondiscord/snekbox-base + tags: latest + + - task: Docker@2 + displayName: 'Push Virtual Environment Image' + condition: > + and( + succeeded(), + ne(variables['Build.Reason'], 'PullRequest'), + or( + eq(variables.BASE_CHANGED, True), + eq(variables.VENV_CHANGED, True) + ) + ) + inputs: + command: push + containerRegistry: DockerHub + repository: pythondiscord/snekbox-venv + tags: latest - - script: echo baz - condition: and(succeeded(), variables.BASE_PULL) - displayName: 'Compare Implicit Cast' + - task: Docker@2 + displayName: 'Push Final Image' + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + inputs: + command: push + containerRegistry: DockerHub + repository: pythondiscord/snekbox + tags: latest diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index 1f1b9a6..1edff49 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -1,2 +1,27 @@ -FROM alpine:3.9.2 -RUN echo hello +FROM alpine:3.10 as builder +RUN apk add --no-cache --update \ + bison~=3.3 \ + bsd-compat-headers~=0.7 \ + flex~=2.6 \ + g++~=8.3 \ + gcc~=8.3 \ + git~=2.22 \ + libnl3-dev~=3.4 \ + linux-headers~=4.19 \ + make~=4.2 \ + protobuf-dev~=3.6 +RUN git clone https://github.com/google/nsjail.git /nsjail \ + && cd /nsjail \ + && git checkout 0b1d5ac03932c140f08536ed72b4b58741e7d3cf +WORKDIR /nsjail +RUN make + +FROM python:3.7.4-alpine3.10 +ENV PIP_NO_CACHE_DIR=false +RUN apk add --no-cache --update \ + libnl3~=3.4 \ + libstdc++~=8.3 \ + protobuf~=3.6 +RUN pip install pipenv==2018.11.26 +COPY --from=builder /nsjail/nsjail /usr/sbin/ +RUN chmod +x /usr/sbin/nsjail diff --git a/scripts/.profile b/scripts/.profile index bff260d..bd46a17 100644 --- a/scripts/.profile +++ b/scripts/.profile @@ -20,8 +20,8 @@ nsjpy() { --chroot / \ -E LANG=en_US.UTF-8 \ -R/usr -R/lib -R/lib64 \ - --user nobody \ - --group nogroup \ + --user 65534 \ + --group 65534 \ --time_limit 2 \ --disable_proc \ --iface_no_lo \ diff --git a/scripts/check_dockerfiles.sh b/scripts/check_dockerfiles.sh index ddc7084..c84c61f 100755 --- a/scripts/check_dockerfiles.sh +++ b/scripts/check_dockerfiles.sh @@ -24,7 +24,8 @@ get_build() { response="$(curl -sSL "${url}")" if [[ -z "${response}" ]] \ - || ! printf '%s' "${response}" | jq -re '.count' + || ! count="$(printf '%s' "${response}" | jq -re '.count')" \ + || (( "${count}" < 1 )) then return 1 else @@ -64,7 +65,7 @@ printf '%s\n' "Comparing HEAD (${head}) against ${prev_commit}." if git diff --quiet "${prev_commit}" -- docker/base.Dockerfile; then echo "No changes detected in docker/base.Dockerfile." - echo "##vso[task.setvariable variable=BASE_CHANGED;isOutput=true]false" + echo "##vso[task.setvariable variable=BASE_CHANGED;isOutput=true]False" else # Always rebuild the venv if the base changes. exit 0 @@ -72,13 +73,14 @@ fi if git diff --quiet "${prev_commit}" -- docker/venv.Dockerfile Pipfile*; then echo "No changes detected in docker/venv.Dockerfile or the Pipfiles." - echo "##vso[task.setvariable variable=VENV_CHANGED;isOutput=true]false" + echo "##vso[task.setvariable variable=VENV_CHANGED;isOutput=true]False" elif master_commit="$( get_build "refs/heads/master" \ | jq -re '.value[0].sourceVersion' )" \ && git diff --quiet "${master_commit}" -- docker/base.Dockerfile then + # Though base image hasn't changed, it's still needed to build the venv. echo "Can pull base image from Docker Hub; no changes made since master." - echo "##vso[task.setvariable variable=BASE_PULL;isOutput=true]true" + echo "##vso[task.setvariable variable=BASE_PULL;isOutput=true]True" fi diff --git a/scripts/dev.sh b/scripts/dev.sh index 097690b..8f5b24f 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -34,16 +34,17 @@ fi # The volume is mounted to same the path in the container as the source # directory on the host to ensure coverage can find the source files. docker run \ - -td \ + --tty \ + --detach \ --name snekbox_test \ --privileged \ --network host \ - -h pdsnk-dev \ + --hostname pdsnk-dev \ -e PYTHONDONTWRITEBYTECODE=1 \ -e PIPENV_PIPFILE="/snekbox/Pipfile" \ -e ENV="${PWD}/scripts/.profile" \ - -v "${PWD}":"${PWD}" \ - -w "${PWD}"\ + --volume "${PWD}":"${PWD}" \ + --workdir "${PWD}"\ --entrypoint /bin/ash \ pythondiscord/snekbox-venv:dev \ >/dev/null \ |