diff options
38 files changed, 1293 insertions, 1173 deletions
diff --git a/.coveragerc b/.coveragerc index c1e718b..8490bab 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,10 +1,9 @@ [run] -omit = .venv/*, - tests/*, - snekweb.py - config.py +branch = True +include = snekbox/* +omit = snekbox/api/app.py [report] -exclude_lines = return jsonify, - raise RuntimeError, - return +exclude_lines = + pragma: no cover + if DEBUG diff --git a/.dockerignore b/.dockerignore index 8914ea8..afc786a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,18 +1,10 @@ -.venv -scripts -htmlcov -__pycache__ -.vagrant -.pytest_cache -.git -.github -.cache -Vagrantfile -.coverage -.coveragerc -.gitignore -.travis.yml -docker -docker-compose.yml -LICENSE -README.md +# Exclude everything +* + +# Make exceptions for what's needed +!docker/.profile +!snekbox +!tests +!Pipfile +!Pipfile.lock +!LICENSE @@ -1,18 +1,20 @@ [flake8] max-line-length=100 -application_import_names=snekbox,config,logs +application_import_names=snekbox +docstring-convention=all ignore= P102,B311,W503,E226,S311, # Missing Docstrings - D100,D104,D107, + D100,D104,D105,D107, # Docstring Whitespace D203,D212,D214,D215, # Docstring Quotes D301,D302, # Docstring Content - D400,D401,D402,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414 + D400,D401,D402,D404,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414,D416,D417 exclude= __pycache__,.cache, - venv,.venv, - tests + venv,.venv +per-file-ignores=tests/*:D1 import-order-style=pycharm +inline-quotes=" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 2dc9608..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,26 +0,0 @@ -image: pythondiscord/snekbox-ci:latest - -variables: - PIPENV_CACHE_DIR: "$CI_PROJECT_DIR/pipenv-cache" - -cache: - paths: - - "$CI_PROJECT_DIR/pipenv-cache" - - "$CI_PROJECT_DIR/.venv" - -services: - - docker:dind - -stages: - - build - -build: - tags: - - docker - - pythondiscord - stage: build - script: - - pipenv install --dev --deploy - - pipenv run lint - - pipenv run test - - sh scripts/deploy.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1d75342..b5f3715 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,15 @@ repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.0.0 + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.2.3 hooks: - - id: flake8
\ No newline at end of file + - id: flake8 + additional_dependencies: [ + pydocstyle ~= 4.0, + "flake8-docstrings >= 1.3.1, == 1.*", + flake8-bugbear ~= 19.3, + flake8-import-order ~= 0.18.1, + flake8-tidy-imports ~= 2.0, + flake8-todo ~= 0.7, + flake8-string-format ~= 0.2.3, + flake8-quotes ~= 2.1 + ] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 429fbf1..73ec91b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,6 +9,7 @@ Note that contributions may be rejected on the basis of a contributor failing to 1. **No force-pushes** or modifying the Git history in any way. 2. If you have direct access to the repository, **create a branch for your changes** and create a pull request for that branch. If not, create a branch on a fork of the repository and create a pull request from there. * It's common practice for a repository to reject direct pushes to `master`, so make branching a habit! + * If PRing from your own fork, **ensure that "Allow edits from maintainers" is checked**. This gives permission for maintainers to commit changes directly to your fork, speeding up the review process. 3. **Adhere to the prevailing code style**, which we enforce using [flake8](http://flake8.pycqa.org/en/latest/index.html). * Run `flake8` against your code **before** you push it. Your commit will be rejected by the build server if it fails to lint. * [Git Hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) are a powerful tool that can be a daunting to set up. Fortunately, [`pre-commit`](https://github.com/pre-commit/pre-commit) abstracts this process away from you and is provided as a dev dependency for this project. Run `pipenv run precommit` when setting up the project and you'll never have to worry about breaking the build for linting errors. @@ -100,6 +101,8 @@ Github [has introduced a new PR feature](https://github.blog/2019-02-14-introduc This feature should be utilized in place of the traditional method of prepending `[WIP]` to the PR title. +As stated earlier, **ensure that "Allow edits from maintainers" is checked**. This gives permission for maintainers to commit changes directly to your fork, speeding up the review process. + ## Footnotes This document was inspired by the [Glowstone contribution guidelines](https://github.com/GlowstoneMC/Glowstone/blob/dev/docs/CONTRIBUTING.md). @@ -4,42 +4,71 @@ verify_ssl = true name = "pypi" [packages] -pika = "*" -docker = "*" -urllib3 = ">=1.24.2,<1.25" +falcon = "~= 2.0.0" +gunicorn = "~= 19.9" +jsonschema = "~= 3.0" [dev-packages] -flask = "*" -flask-sockets = "*" -gevent = "==1.2.2" -gevent-websocket = "*" -gunicorn = "*" -pytest = "*" -pytest-cov = "*" -pytest-dependency = "*" -pre-commit = "*" -flake8 = "*" -flake8-docstrings = "*" -flake8-bugbear = "*" -flake8-import-order = "*" -flake8-tidy-imports = "*" -flake8-todo = "*" -flake8-string-format = "*" -flake8-formatter-junit-xml = "*" +coverage = ">= 4.4.2, == 4.*" +pre-commit = "~= 1.18" +pydocstyle = "~= 4.0" +flake8 = "~= 3.7.8" +flake8-docstrings = "~=1.4" +flake8-bugbear = "~= 19.3" +flake8-import-order = "~= 0.18.1" +flake8-tidy-imports = "~= 2.0" +flake8-todo = "~= 0.7" +flake8-string-format = "~= 0.2.3" +flake8-formatter-junit-xml = "~= 0.0.6" +flake8-quotes = "~= 2.1" +unittest-xml-reporting = ">= 2.5.1, == 2.*" [requires] -python_version = "3.6" +python_version = "3.7" [scripts] lint = "flake8" precommit = "pre-commit install" -test = "py.test tests --cov . --cov-report term-missing -v" -report = "py.test tests --cov . --cov-report=html" -snekbox = "python snekbox.py" -snekweb = "gunicorn -w 2 -b 0.0.0.0:5000 --log-level debug -k geventwebsocket.gunicorn.workers.GeventWebSocketWorker snekweb:app" -buildbox = "docker build -t pythondiscord/snekbox:latest -f docker/Dockerfile ." -pushbox = "docker push pythondiscord/snekbox:latest" -buildboxbase = "docker build -t pythondiscord/snekbox-base:latest -f docker/base.Dockerfile ." -pushboxbase = "docker push pythondiscord/snekbox-base:latest" -buildweb = "docker build -t pythondiscord/snekboxweb:latest -f docker/Dockerfile.webapp ." -pushweb = "docker push pythondiscord/snekboxweb:latest" +test = "sh scripts/dev.sh -c 'pipenv run coverage run -m unittest'" +testb = """ + sh scripts/dev.sh \ + --build \ + --clean \ + -c 'pipenv run coverage run -m unittest' +""" +report = "coverage report" +snekbox = """ + gunicorn \ + -w 2 \ + -b 0.0.0.0:8060 \ + --logger-class snekbox.GunicornLogger \ + --access-logformat '%(m)s %(U)s%(q)s %(s)s %(b)s %(L)ss' \ + --access-logfile - \ + snekbox.api.app +""" +devsh = "sh scripts/dev.sh" +build = """ + docker build \ + -t pythondiscord/snekbox:latest \ + -f docker/Dockerfile \ + . +""" +buildbase = """ + docker build \ + -t pythondiscord/snekbox-base:latest \ + -f docker/base.Dockerfile \ + . +""" +buildvenv = """ + docker build \ + -t pythondiscord/snekbox-venv:latest \ + -f docker/venv.Dockerfile \ + . +""" +builddev = """ + docker build \ + -t pythondiscord/snekbox-venv:dev \ + -f docker/venv.Dockerfile \ + --build-arg DEV=1 \ + . +""" diff --git a/Pipfile.lock b/Pipfile.lock index 262131c..b541730 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "22c5372766fde7766ebb92d523cfc1eb0192bf1cc2b4860c51b50937998da40d" + "sha256": "fa4ef446ed6cd8618914fd2b509e1d437f71cd70140bba4b5b95a4eaf6e53933" }, "pipfile-spec": 6, "requires": { - "python_version": "3.6" + "python_version": "3.7" }, "sources": [ { @@ -16,56 +16,54 @@ ] }, "default": { - "certifi": { - "hashes": [ - "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", - "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" - ], - "version": "==2019.3.9" - }, - "chardet": { + "attrs": { "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==3.0.4" + "version": "==19.1.0" }, - "docker": { - "hashes": [ - "sha256:2b1f48041cfdcc9f6b5da0e04e0e326ded225e736762ade2060000e708f4c9b7", - "sha256:c456ded5420af5860441219ff8e51cdec531d65f4a9e948ccd4133e063b72f50" + "falcon": { + "hashes": [ + "sha256:18157af2a4fc3feedf2b5dcc6196f448639acf01c68bc33d4d5a04c3ef87f494", + "sha256:24adcd2b29a8ffa9d552dc79638cd21736a3fb04eda7d102c6cebafdaadb88ad", + "sha256:54f2cb4b687035b2a03206dbfc538055cc48b59a953187b0458aa1b574d47b53", + "sha256:59d1e8c993b9a37ea06df9d72cf907a46cc8063b30717cdac2f34d1658b6f936", + "sha256:733033ec80c896e30a43ab3e776856096836787197a44eb21022320a61311983", + "sha256:74cf1d18207381c665b9e6292d65100ce146d958707793174b03869dc6e614f4", + "sha256:95bf6ce986c1119aef12c9b348f4dee9c6dcc58391bdd0bc2b0bf353c2b15986", + "sha256:9712975adcf8c6e12876239085ad757b8fdeba223d46d23daef82b47658f83a9", + "sha256:a5ebb22a04c9cc65081938ee7651b4e3b4d2a28522ea8ec04c7bdd2b3e9e8cd8", + "sha256:aa184895d1ad4573fbfaaf803563d02f019ebdf4790e41cc568a330607eae439", + "sha256:e3782b7b92fefd46a6ad1fd8fe63fe6c6f1b7740a95ca56957f48d1aee34b357", + "sha256:e9efa0791b5d9f9dd9689015ea6bce0a27fcd5ecbcd30e6d940bffa4f7f03389", + "sha256:eea593cf466b9c126ce667f6d30503624ef24459f118c75594a69353b6c3d5fc", + "sha256:f93351459f110b4c1ee28556aef9a791832df6f910bea7b3f616109d534df06b" ], "index": "pypi", - "version": "==3.7.2" - }, - "docker-pycreds": { - "hashes": [ - "sha256:6ce3270bcaf404cc4c3e27e4b6c70d3521deae82fb508767870fdbf772d584d4", - "sha256:7266112468627868005106ec19cd0d722702d2b7d5912a28e19b826c3d37af49" - ], - "version": "==0.4.0" + "version": "==2.0.0" }, - "idna": { + "gunicorn": { "hashes": [ - "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", - "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + "sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471", + "sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3" ], - "version": "==2.8" + "index": "pypi", + "version": "==19.9.0" }, - "pika": { + "jsonschema": { "hashes": [ - "sha256:0c50285f00a8b4816f2c9a44469107d9e738ba3a90386f14b625d8cceef4f6ae", - "sha256:5ba83d3daffccb92788d24facdab62a3db6aa03b8a6d709b03dc792d35c0dfe8" + "sha256:5f9c0a719ca2ce14c5de2fd350a64fd2d13e8539db29836a86adc990bb1a068f", + "sha256:8d4a2b7b6c2237e0199c8ea1a6d3e05bf118e289ae2b9d7ba444182a2959560d" ], "index": "pypi", - "version": "==1.0.1" + "version": "==3.0.2" }, - "requests": { + "pyrsistent": { "hashes": [ - "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", - "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + "sha256:34b47fa169d6006b32e99d4b3c4031f155e6e68ebcc107d6454852e8e0ee6533" ], - "version": "==2.21.0" + "version": "==0.15.4" }, "six": { "hashes": [ @@ -73,35 +71,13 @@ "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], "version": "==1.12.0" - }, - "urllib3": { - "hashes": [ - "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", - "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" - ], - "index": "pypi", - "version": "==1.24.2" - }, - "websocket-client": { - "hashes": [ - "sha256:1151d5fb3a62dc129164292e1227655e4bbc5dd5340a5165dfae61128ec50aa9", - "sha256:1fd5520878b68b84b5748bb30e592b10d0a91529d5383f74f4964e72b297fd3a" - ], - "version": "==0.56.0" } }, "develop": { "aspy.yaml": { "hashes": [ - "sha256:ae249074803e8b957c83fdd82a99160d0d6d26dff9ba81ba608b42eebd7d8cd3", - "sha256:c7390d79f58eb9157406966201abf26da0d56c07e0ff0deadc39c8f4dbc13482" - ], - "version": "==1.2.0" - }, - "atomicwrites": { - "hashes": [ - "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", - "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" + "sha256:463372c043f70160a9ec950c3f1e4c3a82db5fca01d334b6bc89c7164d744bdc", + "sha256:e7c742382eff2caed61f87a39d13f99109088e5e93f04d76eb8d4b28aa143f45" ], "version": "==1.3.0" }, @@ -114,53 +90,48 @@ }, "cfgv": { "hashes": [ - "sha256:6e9f2feea5e84bc71e56abd703140d7a2c250fc5ba38b8702fd6a68ed4e3b2ef", - "sha256:e7f186d4a36c099a9e20b04ac3108bd8bb9b9257e692ce18c8c3764d5cb12172" + "sha256:edb387943b665bf9c434f717bf630fa78aecd53d5900d2e05da6ad6048553144", + "sha256:fbd93c9ab0a523bf7daec408f3be2ed99a980e20b2d19b50fc184ca6b820d289" ], - "version": "==1.6.0" - }, - "click": { - "hashes": [ - "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", - "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" - ], - "version": "==7.0" + "version": "==2.0.1" }, "coverage": { "hashes": [ - "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", - "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", - "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", - "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", - "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", - "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", - "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", - "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", - "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", - "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", - "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", - "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", - "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", - "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", - "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", - "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", - "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", - "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", - "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", - "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", - "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", - "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", - "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", - "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", - "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", - "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", - "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", - "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", - "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", - "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", - "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" - ], - "version": "==4.5.3" + "sha256:08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6", + "sha256:0be0f1ed45fc0c185cfd4ecc19a1d6532d72f86a2bac9de7e24541febad72650", + "sha256:141f08ed3c4b1847015e2cd62ec06d35e67a3ac185c26f7635f4406b90afa9c5", + "sha256:19e4df788a0581238e9390c85a7a09af39c7b539b29f25c89209e6c3e371270d", + "sha256:23cc09ed395b03424d1ae30dcc292615c1372bfba7141eb85e11e50efaa6b351", + "sha256:245388cda02af78276b479f299bbf3783ef0a6a6273037d7c60dc73b8d8d7755", + "sha256:331cb5115673a20fb131dadd22f5bcaf7677ef758741312bee4937d71a14b2ef", + "sha256:386e2e4090f0bc5df274e720105c342263423e77ee8826002dcffe0c9533dbca", + "sha256:3a794ce50daee01c74a494919d5ebdc23d58873747fa0e288318728533a3e1ca", + "sha256:60851187677b24c6085248f0a0b9b98d49cba7ecc7ec60ba6b9d2e5574ac1ee9", + "sha256:63a9a5fc43b58735f65ed63d2cf43508f462dc49857da70b8980ad78d41d52fc", + "sha256:6b62544bb68106e3f00b21c8930e83e584fdca005d4fffd29bb39fb3ffa03cb5", + "sha256:6ba744056423ef8d450cf627289166da65903885272055fb4b5e113137cfa14f", + "sha256:7494b0b0274c5072bddbfd5b4a6c6f18fbbe1ab1d22a41e99cd2d00c8f96ecfe", + "sha256:826f32b9547c8091679ff292a82aca9c7b9650f9fda3e2ca6bf2ac905b7ce888", + "sha256:93715dffbcd0678057f947f496484e906bf9509f5c1c38fc9ba3922893cda5f5", + "sha256:9a334d6c83dfeadae576b4d633a71620d40d1c379129d587faa42ee3e2a85cce", + "sha256:af7ed8a8aa6957aac47b4268631fa1df984643f07ef00acd374e456364b373f5", + "sha256:bf0a7aed7f5521c7ca67febd57db473af4762b9622254291fbcbb8cd0ba5e33e", + "sha256:bf1ef9eb901113a9805287e090452c05547578eaab1b62e4ad456fcc049a9b7e", + "sha256:c0afd27bc0e307a1ffc04ca5ec010a290e49e3afbe841c5cafc5c5a80ecd81c9", + "sha256:dd579709a87092c6dbee09d1b7cfa81831040705ffa12a1b248935274aee0437", + "sha256:df6712284b2e44a065097846488f66840445eb987eb81b3cc6e4149e7b6982e1", + "sha256:e07d9f1a23e9e93ab5c62902833bf3e4b1f65502927379148b6622686223125c", + "sha256:e2ede7c1d45e65e209d6093b762e98e8318ddeff95317d07a27a2140b80cfd24", + "sha256:e4ef9c164eb55123c62411f5936b5c2e521b12356037b6e1c2617cef45523d47", + "sha256:eca2b7343524e7ba246cab8ff00cab47a2d6d54ada3b02772e908a45675722e2", + "sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28", + "sha256:ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c", + "sha256:efc89291bd5a08855829a3c522df16d856455297cf35ae827a37edac45f466a7", + "sha256:fa964bae817babece5aa2e8c1af841bebb6d0b9add8e637548809d040443fee0", + "sha256:ff37757e068ae606659c28c3bd0d923f9d29a85de79bf25b2b34b148473b5025" + ], + "index": "pypi", + "version": "==4.5.4" }, "entrypoints": { "hashes": [ @@ -171,27 +142,27 @@ }, "flake8": { "hashes": [ - "sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661", - "sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8" + "sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548", + "sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696" ], "index": "pypi", - "version": "==3.7.7" + "version": "==3.7.8" }, "flake8-bugbear": { "hashes": [ - "sha256:5070774b668be92c4312e5ca82748ddf4ecaa7a24ff062662681bb745c7896eb", - "sha256:fef9c9826d14ec23187ae1edeb3c6513c4e46bf0e70d86bac38f7d9aabae113d" + "sha256:d8c466ea79d5020cb20bf9f11cf349026e09517a42264f313d3f6fddb83e0571", + "sha256:ded4d282778969b5ab5530ceba7aa1a9f1b86fa7618fc96a19a1d512331640f8" ], "index": "pypi", - "version": "==19.3.0" + "version": "==19.8.0" }, "flake8-docstrings": { "hashes": [ - "sha256:4e0ce1476b64e6291520e5570cf12b05016dd4e8ae454b8a8a9a48bc5f84e1cd", - "sha256:8436396b5ecad51a122a2c99ba26e5b4e623bf6e913b0fea0cb6c2c4050f91eb" + "sha256:1666dd069c9c457ee57e80af3c1a6b37b00cc1801c6fde88e455131bb2e186cd", + "sha256:9c0db5a79a1affd70fdf53b8765c8a26bf968e59e0252d7f2fc546b41c0cda06" ], "index": "pypi", - "version": "==1.3.0" + "version": "==1.4.0" }, "flake8-formatter-junit-xml": { "hashes": [ @@ -209,12 +180,12 @@ "index": "pypi", "version": "==0.18.1" }, - "flake8-polyfill": { + "flake8-quotes": { "hashes": [ - "sha256:12be6a34ee3ab795b19ca73505e7b55826d5f6ad7230d31b18e106400169b9e9", - "sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda" + "sha256:5dbaf668887873f28346fb87943d6da2e4b9f77ce9f2169cff21764a0a4934ed" ], - "version": "==1.0.2" + "index": "pypi", + "version": "==2.1.0" }, "flake8-string-format": { "hashes": [ @@ -239,116 +210,19 @@ "index": "pypi", "version": "==0.7" }, - "flask": { - "hashes": [ - "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", - "sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05" - ], - "index": "pypi", - "version": "==1.0.2" - }, - "flask-sockets": { - "hashes": [ - "sha256:072927da8edca0e81e024f5787e643c87d80b351b714de95d723becb30e0643b", - "sha256:350a76d55f5889f64afd2ca9b32f262680b7960965f0830365576307d30cfe1e" - ], - "index": "pypi", - "version": "==0.2.1" - }, - "gevent": { - "hashes": [ - "sha256:0901975628790e8a57fc92bb7062e5b856edea48c8de9caf36cfda14eae07329", - "sha256:1af93825db5753550fa8ff5ab2f2132e8733170b3f8d38347b34fa4a984cb624", - "sha256:2ff045a91509c35664c27a849c8cbf742a227f587b7cdbc88301e9c85dcaedff", - "sha256:35790f1a3c8e431ada3471b70bb2105050009ea4beb15cbe41b86bc716a7ffa9", - "sha256:4791c8ae9c57d6f153354736e1ccab1e2baf6c8d9ae5a77a9ac90f41e2966b2d", - "sha256:552719cec4721673b8c7d2f9de666e3f7591b9b182f801ecaef1c76e638052aa", - "sha256:59e9237af027f8db85e5d78a9da2e328ae96f01d67a0d62abcecad3db7876908", - "sha256:60109741377367eef8ded9283a1bf629621b73acaf3e1e8aac9d1a0f50fa0f05", - "sha256:70558dd45c7a1f8046ba45792e489dd0f409bd8a3b7a0635ca9d3055223b3dff", - "sha256:81cb24e0f7bd9888596364e8d8ed0d65c2547c84884c67bb46d956faeed67396", - "sha256:833bebdc36bfeeedefc200ca9aee9b8eddd80f56b63ca1e886e18b97b1240edd", - "sha256:8a710eddb3e9e5f22bdbd458b5f211b94f59409ecd6896f15b9fee2cba266a59", - "sha256:9b492bb1a043540abb6e54fdb5537531e24962ca49c09f3b47dc4f9c37f6297c", - "sha256:a16db4f56699ef07f0249b953ff949aae641e50b2bdc4710f11c0d8d9089b296", - "sha256:a66cf99f08da65c501826a19e30f5a6e7ba942fdd79baba5ce2d51eebaa13444", - "sha256:b67a10799923f9fed546ca5f8b93a2819c71a60132d7a97b4a13fbdab66b278a", - "sha256:b7e0e6400c2f3ce78a9ae1cdd55b53166feedd003d60c033863881227129a4d3", - "sha256:c9dd6534c46ed782e2d7236767cd07115cb29ce8670c2fc0794f264de9024fe0", - "sha256:de13a8e378103af84a8bf6015ad1d2761d46f29b8393e8dd6d9bb7cb51bbb713", - "sha256:deafd70d04ab62428d4e291e8e2c0fb22f38690e6a9f23a67ee6c304087634da", - "sha256:df52e06a2754c2d905aad75a7dc06a732c804d9edbc87f06f47c8f483ba98bca" - ], - "index": "pypi", - "version": "==1.2.2" - }, - "gevent-websocket": { - "hashes": [ - "sha256:17b67d91282f8f4c973eba0551183fc84f56f1c90c8f6b6b30256f31f66f5242", - "sha256:7eaef32968290c9121f7c35b973e2cc302ffb076d018c9068d2f5ca8b2d85fb0" - ], - "index": "pypi", - "version": "==0.10.1" - }, - "greenlet": { - "hashes": [ - "sha256:000546ad01e6389e98626c1367be58efa613fa82a1be98b0c6fc24b563acc6d0", - "sha256:0d48200bc50cbf498716712129eef819b1729339e34c3ae71656964dac907c28", - "sha256:23d12eacffa9d0f290c0fe0c4e81ba6d5f3a5b7ac3c30a5eaf0126bf4deda5c8", - "sha256:37c9ba82bd82eb6a23c2e5acc03055c0e45697253b2393c9a50cef76a3985304", - "sha256:51503524dd6f152ab4ad1fbd168fc6c30b5795e8c70be4410a64940b3abb55c0", - "sha256:8041e2de00e745c0e05a502d6e6db310db7faa7c979b3a5877123548a4c0b214", - "sha256:81fcd96a275209ef117e9ec91f75c731fa18dcfd9ffaa1c0adbdaa3616a86043", - "sha256:853da4f9563d982e4121fed8c92eea1a4594a2299037b3034c3c898cb8e933d6", - "sha256:8b4572c334593d449113f9dc8d19b93b7b271bdbe90ba7509eb178923327b625", - "sha256:9416443e219356e3c31f1f918a91badf2e37acf297e2fa13d24d1cc2380f8fbc", - "sha256:9854f612e1b59ec66804931df5add3b2d5ef0067748ea29dc60f0efdcda9a638", - "sha256:99a26afdb82ea83a265137a398f570402aa1f2b5dfb4ac3300c026931817b163", - "sha256:a19bf883b3384957e4a4a13e6bd1ae3d85ae87f4beb5957e35b0be287f12f4e4", - "sha256:a9f145660588187ff835c55a7d2ddf6abfc570c2651c276d3d4be8a2766db490", - "sha256:ac57fcdcfb0b73bb3203b58a14501abb7e5ff9ea5e2edfa06bb03035f0cff248", - "sha256:bcb530089ff24f6458a81ac3fa699e8c00194208a724b644ecc68422e1111939", - "sha256:beeabe25c3b704f7d56b573f7d2ff88fc99f0138e43480cecdfcaa3b87fe4f87", - "sha256:d634a7ea1fc3380ff96f9e44d8d22f38418c1c381d5fac680b272d7d90883720", - "sha256:d97b0661e1aead761f0ded3b769044bb00ed5d33e1ec865e891a8b128bf7c656" - ], - "version": "==0.4.15" - }, - "gunicorn": { - "hashes": [ - "sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471", - "sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3" - ], - "index": "pypi", - "version": "==19.9.0" - }, "identify": { "hashes": [ - "sha256:244e7864ef59f0c7c50c6db73f58564151d91345cd9b76ed793458953578cadd", - "sha256:8ff062f90ad4b09cfe79b5dfb7a12e40f19d2e68a5c9598a49be45f16aba7171" + "sha256:4f1fe9a59df4e80fcb0213086fcf502bc1765a01ea4fe8be48da3b65afd2a017", + "sha256:d8919589bd2a5f99c66302fec0ef9027b12ae150b0b0213999ad3f695fc7296e" ], - "version": "==1.4.1" + "version": "==1.4.7" }, "importlib-metadata": { "hashes": [ - "sha256:46fc60c34b6ed7547e2a723fc8de6dc2e3a1173f8423246b3ce497f064e9c3de", - "sha256:bc136180e961875af88b1ab85b4009f4f1278f8396a60526c0009f503a1a96ca" - ], - "version": "==0.9" - }, - "itsdangerous": { - "hashes": [ - "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", - "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" + "sha256:9ff1b1c5a354142de080b8a4e9803e5d0d59283c93aed808617c787d16768375", + "sha256:b7143592e374e50584564794fcb8aaf00a23025f9db866627f89a21491847a8d" ], - "version": "==1.1.0" - }, - "jinja2": { - "hashes": [ - "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", - "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" - ], - "version": "==2.10.1" + "version": "==0.20" }, "junit-xml": { "hashes": [ @@ -356,39 +230,6 @@ ], "version": "==1.8" }, - "markupsafe": { - "hashes": [ - "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", - "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", - "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", - "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", - "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", - "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", - "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", - "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", - "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", - "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", - "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", - "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", - "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", - "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", - "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", - "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", - "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", - "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", - "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", - "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", - "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", - "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", - "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", - "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", - "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", - "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", - "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", - "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" - ], - "version": "==1.1.1" - }, "mccabe": { "hashes": [ "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", @@ -398,11 +239,10 @@ }, "more-itertools": { "hashes": [ - "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7", - "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a" + "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", + "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" ], - "markers": "python_version > '2.7'", - "version": "==7.0.0" + "version": "==7.2.0" }, "nodeenv": { "hashes": [ @@ -410,27 +250,13 @@ ], "version": "==1.3.3" }, - "pluggy": { - "hashes": [ - "sha256:19ecf9ce9db2fce065a7a0586e07cfb4ac8614fe96edf628a264b1c70116cf8f", - "sha256:84d306a647cc805219916e62aab89caa97a33a1dd8c342e87a37f91073cd4746" - ], - "version": "==0.9.0" - }, "pre-commit": { "hashes": [ - "sha256:2576a2776098f3902ef9540a84696e8e06bf18a337ce43a6a889e7fa5d26c4c5", - "sha256:82f2f2d657d7f9280de9f927ae56886d60b9ef7f3714eae92d12713cd9cb9e11" + "sha256:1d3c0587bda7c4e537a46c27f2c84aa006acc18facf9970bf947df596ce91f3f", + "sha256:fa78ff96e8e9ac94c748388597693f18b041a181c94a4f039ad20f45287ba44a" ], "index": "pypi", - "version": "==1.15.2" - }, - "py": { - "hashes": [ - "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", - "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" - ], - "version": "==1.8.0" + "version": "==1.18.3" }, "pycodestyle": { "hashes": [ @@ -441,11 +267,11 @@ }, "pydocstyle": { "hashes": [ - "sha256:2258f9b0df68b97bf3a6c29003edc5238ff8879f1efb6f1999988d934e432bd8", - "sha256:5741c85e408f9e0ddf873611085e819b809fca90b619f5fd7f34bd4959da3dd4", - "sha256:ed79d4ec5e92655eccc21eb0c6cf512e69512b4a97d215ace46d17e4990f2039" + "sha256:04c84e034ebb56eb6396c820442b8c4499ac5eb94a3bda88951ac3dc519b6058", + "sha256:66aff87ffe34b1e49bff2dd03a88ce6843be2f3346b0c9814410d34987fbab59" ], - "version": "==3.0.0" + "index": "pypi", + "version": "==4.0.1" }, "pyflakes": { "hashes": [ @@ -454,44 +280,23 @@ ], "version": "==2.1.1" }, - "pytest": { - "hashes": [ - "sha256:3773f4c235918987d51daf1db66d51c99fac654c81d6f2f709a046ab446d5e5d", - "sha256:b7802283b70ca24d7119b32915efa7c409982f59913c1a6c0640aacf118b95f5" - ], - "index": "pypi", - "version": "==4.4.1" - }, - "pytest-cov": { - "hashes": [ - "sha256:0ab664b25c6aa9716cbf203b17ddb301932383046082c081b9848a0edf5add33", - "sha256:230ef817450ab0699c6cc3c9c8f7a829c34674456f2ed8df1fe1d39780f7c87f" - ], - "index": "pypi", - "version": "==2.6.1" - }, - "pytest-dependency": { - "hashes": [ - "sha256:bda0ef48e6a44c091399b12ab4a7e580d2dd8294c222b301f88d7d57f47ba142" - ], - "index": "pypi", - "version": "==0.4.0" - }, "pyyaml": { "hashes": [ - "sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c", - "sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95", - "sha256:460a5a4248763f6f37ea225d19d5c205677d8d525f6a83357ca622ed541830c2", - "sha256:5a22a9c84653debfbf198d02fe592c176ea548cccce47553f35f466e15cf2fd4", - "sha256:7a5d3f26b89d688db27822343dfa25c599627bc92093e788956372285c6298ad", - "sha256:9372b04a02080752d9e6f990179a4ab840227c6e2ce15b95e1278456664cf2ba", - "sha256:a5dcbebee834eaddf3fa7366316b880ff4062e4bcc9787b78c7fbb4a26ff2dd1", - "sha256:aee5bab92a176e7cd034e57f46e9df9a9862a71f8f37cad167c6fc74c65f5b4e", - "sha256:c51f642898c0bacd335fc119da60baae0824f2cde95b0330b56c0553439f0673", - "sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13", - "sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19" - ], - "version": "==5.1" + "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", + "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", + "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", + "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", + "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", + "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", + "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", + "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", + "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", + "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", + "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", + "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" + ], + "version": "==5.1.2" }, "six": { "hashes": [ @@ -502,10 +307,9 @@ }, "snowballstemmer": { "hashes": [ - "sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128", - "sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89" + "sha256:713e53b79cbcf97bc5245a06080a33d54a77e7cce2f789c835a143bcdb5c033e" ], - "version": "==1.2.1" + "version": "==1.9.1" }, "toml": { "hashes": [ @@ -514,26 +318,27 @@ ], "version": "==0.10.0" }, - "virtualenv": { + "unittest-xml-reporting": { "hashes": [ - "sha256:6aebaf4dd2568a0094225ebbca987859e369e3e5c22dc7d52e5406d504890417", - "sha256:984d7e607b0a5d1329425dd8845bd971b957424b5ba664729fab51ab8c11bc39" + "sha256:140982e4b58e4052d9ecb775525b246a96bfc1fc26097806e05ea06e9166dd6c", + "sha256:d1fbc7a1b6c6680ccfe75b5e9701e5431c646970de049e687b4bb35ba4325d72" ], - "version": "==16.4.3" + "index": "pypi", + "version": "==2.5.1" }, - "werkzeug": { + "virtualenv": { "hashes": [ - "sha256:0a73e8bb2ff2feecfc5d56e6f458f5b99290ef34f565ffb2665801ff7de6af7a", - "sha256:7fad9770a8778f9576693f0cc29c7dcc36964df916b83734f4431c0e612a7fbc" + "sha256:680af46846662bb38c5504b78bad9ed9e4f3ba2d54f54ba42494fdf94337fe30", + "sha256:f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2" ], - "version": "==0.15.2" + "version": "==16.7.5" }, "zipp": { "hashes": [ - "sha256:55ca87266c38af6658b84db8cfb7343cdb0bf275f93c7afaea0d8e7a209c7478", - "sha256:682b3e1c62b7026afe24eadf6be579fb45fec54c07ea218bded8092af07a68c4" + "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", + "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" ], - "version": "==0.3.3" + "version": "==0.6.0" } } } @@ -2,135 +2,144 @@ # snekbox -Python sandbox runners for executing code in isolation aka snekbox - -The user sends a piece of python code to a snekbox, the snekbox executes the code and sends the result back to the users. - -``` - +-------------+ +------------+ +-----------+ - input -> | |---------->| |-------->| | >----------+ - | WEBSERVER | | RABBITMQ | | SNEKBOX | execution | -result <- | |<----------| |<--------| | <----------+ - +-------------+ +------------+ +-----------+ - ^ ^ ^ - | | |- Executes python code - | | |- Returns result - | | +----------------------- - | | - | |- Message queues opens on demand and closes automatically - | +--------------------------------------------------------- +Python sandbox runners for executing code in isolation aka snekbox. + +A client sends Python code to a snekbox, the snekbox executes the code, and finally the results of the execution are returned to the client. + +``` + +-------------+ +-----------+ + input -> | |---------->| | >----------+ + | HTTP POST | | SNEKBOX | execution | +result <- | |<----------| | <----------+ + +-------------+ +-----------+ + ^ ^ + | |- Executes python code + | |- Returns result + | +----------------------- | - |- Uses websockets for asynchronous connection between webui and webserver - +------------------------------------------------------------------------- + |- HTTP POST Endpoint receives request and returns result + +--------------------------------------------------------- ``` +The code is executed in a Python process that is launched through [NsJail](https://github.com/google/nsjail), which is responsible for sandboxing the Python process. NsJail is configured as follows: -## Dependencies +* Root directory is mounted as read-only +* Time limit of 2 seconds +* Maximum of 1 PID +* Maximum memory of 52428800 bytes +* Loopback interface is down +* procfs is disabled -| dep | version (or greater) | -|----------------|:---------------------| -| python | 3.6.5 | -| pip | 10.0.1 | -| pipenv | 2018.05.18 | -| docker | 18.03.1-ce | -| docker-compose | 1.21.2 | -| nsjail | 2.5 | +The Python process is configured as follows: -_________________________________________ -## Setup local test +* Version 3.7.4 +* Isolated mode + * Neither the script's directory nor the user's site packages are in `sys.path` + * All `PYTHON*` environment variables are ignored -install python packages -```bash -apt-get install -y libprotobuf-dev #needed by nsjail -pipenv sync --dev -``` +## HTTP REST API -## NSJail +Communication with snekbox is done over a HTTP REST API. The framework for the HTTP REST API is [Falcon](https://falconframework.org/) and the WSGI being used is [Gunicorn](https://gunicorn.org/). By default, the server is hosted on `0.0.0.0:8060` with two workers. -Copy the appropriate binary to an appropriate path +See [`snekapi.py`](snekbox/api/snekapi.py) and [`resources`](snekbox/api/resources) for API documentation. -```bash -cp binaries/nsjail2.6-ubuntu-x86_64 /usr/bin/nsjail -chmod +x /usr/bin/nsjail -``` +## Development Environment -give nsjail a test run +### Initial Setup -```bash -# This is a workaround because nsjail can't create the directories automatically -sudo mkdir -p /sys/fs/cgroup/pids/NSJAIL \ - && mkdir -p /sys/fs/cgroup/memory/NSJAIL +A Python 3.7 interpreter and the [pipenv](https://docs.pipenv.org/en/latest/) package are required. Once those requirements are satisfied, install the project's dependencies: -nsjail -Mo \ ---rlimit_as 700 \ ---chroot / \ --E LANG=en_US.UTF-8 \ --R/usr -R/lib -R/lib64 \ ---user nobody \ ---group nogroup \ ---time_limit 2 \ ---disable_proc \ ---iface_no_lo \ ---cgroup_pids_max=1 \ ---cgroup_mem_max=52428800 \ ---quiet -- \ -python3.6 -ISq -c "print('test')" +``` +pipenv --sync ``` -> if it fails, try without the `--cgroup_pids_max=1` and `--cgroup_mem_max=52428800` +Follow that up with setting up the pre-commit hook: -## Development environment +``` +pipenv run precommit +``` -Start a rabbitmq instance and get the container IP +Now Flake8 will run and lint staged changes whenever an attempt to commit the changes is made. Flake8 can still be invoked manually: -```bash -docker-compose up -d pdrmq -docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' rmq -# expected output with default setting: 172.17.0.2 -# If not, change the config.py file to match +``` +pipenv run lint ``` -rabbitmq webinterface: `http://localhost:15672` +### Running snekbox -start the webserver +The Docker images can be built with: -```bash -docker-compose up -d pdsnkweb -netstat -plnt -# tcp 0.0.0.0:5000 LISTEN +``` +pipenv run buildbase +pipenv run buildvenv +pipenv run build ``` -`http://localhost:5000` +Use Docker Compose to start snekbox: -```bash -pipenv run snekbox # for debugging -# or -docker-compose up pdsnk # for running the container +``` +docker-compose up ``` -________________________________________ -## Unit testing and lint +### Running Tests -Make sure rabbitmq is running before running tests +Tests are run through coverage.py using unittest. Before tests can run, the dev venv Docker image has to be built: -```bash -pipenv run lint +``` +pipenv run builddev +``` + +Alternatively, the following command will build the image and then run the tests: + +``` +pipenv run testb +``` + +If the image doesn't need to be built, the tests can be run with: + +``` pipenv run test ``` -________________________________________ -## Build the containers +### Coverage -```bash -# Build -pipenv run buildbox -pipenv run buildweb +To see a coverage report, run + +``` +pipenv run report +``` + +Alternatively, a report can be generated as HTML: + +``` +pipenv run coverage html +``` + +The HTML will output to `./htmlcov/` by default + + +### The `devsh` Helper Script + +This script starts an `ash` shell inside the venv Docker container and attaches to it. Unlike the production image, the venv image that is built by this script contains dev dependencies too. The project directory is mounted inside the container so any filesystem changes made inside the container affect the actual local project. + +#### Usage -# Push -pipenv run pushbox -pipenv run pushweb ``` +pipenv run devsh [--build [--clean]] [ash_args ...] +``` + +* `--build` Build the venv Docker image +* `--clean` Clean up dangling Docker images (only works if `--build` precedes it) +* `ash_args` Arguments to pass to `/bin/ash` (for example `-c "echo hello"`). An interactive shell is launched if no arguments are given + +#### Invoking NsJail +A shell alias named `nsjpy` is included and is basically `nsjail python -c <args>` but NsJail is configured as it would be if snekbox invoked it (such as the time and memory limits). It provides an easy way to run Python code inside NsJail without the need to run snekbox with its webserver and send HTTP requests. Example usage: + +```bash +nsjpy "print('hello world!')" +``` +The alias can be found in `./scripts/.profile`, which is automatically added when the shell is launched in the container. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e423b28..424b1a3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,50 +1,228 @@ # https://aka.ms/yaml jobs: -- job: test - displayName: 'Lint' + - job: test + displayName: 'Lint & Test' - pool: - vmImage: 'Ubuntu-16.04' + pool: + vmImage: 'ubuntu-16.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Set Python version' - inputs: - versionSpec: '3.6.x' - addToPath: true + steps: + - task: ShellScript@2 + displayName: 'Check If Images Need to Be Built' + name: check + inputs: + scriptPath: scripts/check_dockerfiles.sh - - script: pip3 install pipenv - displayName: 'Install pipenv' + # Without a login the following Docker build tasks won't add image tags. + - task: Docker@2 + displayName: 'Log into Docker Hub' + inputs: + command: login + containerRegistry: DockerHubV2 - - script: pipenv install --dev --deploy --system - displayName: 'Install project using pipenv' + # The venv image depends on this image. Build it if it can't be pulled + # from Docker Hub, which will be the case if the base Dockerfile has had + # changes. + - task: Docker@2 + displayName: 'Build Base Image' + condition: and(succeeded(), ne(variables['check.BASE_PULL'], True)) + inputs: + command: build + repository: pythondiscord/snekbox-base + tags: latest + Dockerfile: docker/base.Dockerfile + buildContext: . - - script: python3 -m flake8 --format junit-xml --output-file test-lint.xml - displayName: 'Run linter' + # The dev image is never pushed and therefore is always built. + - task: Docker@2 + displayName: 'Build Development Image' + inputs: + command: build + repository: pythondiscord/snekbox-venv + tags: dev + Dockerfile: docker/venv.Dockerfile + buildContext: . + arguments: --build-arg DEV=1 - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - testRunTitle: 'Snekbox Flake8 Lint Results' + # The linter and all tests run inside this container. + - script: | + docker run \ + --tty \ + --detach \ + --name snekbox_test \ + --privileged \ + --network host \ + --hostname pdsnk-dev \ + -e PYTHONDONTWRITEBYTECODE=1 \ + -e PIPENV_PIPFILE="/snekbox/Pipfile" \ + -e ENV="${PWD}/scripts/.profile" \ + --volume "${PWD}":"${PWD}" \ + --workdir "${PWD}"\ + --entrypoint /bin/ash \ + pythondiscord/snekbox-venv:dev + displayName: 'Start Container' -- job: build - displayName: 'Build' - dependsOn: test - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + - script: | + docker exec snekbox_test /bin/ash -c \ + 'pipenv run lint --format junit-xml --output-file test-lint.xml' + displayName: 'Run Linter' - steps: - - task: Docker@1 - displayName: 'Login: Docker Hub' + - task: PublishTestResults@2 + displayName: 'Publish Lint Results' + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/test-lint.xml' + testRunTitle: 'Lint Results' - inputs: - containerregistrytype: 'Container Registry' - dockerRegistryEndpoint: 'DockerHub' - command: 'login' + # Memory limit tests would fail if this isn't disabled. + - script: sudo swapoff -a + displayName: 'Disable Swap Memory' - - script: docker build -t pythondiscord/snekbox:latest -f docker/Dockerfile . - displayName: 'Build Final Image' + - script: | + docker exec snekbox_test /bin/ash -c \ + 'pipenv run coverage run -m xmlrunner' + displayName: 'Run Unit Tests' - - script: docker push pythondiscord/snekbox:latest - displayName: 'Push Image to Dockerhub' + - task: PublishTestResults@2 + displayName: 'Publish Test Results' + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/TEST-*.xml' + testRunTitle: 'Test Results' + + # Run report too because the XML report doesn't output to stdout. + - script: | + docker exec snekbox_test /bin/ash -c \ + 'pipenv run /bin/ash -c "coverage report && coverage xml"' + displayName: 'Generate Coverage Report' + + - task: PublishCodeCoverageResults@1 + displayName: 'Publish Coverage Results' + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '**/coverage.xml' + + # When a pull request, only perform this job if images need to be built. + # It's always performed for non-PRs because the final image will always need + # to be built. + - job: build + displayName: 'Build' + condition: > + and( + succeeded(), + or( + ne(variables['Build.Reason'], 'PullRequest'), + eq(coalesce(dependencies.test.outputs['check.BASE_CHANGED'], True), True), + eq(coalesce(dependencies.test.outputs['check.VENV_CHANGED'], True), True) + ) + ) + dependsOn: test + + # coalesce() gives variables default values if they are null (i.e. unset). + variables: + BASE_CHANGED: $[ coalesce(dependencies.test.outputs['check.BASE_CHANGED'], True) ] + VENV_CHANGED: $[ coalesce(dependencies.test.outputs['check.VENV_CHANGED'], True) ] + BASE_PULL: $[ coalesce(dependencies.test.outputs['check.BASE_PULL'], False) ] + + steps: + - task: Docker@2 + displayName: 'Log into Docker Hub' + inputs: + command: login + containerRegistry: DockerHubV2 + + # Because this is the base image for the venv image, if the venv needs to + # be built, this base image must also be present. Build it if it has + # changed or can't be pulled from Docker Hub. + - task: Docker@2 + displayName: 'Build Base Image' + condition: > + and( + succeeded(), + ne(variables.BASE_PULL, True), + or( + eq(variables.BASE_CHANGED, True), + eq(variables.VENV_CHANGED, True) + ) + ) + inputs: + command: build + repository: pythondiscord/snekbox-base + tags: latest + Dockerfile: docker/base.Dockerfile + buildContext: . + + # Also build this image if base has changed - even if this image hasn't. + - task: Docker@2 + displayName: 'Build Virtual Environment Image' + condition: > + and( + succeeded(), + or( + eq(variables.BASE_CHANGED, True), + eq(variables.VENV_CHANGED, True) + ) + ) + inputs: + command: build + repository: pythondiscord/snekbox-venv + tags: latest + Dockerfile: docker/venv.Dockerfile + buildContext: . + + # Always build this image unless it's for a pull request. + - task: Docker@2 + displayName: 'Build Final Image' + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + inputs: + command: build + repository: pythondiscord/snekbox + tags: latest + Dockerfile: docker/Dockerfile + buildContext: . + + # Push images only after they've all successfully been built. + # These have the same conditions as the build tasks. However, for safety, + # a condition for not being a pull request is added. + - task: Docker@2 + displayName: 'Push Base Image' + condition: > + and( + succeeded(), + ne(variables['Build.Reason'], 'PullRequest'), + ne(variables.BASE_PULL, True), + or( + eq(variables.BASE_CHANGED, True), + eq(variables.VENV_CHANGED, True) + ) + ) + inputs: + command: push + repository: pythondiscord/snekbox-base + tags: latest + + - task: Docker@2 + displayName: 'Push Virtual Environment Image' + condition: > + and( + succeeded(), + ne(variables['Build.Reason'], 'PullRequest'), + or( + eq(variables.BASE_CHANGED, True), + eq(variables.VENV_CHANGED, True) + ) + ) + inputs: + command: push + repository: pythondiscord/snekbox-venv + tags: latest + + - task: Docker@2 + displayName: 'Push Final Image' + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + inputs: + command: push + repository: pythondiscord/snekbox + tags: latest diff --git a/binaries/nsjail2.5-alpine-x86_64 b/binaries/nsjail2.5-alpine-x86_64 Binary files differdeleted file mode 100644 index 9af91fc..0000000 --- a/binaries/nsjail2.5-alpine-x86_64 +++ /dev/null diff --git a/binaries/nsjail2.6-ubuntu-x86_64 b/binaries/nsjail2.6-ubuntu-x86_64 Binary files differdeleted file mode 100644 index d8df21b..0000000 --- a/binaries/nsjail2.6-ubuntu-x86_64 +++ /dev/null diff --git a/config.py b/config.py deleted file mode 100644 index 5ca23bb..0000000 --- a/config.py +++ /dev/null @@ -1,34 +0,0 @@ -import os - -import docker -from docker.errors import NotFound - - -def autodiscover(): - """Search for the snekbox container and return its IPv4 address.""" - container_names = ["rmq", "pdrmq", "snekbox_pdrmq_1"] - - client = docker.from_env() - for name in container_names: - try: - container = client.containers.get(name) - if container.status == "running": - host = list(container.attrs.get('NetworkSettings').get('Networks').values()) - host = host[0]['IPAddress'] - return host - except NotFound: - continue - except Exception: - pass - - return '127.0.0.1' - - -USERNAME = os.environ.get('RMQ_USERNAME', 'guest') -PASSWORD = os.environ.get('RMQ_PASSWORD', 'guest') -HOST = os.environ.get('RMQ_HOST', autodiscover()) -PORT = 5672 -QUEUE = 'input' -EXCHANGE = QUEUE -ROUTING_KEY = QUEUE -EXCHANGE_TYPE = 'direct' diff --git a/docker-compose.yml b/docker-compose.yml index 3aedf14..d071a71 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,43 +1,11 @@ -version: '3' +version: "3.7" services: - pdrmq: - hostname: "pdrmq" - image: pythondiscord/rmq:latest - expose: - - "15672" - ports: - - "15672:15672" - networks: - - sneknet - environment: - RABBITMQ_DEFAULT_USER: guest - RABBITMQ_DEFAULT_PASS: guest - pdsnk: - privileged: true hostname: "pdsnk" + privileged: true image: pythondiscord/snekbox:latest - networks: - - sneknet - environment: - RMQ_HOST: pdrmq - RMQ_USERNAME: guest - RMQ_PASSWORD: guest - - pdsnkweb: - hostname: "pdsnkweb" - image: pythondiscord/snekboxweb:latest - networks: - - sneknet - ports: - - "5000:5000" - expose: - - "5000" - environment: - RMQ_HOST: pdrmq - RMQ_USERNAME: guest - RMQ_PASSWORD: guest - - -networks: - sneknet: + network_mode: "host" + init: true + build: + context: . + dockerfile: docker/Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile index e8fa8a5..5ef8a88 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,10 +1,7 @@ -FROM pythondiscord/snekbox-base:latest +FROM pythondiscord/snekbox-venv:latest + +ENTRYPOINT ["pipenv", "run"] +CMD ["snekbox"] -RUN mkdir -p /snekbox COPY . /snekbox WORKDIR /snekbox - -RUN pipenv --rm -RUN pipenv sync - -CMD ["pipenv", "run", "snekbox"] diff --git a/docker/Dockerfile.webapp b/docker/Dockerfile.webapp deleted file mode 100644 index 988926d..0000000 --- a/docker/Dockerfile.webapp +++ /dev/null @@ -1,25 +0,0 @@ -FROM python:3.6.6-alpine3.7 - -RUN apk add --update tini -RUN apk add --update build-base - -ENV PIPENV_VENV_IN_PROJECT=1 -ENV PIPENV_IGNORE_VIRTUALENVS=1 -ENV PIPENV_NOSPIN=1 -ENV PIPENV_HIDE_EMOJIS=1 -ENV PYTHONPATH=/webapp - -RUN pip install pipenv - -RUN mkdir -p /webapp -COPY Pipfile /webapp -COPY Pipfile.lock /webapp -COPY . /webapp -WORKDIR /webapp - -RUN pipenv sync --dev - -EXPOSE 5000 - -ENTRYPOINT ["/sbin/tini", "--"] -CMD ["pipenv", "run", "snekweb"] diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index cdbd98e..1edff49 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -1,23 +1,27 @@ -FROM python:3.6.6-alpine3.7 +FROM alpine:3.10 as builder +RUN apk add --no-cache --update \ + bison~=3.3 \ + bsd-compat-headers~=0.7 \ + flex~=2.6 \ + g++~=8.3 \ + gcc~=8.3 \ + git~=2.22 \ + libnl3-dev~=3.4 \ + linux-headers~=4.19 \ + make~=4.2 \ + protobuf-dev~=3.6 +RUN git clone https://github.com/google/nsjail.git /nsjail \ + && cd /nsjail \ + && git checkout 0b1d5ac03932c140f08536ed72b4b58741e7d3cf +WORKDIR /nsjail +RUN make -RUN apk add --no-cache libstdc++ protobuf -RUN apk add --update build-base - -ENV PIPENV_VENV_IN_PROJECT=1 -ENV PIPENV_IGNORE_VIRTUALENVS=1 -ENV PIPENV_NOSPIN=1 -ENV PIPENV_HIDE_EMOJIS=1 -ENV PYTHONPATH=/snekbox - -RUN pip install pipenv - -RUN mkdir -p /snekbox -COPY Pipfile /snekbox -COPY Pipfile.lock /snekbox -COPY . /snekbox -WORKDIR /snekbox - -RUN pipenv sync --dev - -RUN cp binaries/nsjail2.5-alpine-x86_64 /usr/sbin/nsjail +FROM python:3.7.4-alpine3.10 +ENV PIP_NO_CACHE_DIR=false +RUN apk add --no-cache --update \ + libnl3~=3.4 \ + libstdc++~=8.3 \ + protobuf~=3.6 +RUN pip install pipenv==2018.11.26 +COPY --from=builder /nsjail/nsjail /usr/sbin/ RUN chmod +x /usr/sbin/nsjail diff --git a/docker/venv.Dockerfile b/docker/venv.Dockerfile new file mode 100644 index 0000000..85188fd --- /dev/null +++ b/docker/venv.Dockerfile @@ -0,0 +1,13 @@ +FROM pythondiscord/snekbox-base:latest + +ARG DEV +ENV PIP_NO_CACHE_DIR=false \ + PIPENV_DONT_USE_PYENV=1 \ + PIPENV_HIDE_EMOJIS=1 \ + PIPENV_NOSPIN=1 \ + PIPENV_VENV_IN_PROJECT=1 + +COPY Pipfile Pipfile.lock /snekbox/ +WORKDIR /snekbox + +RUN if [ -n "${DEV}" ]; pipenv sync --dev; then pipenv sync; fi diff --git a/logs.py b/logs.py deleted file mode 100644 index fc6070e..0000000 --- a/logs.py +++ /dev/null @@ -1,10 +0,0 @@ -import logging -import sys - -logformat = logging.Formatter(fmt='[%(asctime)s] [%(process)s] [%(levelname)s] %(message)s', - datefmt='%Y-%m-%d %H:%M:%S %z') -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) -console = logging.StreamHandler(sys.stdout) -console.setFormatter(logformat) -log.addHandler(console) @@ -1,111 +0,0 @@ -import time -import traceback - -import pika -from pika.exceptions import ConnectionClosed - -from config import EXCHANGE, EXCHANGE_TYPE, HOST, PASSWORD, PORT, QUEUE, ROUTING_KEY, USERNAME -from logs import log - - -class Rmq: - """Rabbit MQ (RMQ) implementation used for communication with the bot.""" - - def __init__(self): - self.credentials = pika.PlainCredentials(USERNAME, PASSWORD) - self.con_params = pika.ConnectionParameters(HOST, PORT, '/', self.credentials) - self.properties = pika.BasicProperties(content_type='text/plain', delivery_mode=1) - - def _declare(self, channel, queue): - channel.queue_declare( - queue=queue, - durable=False, # Do not commit messages to disk - arguments={'x-message-ttl': 5000}, # Delete message automatically after x milliseconds - auto_delete=True) # Delete queue when all connection are closed - - def consume(self, queue=QUEUE, callback=None, thread_ws=None, run_once=False): - """Subscribe to read from a RMQ channel.""" - while True: - try: - connection = pika.BlockingConnection(self.con_params) - - try: - channel = connection.channel() - self._declare(channel, queue) - channel.basic_qos(prefetch_count=1) - - if not run_once: - channel.basic_consume( - lambda ch, method, properties, body: - callback(ch, method, properties, body, thread_ws=thread_ws), - queue=queue) - - log.info(f"Connected to host: {HOST} port: {PORT} queue: {queue}") - - if thread_ws: - if not thread_ws.closed: - thread_ws.send('{"service": "connected"}') - - if run_once: - return channel.basic_get(queue=queue) - - channel.start_consuming() - - except Exception: - exc = traceback.format_exc() - log.error(exc) - - finally: - connection.close() - - except ConnectionClosed: - if thread_ws: - if not thread_ws.closed: - log.error(f"Connection to {HOST} could not be established") - thread_ws.send('{"service": "disconnected"}') - exit(1) - - log.error(f"Connection lost, reconnecting to {HOST}") - - time.sleep(2) - - def publish(self, message, queue=QUEUE, routingkey=ROUTING_KEY, exchange=EXCHANGE): - """Open a connection to publish (write) to a RMQ channel.""" - try: - connection = pika.BlockingConnection(self.con_params) - - try: - channel = connection.channel() - - self._declare(channel, queue) - - channel.exchange_declare( - exchange=exchange, - exchange_type=EXCHANGE_TYPE) - - channel.queue_bind( - exchange=exchange, - queue=queue, - routing_key=routingkey) - - result = channel.basic_publish( - exchange=exchange, - routing_key=routingkey, - body=message, - properties=self.properties) - - if result: - return result - - else: - log.error(f"Message '{message}' not delivered") - - except ConnectionClosed: - log.error(f"Could not send message, connection to {HOST} was lost") - exit(1) - - finally: - connection.close() - - except ConnectionClosed: - log.error(f"Could not connect to {HOST}") diff --git a/scripts/.profile b/scripts/.profile new file mode 100644 index 0000000..bd46a17 --- /dev/null +++ b/scripts/.profile @@ -0,0 +1,32 @@ +nsjpy() { + local MEM_MAX=52428800 + + # All arguments except the last are considered to be for NsJail, not Python. + local nsj_args="" + while [ "$#" -gt 1 ]; do + nsj_args="${nsj_args:+${nsj_args} }$1" + shift + done + + # Set up cgroups and disable memory swapping. + mkdir -p /sys/fs/cgroup/pids/NSJAIL + mkdir -p /sys/fs/cgroup/memory/NSJAIL + echo "${MEM_MAX}" > /sys/fs/cgroup/memory/NSJAIL/memory.limit_in_bytes + echo "${MEM_MAX}" > /sys/fs/cgroup/memory/NSJAIL/memory.memsw.limit_in_bytes + + nsjail \ + -Mo \ + --rlimit_as 700 \ + --chroot / \ + -E LANG=en_US.UTF-8 \ + -R/usr -R/lib -R/lib64 \ + --user 65534 \ + --group 65534 \ + --time_limit 2 \ + --disable_proc \ + --iface_no_lo \ + --cgroup_pids_max=1 \ + --cgroup_mem_max="${MEM_MAX}" \ + $nsj_args -- \ + /snekbox/.venv/bin/python3 -Iq -c "$@" +} diff --git a/scripts/check_dockerfiles.sh b/scripts/check_dockerfiles.sh new file mode 100755 index 0000000..c84c61f --- /dev/null +++ b/scripts/check_dockerfiles.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +set -euo pipefail +exec 3>&1 # New file descriptor to stdout + +BASE_URL="https://dev.azure.com/\ +python-discord/${SYSTEM_TEAMPROJECTID}/_apis/build/builds?\ +queryOrder=finishTimeDescending&\ +resultFilter=succeeded&\ +\$top=1&\ +repositoryType=${BUILD_REPOSITORY_PROVIDER}&\ +repositoryId=${BUILD_REPOSITORY_NAME}&\ +api-version=5.0" + +get_build() { + set -e # Poor Ubuntu LTS doesn't have Bash 4.4's inherit_errexit + + local branch="${1:?"get_build: argument 1 'branch' is unset"}" + local url="${BASE_URL}&branchName=${branch}" + + printf '%s\n' "Retrieving the latest successful build using ${url}" >&3 + + local response + response="$(curl -sSL "${url}")" + + if [[ -z "${response}" ]] \ + || ! count="$(printf '%s' "${response}" | jq -re '.count')" \ + || (( "${count}" < 1 )) + then + return 1 + else + printf '%s' "${response}" + fi +} + +# Get the previous commit +if [[ "${BUILD_REASON}" = "PullRequest" ]]; then + if ! prev_commit="$( + get_build "${BUILD_SOURCEBRANCH}" \ + | jq -re '.value[0].triggerInfo."pr.sourceSha"' + )" + then + echo \ + "Could not retrieve the previous build's commit." \ + "Falling back to the head of the target branch." + + prev_commit="origin/${SYSTEM_PULLREQUEST_TARGETBRANCH}" + fi +elif ! prev_commit="$( + get_build "${BUILD_SOURCEBRANCH}" \ + | jq -re '.value[0].sourceVersion' + )" +then + echo \ + "No previous build was found." \ + "Either the previous build is too old and was deleted" \ + "or the branch was empty before this build." \ + "All images will be built." + exit 0 +fi + +# Compare diffs +head="$(git rev-parse HEAD)" +printf '%s\n' "Comparing HEAD (${head}) against ${prev_commit}." + +if git diff --quiet "${prev_commit}" -- docker/base.Dockerfile; then + echo "No changes detected in docker/base.Dockerfile." + echo "##vso[task.setvariable variable=BASE_CHANGED;isOutput=true]False" +else + # Always rebuild the venv if the base changes. + exit 0 +fi + +if git diff --quiet "${prev_commit}" -- docker/venv.Dockerfile Pipfile*; then + echo "No changes detected in docker/venv.Dockerfile or the Pipfiles." + echo "##vso[task.setvariable variable=VENV_CHANGED;isOutput=true]False" +elif master_commit="$( + get_build "refs/heads/master" \ + | jq -re '.value[0].sourceVersion' + )" \ + && git diff --quiet "${master_commit}" -- docker/base.Dockerfile +then + # Though base image hasn't changed, it's still needed to build the venv. + echo "Can pull base image from Docker Hub; no changes made since master." + echo "##vso[task.setvariable variable=BASE_PULL;isOutput=true]True" +fi diff --git a/scripts/dev.sh b/scripts/dev.sh new file mode 100755 index 0000000..8f5b24f --- /dev/null +++ b/scripts/dev.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env sh + +# Sets up a development environment and runs a shell in a docker container. +# Usage: dev.sh [--build [--clean]] [ash_args ...] + +if [ "$1" = "--build" ]; then + shift + printf "Building pythondiscord/snekbox-venv:dev..." + + docker build \ + -t pythondiscord/snekbox-venv:dev \ + -f docker/venv.Dockerfile \ + --build-arg DEV=1 \ + -q \ + . \ + >/dev/null \ + && printf " done!\n" || exit "$?" + + if [ "$1" = "--clean" ]; then + shift + dangling_imgs=$(docker images -f "dangling=true" -q) + + if [ -n "${dangling_imgs}" ]; then + printf "Removing dangling images..." + + docker rmi $dangling_imgs >/dev/null \ + && printf " done!\n" || exit "$?" + fi + fi +fi + +# Keep the container up in the background so it doesn't have to be restarted +# for the ownership fix. +# The volume is mounted to same the path in the container as the source +# directory on the host to ensure coverage can find the source files. +docker run \ + --tty \ + --detach \ + --name snekbox_test \ + --privileged \ + --network host \ + --hostname pdsnk-dev \ + -e PYTHONDONTWRITEBYTECODE=1 \ + -e PIPENV_PIPFILE="/snekbox/Pipfile" \ + -e ENV="${PWD}/scripts/.profile" \ + --volume "${PWD}":"${PWD}" \ + --workdir "${PWD}"\ + --entrypoint /bin/ash \ + pythondiscord/snekbox-venv:dev \ + >/dev/null \ + +# Execute the given command(s) +docker exec -it snekbox_test /bin/ash "$@" + +# Fix ownership of coverage file +# BusyBox doesn't support --reference for chown +docker exec \ + -it \ + -e CWD="${PWD}" \ + snekbox_test \ + /bin/ash \ + -c 'chown "$(stat -c "%u:%g" "${CWD}")" "${CWD}/.coverage"' + +docker rm -f snekbox_test >/dev/null # Stop and remove the container diff --git a/snekbox.py b/snekbox.py deleted file mode 100644 index 7491672..0000000 --- a/snekbox.py +++ /dev/null @@ -1,140 +0,0 @@ -import json -import multiprocessing -import os -import subprocess -import sys - -from rmq import Rmq - - -class Snekbox: - """Core snekbox functionality, providing safe execution of Python code.""" - - def __init__(self, - nsjail_binary='nsjail', - python_binary=os.path.dirname(sys.executable) + os.sep + 'python3.6'): - self.nsjail_binary = nsjail_binary - self.python_binary = python_binary - self._nsjail_workaround() - - env = { - 'PATH': ( - '/snekbox/.venv/bin:/usr/local/bin:/usr/local/' - 'sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' - ), - 'LANG': 'en_US.UTF-8', - 'PYTHON_VERSION': '3.6.5', - 'PYTHON_PIP_VERSION': '10.0.1', - 'PYTHONDONTWRITEBYTECODE': '1', - } - - def _nsjail_workaround(self): - dirs = ['/sys/fs/cgroup/pids/NSJAIL', '/sys/fs/cgroup/memory/NSJAIL'] - for d in dirs: - if not os.path.exists(d): - os.makedirs(d) - - def python3(self, cmd): - """ - Execute Python 3 code in a isolated environment. - - The value of ``cmd`` is passed using '-c' to a Python - interpreter that is started in a ``nsjail``, isolating it - from the rest of the system. - - Returns the output of executing the command (stdout) if - successful, or a error message if the execution failed. - """ - args = [self.nsjail_binary, '-Mo', - '--rlimit_as', '700', - '--chroot', '/', - '-E', 'LANG=en_US.UTF-8', - '-R/usr', '-R/lib', '-R/lib64', - '--user', 'nobody', - '--group', 'nogroup', - '--time_limit', '2', - '--disable_proc', - '--iface_no_lo', - '--cgroup_pids_max=1', - '--cgroup_mem_max=52428800', - '--quiet', '--', - self.python_binary, '-ISq', '-c', cmd] - try: - proc = subprocess.Popen(args, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=self.env, - universal_newlines=True) - except ValueError: - return 'ValueError: embedded null byte' - - stdout, stderr = proc.communicate() - if proc.returncode == 0: - output = stdout - - elif proc.returncode == 1: - try: - filtered = [] - for line in stderr.split('\n'): - if not line.startswith('['): - filtered.append(line) - output = '\n'.join(filtered) - except IndexError: - output = '' - - elif proc.returncode == 109: - return 'timed out or memory limit exceeded' - - elif proc.returncode == 255: - return 'permission denied (root required)' - - elif proc.returncode: - return f'unknown error, code: {proc.returncode}' - - else: - return 'unknown error, no error code' - - return output - - def execute(self, body): - """ - Handles execution of a raw JSON-formatted RMQ message, contained in ``body``. - - The message metadata, including the Python code to be executed, is - extracted from the message body. The code is then executed in the - isolated environment, and the results of the execution published - to RMQ. Once published, the system exits, since the snekboxes - are created and disposed of per-execution. - """ - msg = body.decode('utf-8') - result = '' - snek_msg = json.loads(msg) - snekid = snek_msg['snekid'] - snekcode = snek_msg['message'].strip() - - result = self.python3(snekcode) - - rmq.publish(result, - queue=snekid, - routingkey=snekid, - exchange=snekid) - exit(0) - - def message_handler(self, ch, method, properties, body, thread_ws=None): - """Spawns a daemon process that handles RMQ messages.""" - p = multiprocessing.Process(target=self.execute, args=(body,)) - p.daemon = True - p.start() - - ch.basic_ack(delivery_tag=method.delivery_tag) - - -if __name__ == '__main__': - try: - rmq = Rmq() - snkbx = Snekbox() - rmq.consume(callback=snkbx.message_handler) - except KeyboardInterrupt: - print('Exited') - exit(0) diff --git a/snekbox/__init__.py b/snekbox/__init__.py new file mode 100644 index 0000000..40b76db --- /dev/null +++ b/snekbox/__init__.py @@ -0,0 +1,40 @@ +import logging +import os +import sys + +from gunicorn import glogging + +DEBUG = os.environ.get("DEBUG", False) + + +class GunicornLogger(glogging.Logger): + """Logger for Gunicorn with custom formatting and support for the DEBUG environment variable.""" + + error_fmt = "%(asctime)s | %(process)5s | %(name)30s | %(levelname)8s | %(message)s" + access_fmt = error_fmt + datefmt = None # Use the default ISO 8601 format + + def setup(self, cfg): + """ + Set up loggers and set error logger's level to DEBUG if the DEBUG env var is set. + + Note: Access and syslog handlers would need to be recreated to use a custom date format + because they are created with an unspecified datefmt argument by default. + """ + super().setup(cfg) + + if DEBUG: + self.loglevel = logging.DEBUG + else: + self.loglevel = self.LOG_LEVELS.get(cfg.loglevel.lower(), logging.INFO) + + self.error_log.setLevel(self.loglevel) + + +log = logging.getLogger("snekbox") +log.setLevel(logging.DEBUG if DEBUG else logging.INFO) +log.propagate = True +formatter = logging.Formatter(GunicornLogger.error_fmt) +handler = logging.StreamHandler(sys.stdout) +handler.setFormatter(formatter) +log.addHandler(handler) diff --git a/snekbox/api/__init__.py b/snekbox/api/__init__.py new file mode 100644 index 0000000..d106f26 --- /dev/null +++ b/snekbox/api/__init__.py @@ -0,0 +1,3 @@ +from .snekapi import SnekAPI + +__all__ = ("SnekAPI",) diff --git a/snekbox/api/app.py b/snekbox/api/app.py new file mode 100644 index 0000000..c71e246 --- /dev/null +++ b/snekbox/api/app.py @@ -0,0 +1,3 @@ +from . import SnekAPI + +application = SnekAPI() diff --git a/snekbox/api/resources/__init__.py b/snekbox/api/resources/__init__.py new file mode 100644 index 0000000..fe422b8 --- /dev/null +++ b/snekbox/api/resources/__init__.py @@ -0,0 +1,3 @@ +from .eval import EvalResource + +__all__ = ("EvalResource",) diff --git a/snekbox/api/resources/eval.py b/snekbox/api/resources/eval.py new file mode 100644 index 0000000..c4bd666 --- /dev/null +++ b/snekbox/api/resources/eval.py @@ -0,0 +1,83 @@ +import logging + +import falcon +from falcon.media.validators.jsonschema import validate + +from snekbox.nsjail import NsJail + +log = logging.getLogger(__name__) + + +class EvalResource: + """ + Evaluation of Python code. + + Supported methods: + + - POST /eval + Evaluate Python code and return the result + """ + + REQ_SCHEMA = { + "type": "object", + "properties": { + "input": { + "type": "string" + } + }, + "required": [ + "input" + ] + } + + def __init__(self): + self.nsjail = NsJail() + + @validate(REQ_SCHEMA) + def on_post(self, req, resp): + """ + Evaluate Python code and return stdout, stderr, and the return code. + + The return codes mostly resemble those of a Unix shell. Some noteworthy cases: + + - None + The NsJail process failed to launch + - 137 (SIGKILL) + Typically because NsJail killed the Python process due to time or memory constraints + - 255 + NsJail encountered a fatal error + + Request body: + + >>> { + ... "input": "print(1 + 1)" + ... } + + Response format: + + >>> { + ... "stdout": "2\\n", + ... "returncode": 0 + ... } + + Status codes: + + - 200 + Successful evaluation; not indicative that the input code itself works + - 400 + Input's JSON schema is invalid + - 415 + Unsupported content type; only application/JSON is supported + """ + code = req.media["input"] + + try: + result = self.nsjail.python3(code) + except Exception: + log.exception("An exception occurred while trying to process the request") + raise falcon.HTTPInternalServerError + + resp.media = { + "stdout": result.stdout, + "returncode": result.returncode + } diff --git a/snekbox/api/snekapi.py b/snekbox/api/snekapi.py new file mode 100644 index 0000000..cb0356a --- /dev/null +++ b/snekbox/api/snekapi.py @@ -0,0 +1,26 @@ +import falcon + +from .resources import EvalResource + + +class SnekAPI(falcon.API): + """ + The main entry point to the snekbox JSON API. + + Routes: + + - /eval + Evaluation of Python code + + Error response format: + + >>> { + ... "title": "Unsupported media type", + ... "description": "application/xml is an unsupported media type." + ... } + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.add_route("/eval", EvalResource()) diff --git a/snekbox/nsjail.py b/snekbox/nsjail.py new file mode 100644 index 0000000..d980f09 --- /dev/null +++ b/snekbox/nsjail.py @@ -0,0 +1,160 @@ +import logging +import os +import re +import subprocess +import sys +import textwrap +from pathlib import Path +from subprocess import CompletedProcess +from tempfile import NamedTemporaryFile +from typing import Iterable + +from snekbox import DEBUG + +log = logging.getLogger(__name__) + +# [level][timestamp][PID]? function_signature:line_no? message +LOG_PATTERN = re.compile( + r"\[(?P<level>(I)|[DWEF])\]\[.+?\](?(2)|(?P<func>\[\d+\] .+?:\d+ )) ?(?P<msg>.+)" +) +LOG_BLACKLIST = ("Process will be ",) + +# Explicitly define constants for NsJail's default values. +CGROUP_PIDS_PARENT = Path("/sys/fs/cgroup/pids/NSJAIL") +CGROUP_MEMORY_PARENT = Path("/sys/fs/cgroup/memory/NSJAIL") + +NSJAIL_PATH = os.getenv("NSJAIL_PATH", "/usr/sbin/nsjail") +MEM_MAX = 52428800 + + +class NsJail: + """ + Core Snekbox functionality, providing safe execution of Python code. + + NsJail configuration: + + - Root directory is mounted as read-only + - Time limit of 2 seconds + - Maximum of 1 PID + - Maximum memory of 52428800 bytes + - Loopback interface is down + - procfs is disabled + + Python configuration: + + - Isolated mode + - Neither the script's directory nor the user's site packages are in sys.path + - All PYTHON* environment variables are ignored + """ + + def __init__(self, nsjail_binary: str = NSJAIL_PATH, python_binary: str = sys.executable): + self.nsjail_binary = nsjail_binary + self.python_binary = python_binary + + self._create_parent_cgroups() + + @staticmethod + def _create_parent_cgroups(pids: Path = CGROUP_PIDS_PARENT, mem: Path = CGROUP_MEMORY_PARENT): + """ + Create the PIDs and memory cgroups which NsJail will use as its parent cgroups. + + NsJail doesn't do this automatically because it requires privileges NsJail usually doesn't + have. + + Disables memory swapping. + """ + pids.mkdir(parents=True, exist_ok=True) + mem.mkdir(parents=True, exist_ok=True) + + # Swap limit cannot be set to a value lower than memory.limit_in_bytes. + # Therefore, this must be set first. + (mem / "memory.limit_in_bytes").write_text(str(MEM_MAX), encoding="utf-8") + + try: + # Swap limit is specified as the sum of the memory and swap limits. + (mem / "memory.memsw.limit_in_bytes").write_text(str(MEM_MAX), encoding="utf-8") + except PermissionError: + log.warning( + "Failed to set the memory swap limit for the cgroup. " + "This is probably because CONFIG_MEMCG_SWAP or CONFIG_MEMCG_SWAP_ENABLED is unset. " + "Please ensure swap memory is disabled on the system." + ) + + @staticmethod + def _parse_log(log_lines: Iterable[str]): + """Parse and log NsJail's log messages.""" + for line in log_lines: + match = LOG_PATTERN.fullmatch(line) + if match is None: + log.warning(f"Failed to parse log line '{line}'") + continue + + msg = match["msg"] + if not DEBUG and any(msg.startswith(s) for s in LOG_BLACKLIST): + # Skip blacklisted messages if not debugging. + continue + + if DEBUG and match["func"]: + # Prepend PID, function signature, and line number if debugging. + msg = f"{match['func']}{msg}" + + if match["level"] == "D": + log.debug(msg) + elif match["level"] == "I": + if DEBUG or msg.startswith("pid="): + # Skip messages unrelated to process exit if not debugging. + log.info(msg) + elif match["level"] == "W": + log.warning(msg) + else: + # Treat fatal as error. + log.error(msg) + + def python3(self, code: str) -> CompletedProcess: + """Execute Python 3 code in an isolated environment and return the completed process.""" + with NamedTemporaryFile() as nsj_log: + args = ( + self.nsjail_binary, "-Mo", + "--rlimit_as", "700", + "--chroot", "/", + "-E", "LANG=en_US.UTF-8", + "-R/usr", "-R/lib", "-R/lib64", + "--user", "65534", # nobody + "--group", "65534", # nobody/nogroup + "--time_limit", "2", + "--disable_proc", + "--iface_no_lo", + "--log", nsj_log.name, + f"--cgroup_mem_max={MEM_MAX}", + "--cgroup_mem_mount", str(CGROUP_MEMORY_PARENT.parent), + "--cgroup_mem_parent", CGROUP_MEMORY_PARENT.name, + "--cgroup_pids_max=1", + "--cgroup_pids_mount", str(CGROUP_PIDS_PARENT.parent), + "--cgroup_pids_parent", CGROUP_PIDS_PARENT.name, + "--", + self.python_binary, "-Iq", "-c", code + ) + + msg = "Executing code..." + if DEBUG: + msg = f"{msg[:-3]}:\n{textwrap.indent(code, ' ')}" + log.info(msg) + + try: + result = subprocess.run( + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True + ) + except ValueError: + return CompletedProcess(args, None, "ValueError: embedded null byte", None) + + log_lines = nsj_log.read().decode("utf-8").splitlines() + if not log_lines and result.returncode == 255: + # NsJail probably failed to parse arguments so log output will still be in stdout + log_lines = result.stdout.splitlines() + + self._parse_log(log_lines) + + return result diff --git a/snekweb.py b/snekweb.py deleted file mode 100644 index 3e20fda..0000000 --- a/snekweb.py +++ /dev/null @@ -1,72 +0,0 @@ -import json -import logging -import threading -import traceback - -from flask import Flask, render_template -from flask_sockets import Sockets -from rmq import Rmq - -# Load app -app = Flask(__name__) -app.jinja_env.auto_reload = True -sockets = Sockets(app) - -# Logging -gunicorn_logger = logging.getLogger('gunicorn.error') -app.logger.handlers = gunicorn_logger.handlers -app.logger.setLevel(gunicorn_logger.level) -log = app.logger - - [email protected]('/') -def index(): - """Root path returns standard index.html.""" - return render_template('index.html') - - [email protected]('/ws/<snekboxid>') -def websocket_route(ws, snekboxid): - """Opens a websocket that spawns and connects to a snekbox daemon.""" - localdata = threading.local() - localdata.thread_ws = ws - - rmq = Rmq() - - def message_handler(ch, method, properties, body, thread_ws): - msg = body.decode('utf-8') - thread_ws.send(msg) - ch.basic_ack(delivery_tag=method.delivery_tag) - - consumer_parameters = {'queue': snekboxid, - 'callback': message_handler, - 'thread_ws': localdata.thread_ws} - - consumer = threading.Thread( - target=rmq.consume, - kwargs=consumer_parameters) - - consumer.daemon = True - consumer.start() - - try: - while not ws.closed: - message = ws.receive() - if message: - snek_msg = json.dumps({"snekid": snekboxid, "message": message}) - log.info(f"User {snekboxid} sends message\n{message.strip()}") - rmq.publish(snek_msg) - - except Exception: - log.info(traceback.format_exc()) - - finally: - if not ws.closed: - ws.close() - - -if __name__ == '__main__': - from gevent import pywsgi - from geventwebsocket.handler import WebSocketHandler - server = pywsgi.WSGIServer(('0.0.0.0', 5000), app, handler_class=WebSocketHandler) - server.serve_forever() diff --git a/templates/index.html b/templates/index.html deleted file mode 100644 index 8de9627..0000000 --- a/templates/index.html +++ /dev/null @@ -1,106 +0,0 @@ -<!DOCTYPE html> -<meta charset="utf-8" /> -<title>snekboxweb</title> -<script language="javascript" type="text/javascript"> - - -let _ready = false -let snekbox_id -var output; - -snekbox_id = sessionStorage.getItem("snekbox_id"); -console.log(snekbox_id) -if (snekbox_id == null) { - snekbox_id = generate_id() - sessionStorage.setItem("snekbox_id", snekbox_id) - console.log(snekbox_id) -} - -function init(){ - output = document.getElementById("output"); - websocketHandler(); -} - -function websocketHandler(){ - var here = window.location.host; - var wsUri = `ws://${here}/ws/`+snekbox_id; - websocket = new WebSocket(wsUri); - websocket.onopen = function(evt) { onOpen(evt) }; - websocket.onclose = function(evt) { onClose(evt) }; - websocket.onmessage = function(evt) { onMessage(evt) }; - websocket.onerror = function(evt) { onError(evt) }; -} - -function onOpen(evt){ - _ready = true - console.log("CONNECTED"); -} - -function onClose(evt){ - _ready = false - console.log("DISCONNECTED"); -} - -function onMessage(evt){ - writeToScreen('<span style="color: blue;">RESPONSE: ' + evt.data+'</span>'); -} - -function exit(){ - websocket.close(); -} - -function onError(evt){ - _ready = false - writeToScreen('<span style="color: red;">ERROR:</span> ' + evt.data); -} - -function sendMessage(msg){ - waitForSocketConnection(function(){ - websocket.send(msg); - }); - console.log("sent message "+msg) -} - -function waitForSocketConnection(callback){ - setTimeout( - function () { - if (_ready === true) { - if(callback != null){ - callback();} - return; - } - else { - waitForSocketConnection(callback);} - - }, 500); // milliseconds -} - -function writeToScreen(message){ - var pre = document.createElement("p"); - pre.style.wordWrap = "break-word"; - pre.innerHTML = message; - output.appendChild(pre); -} - -function sendFromInput(){ - var msg = document.getElementById("field1").value; - sendMessage(msg) -} - -function generate_id(){ - return Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15); -} - -window.addEventListener("load", init, false); - -</script> - -<textarea rows="4" cols="50" type="text" id="field1"> -def sum(a,b): - return a+b -print( sum(1,2) ) -</textarea> -<br> -<button onclick="sendFromInput()">Send</button> -<button onclick="exit()">disconnect from websocket</button> -<div id="output"></div> diff --git a/tests/__init__.py b/tests/__init__.py index 792d600..e69de29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +0,0 @@ -# diff --git a/tests/api/__init__.py b/tests/api/__init__.py new file mode 100644 index 0000000..dcee5b5 --- /dev/null +++ b/tests/api/__init__.py @@ -0,0 +1,23 @@ +from subprocess import CompletedProcess +from unittest import mock + +from falcon import testing + +from snekbox.api import SnekAPI + + +class SnekAPITestCase(testing.TestCase): + def setUp(self): + super().setUp() + + self.patcher = mock.patch("snekbox.api.resources.eval.NsJail", autospec=True) + self.mock_nsjail = self.patcher.start() + self.mock_nsjail.return_value.python3.return_value = CompletedProcess( + args=[], + returncode=0, + stdout="output", + stderr="error" + ) + self.addCleanup(self.patcher.stop) + + self.app = SnekAPI() diff --git a/tests/api/test_eval.py b/tests/api/test_eval.py new file mode 100644 index 0000000..3350763 --- /dev/null +++ b/tests/api/test_eval.py @@ -0,0 +1,49 @@ +from tests.api import SnekAPITestCase + + +class TestEvalResource(SnekAPITestCase): + PATH = "/eval" + + def test_post_valid_200(self): + body = {"input": "foo"} + result = self.simulate_post(self.PATH, json=body) + + self.assertEqual(result.status_code, 200) + self.assertEqual("output", result.json["stdout"]) + self.assertEqual(0, result.json["returncode"]) + + def test_post_invalid_schema_400(self): + body = {"stuff": "foo"} + result = self.simulate_post(self.PATH, json=body) + + self.assertEqual(result.status_code, 400) + + expected = { + "title": "Request data failed validation", + "description": "'input' is a required property" + } + + self.assertEqual(expected, result.json) + + def test_post_invalid_content_type_415(self): + body = "{'input': 'foo'}" + headers = {"Content-Type": "application/xml"} + result = self.simulate_post(self.PATH, body=body, headers=headers) + + self.assertEqual(result.status_code, 415) + + expected = { + "title": "Unsupported media type", + "description": "application/xml is an unsupported media type." + } + + self.assertEqual(expected, result.json) + + def test_disallowed_method_405(self): + result = self.simulate_get(self.PATH) + self.assertEqual(result.status_code, 405) + + def test_options_allow_post_only(self): + result = self.simulate_options(self.PATH) + self.assertEqual(result.status_code, 200) + self.assertEqual(result.headers.get("Allow"), "POST") diff --git a/tests/test_nsjail.py b/tests/test_nsjail.py new file mode 100644 index 0000000..bb176d9 --- /dev/null +++ b/tests/test_nsjail.py @@ -0,0 +1,124 @@ +import logging +import unittest +from textwrap import dedent + +from snekbox.nsjail import MEM_MAX, NsJail + + +class NsJailTests(unittest.TestCase): + def setUp(self): + super().setUp() + + self.nsjail = NsJail() + self.nsjail.DEBUG = False + self.logger = logging.getLogger("snekbox.nsjail") + + def test_print_returns_0(self): + result = self.nsjail.python3("print('test')") + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, "test\n") + self.assertEqual(result.stderr, None) + + def test_timeout_returns_137(self): + code = dedent(""" + while True: + pass + """).strip() + + with self.assertLogs(self.logger) as log: + result = self.nsjail.python3(code) + + self.assertEqual(result.returncode, 137) + self.assertEqual(result.stdout, "") + self.assertEqual(result.stderr, None) + self.assertIn("run time >= time limit", "\n".join(log.output)) + + def test_memory_returns_137(self): + # Add a kilobyte just to be safe. + code = dedent(f""" + x = ' ' * {MEM_MAX + 1000} + """).strip() + + result = self.nsjail.python3(code) + self.assertEqual(result.returncode, 137) + self.assertEqual(result.stdout, "") + self.assertEqual(result.stderr, None) + + def test_subprocess_resource_unavailable(self): + code = dedent(""" + import subprocess + print(subprocess.check_output('kill -9 6', shell=True).decode()) + """).strip() + + result = self.nsjail.python3(code) + self.assertEqual(result.returncode, 1) + self.assertIn("Resource temporarily unavailable", result.stdout) + self.assertEqual(result.stderr, None) + + def test_read_only_file_system(self): + code = dedent(""" + open('hello', 'w').write('world') + """).strip() + + result = self.nsjail.python3(code) + self.assertEqual(result.returncode, 1) + self.assertIn("Read-only file system", result.stdout) + self.assertEqual(result.stderr, None) + + def test_forkbomb_resource_unavailable(self): + code = dedent(""" + import os + while 1: + os.fork() + """).strip() + + result = self.nsjail.python3(code) + self.assertEqual(result.returncode, 1) + self.assertIn("Resource temporarily unavailable", result.stdout) + self.assertEqual(result.stderr, None) + + def test_sigsegv_returns_139(self): # In honour of Juan. + code = dedent(""" + import ctypes + ctypes.string_at(0) + """).strip() + + result = self.nsjail.python3(code) + self.assertEqual(result.returncode, 139) + self.assertEqual(result.stdout, "") + self.assertEqual(result.stderr, None) + + def test_null_byte_value_error(self): + result = self.nsjail.python3("\0") + self.assertEqual(result.returncode, None) + self.assertEqual(result.stdout, "ValueError: embedded null byte") + self.assertEqual(result.stderr, None) + + def test_log_parser(self): + log_lines = ( + "[D][2019-06-22T20:07:00+0000][16] void foo::bar()():100 This is a debug message.", + "[I][2019-06-22T20:07:48+0000] pid=20 ([STANDALONE MODE]) " + "exited with status: 2, (PIDs left: 0)", + "[W][2019-06-22T20:06:04+0000][14] void cmdline::logParams(nsjconf_t*)():250 " + "Process will be UID/EUID=0 in the global user namespace, and will have user " + "root-level access to files", + "[W][2019-06-22T20:07:00+0000][16] void foo::bar()():100 This is a warning!", + "[E][2019-06-22T20:07:00+0000][16] bool " + "cmdline::setupArgv(nsjconf_t*, int, char**, int)():316 No command-line provided", + "[F][2019-06-22T20:07:00+0000][16] int main(int, char**)():204 " + "Couldn't parse cmdline options", + "Invalid Line" + ) + + with self.assertLogs(self.logger, logging.DEBUG) as log: + self.nsjail._parse_log(log_lines) + + self.assertIn("DEBUG:snekbox.nsjail:This is a debug message.", log.output) + self.assertIn("ERROR:snekbox.nsjail:Couldn't parse cmdline options", log.output) + self.assertIn("ERROR:snekbox.nsjail:No command-line provided", log.output) + self.assertIn("WARNING:snekbox.nsjail:Failed to parse log line 'Invalid Line'", log.output) + self.assertIn("WARNING:snekbox.nsjail:This is a warning!", log.output) + self.assertIn( + "INFO:snekbox.nsjail:pid=20 ([STANDALONE MODE]) exited with status: 2, (PIDs left: 0)", + log.output + ) diff --git a/tests/test_snekbox.py b/tests/test_snekbox.py deleted file mode 100644 index e2505d6..0000000 --- a/tests/test_snekbox.py +++ /dev/null @@ -1,60 +0,0 @@ -import unittest -import pytest -import os -import json - -from snekbox import Snekbox -from rmq import Rmq - -r = Rmq() - -snek = Snekbox() - - -class SnekTests(unittest.TestCase): - def test_nsjail(self): - result = snek.python3('print("test")') - self.assertEquals(result.strip(), 'test') - - # def test_memory_error(self): - # code = ('x = "*"\n' - # 'while True:\n' - # ' x = x * 99\n') - # result = snek.python3(code) - # self.assertEquals(result.strip(), 'timed out or memory limit exceeded') - - def test_timeout(self): - code = ('x = "*"\n' - 'while True:\n' - ' try:\n' - ' x = x * 99\n' - ' except:\n' - ' continue\n') - - result = snek.python3(code) - self.assertEquals(result.strip(), 'timed out or memory limit exceeded') - - def test_kill(self): - code = ('import subprocess\n' - 'print(subprocess.check_output("kill -9 6", shell=True).decode())') - result = snek.python3(code) - if 'ModuleNotFoundError' in result.strip(): - self.assertIn('ModuleNotFoundError', result.strip()) - else: - self.assertIn('(PIDs left: 0)', result.strip()) - - def test_forkbomb(self): - code = ('import os\n' - 'while 1:\n' - ' os.fork()') - result = snek.python3(code) - self.assertIn('Resource temporarily unavailable', result.strip()) - - def test_juan_golf(self): # in honour of Juan - code = ("func = lambda: None\n" - "CodeType = type(func.__code__)\n" - "bytecode = CodeType(0,1,0,0,0,b'',(),(),(),'','',1,b'')\n" - "exec(bytecode)") - - result = snek.python3(code) - self.assertEquals('unknown error, code: 111', result.strip()) |