From 78757589a2cc6a76b83041dbb75b02896308da69 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Wed, 29 May 2019 22:47:31 -0700 Subject: Add flake8 plugin to only allow double quotes --- .flake8 | 1 + Pipfile | 1 + Pipfile.lock | 94 ++++++++++++++++++++++++++++--------------------- snekbox/__init__.py | 4 +-- snekbox/nsjail.py | 66 +++++++++++++++++----------------- snekbox/site/snekapp.py | 10 +++--- tests/test_snekbox.py | 46 ++++++++++++------------ 7 files changed, 119 insertions(+), 103 deletions(-) diff --git a/.flake8 b/.flake8 index f8eec98..347bdb0 100644 --- a/.flake8 +++ b/.flake8 @@ -16,3 +16,4 @@ exclude= venv,.venv, tests import-order-style=pycharm +inline-quotes = " diff --git a/Pipfile b/Pipfile index 788e900..69bb0df 100644 --- a/Pipfile +++ b/Pipfile @@ -20,6 +20,7 @@ flake8-tidy-imports = "*" flake8-todo = "*" flake8-string-format = "*" flake8-formatter-junit-xml = "*" +flake8-quotes = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 466a42b..b73b997 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "814185e2e1b964ab58af9a9df416ace7b5b416475d828ec9b31a9dfecb5693e1" + "sha256": "c1f4c3df791d8c4758f72cb8fb148a34d5c1ca02298b0d660844899f15f6ba85" }, "pipfile-spec": 6, "requires": { @@ -25,11 +25,11 @@ }, "flask": { "hashes": [ - "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", - "sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05" + "sha256:ad7c6d841e64296b962296c2c2dabc6543752985727af86a975072dea984b6f3", + "sha256:e7d32475d1de5facaa55e3958bc4ec66d3762076b074296aa50ef8fdc5b9df61" ], "index": "pypi", - "version": "==1.0.2" + "version": "==1.0.3" }, "gunicorn": { "hashes": [ @@ -48,10 +48,10 @@ }, "jinja2": { "hashes": [ - "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", - "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" + "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", + "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" ], - "version": "==2.10" + "version": "==2.10.1" }, "markupsafe": { "hashes": [ @@ -88,19 +88,19 @@ }, "werkzeug": { "hashes": [ - "sha256:96da23fa8ccecbc3ae832a83df5c722c11547d021637faacb0bec4dd2f4666c8", - "sha256:ca5c2dcd367d6c0df87185b9082929d255358f5391923269335782b213d52655" + "sha256:865856ebb55c4dcd0630cdd8f3331a1847a819dda7e8c750d3db6f2aa6c0209c", + "sha256:a0b915f0815982fb2a09161cb8f31708052d0951c3ba433ccc5e1aa276507ca6" ], - "version": "==0.15.1" + "version": "==0.15.4" } }, "develop": { "aspy.yaml": { "hashes": [ - "sha256:ae249074803e8b957c83fdd82a99160d0d6d26dff9ba81ba608b42eebd7d8cd3", - "sha256:c7390d79f58eb9157406966201abf26da0d56c07e0ff0deadc39c8f4dbc13482" + "sha256:463372c043f70160a9ec950c3f1e4c3a82db5fca01d334b6bc89c7164d744bdc", + "sha256:e7c742382eff2caed61f87a39d13f99109088e5e93f04d76eb8d4b28aa143f45" ], - "version": "==1.2.0" + "version": "==1.3.0" }, "atomicwrites": { "hashes": [ @@ -118,10 +118,10 @@ }, "cfgv": { "hashes": [ - "sha256:39f8475d8eca48639f900daffa3f8bd2f60a31d989df41a9f81c5ad1779a66eb", - "sha256:a6a4366d32799a6bfb6f577ebe113b27ba8d1bae43cb57133b1472c1c3dae227" + "sha256:32edbe09de6f4521224b87822103a8c16a614d31a894735f7a5b3bcf0eb3c37e", + "sha256:3bd31385cd2bebddbba8012200aaf15aa208539f1b33973759b4d02fc2148da5" ], - "version": "==1.5.0" + "version": "==2.0.0" }, "coverage": { "hashes": [ @@ -213,6 +213,13 @@ ], "version": "==1.0.2" }, + "flake8-quotes": { + "hashes": [ + "sha256:10c9af6b472d4302a8e721c5260856c3f985c5c082b04841aefd2f808ac02038" + ], + "index": "pypi", + "version": "==2.0.1" + }, "flake8-string-format": { "hashes": [ "sha256:68ea72a1a5b75e7018cae44d14f32473c798cf73d75cbaed86c6a9a907b770b2", @@ -238,17 +245,17 @@ }, "identify": { "hashes": [ - "sha256:244e7864ef59f0c7c50c6db73f58564151d91345cd9b76ed793458953578cadd", - "sha256:8ff062f90ad4b09cfe79b5dfb7a12e40f19d2e68a5c9598a49be45f16aba7171" + "sha256:432c548d6138cb57a3d8f62f079a025a29b8ae34a50dd3b496bbf661818f2bc0", + "sha256:d4401d60bf1938aa3074a352a5cc9044107edf11a6fedd3a1db172c141619b81" ], - "version": "==1.4.1" + "version": "==1.4.3" }, "importlib-metadata": { "hashes": [ - "sha256:a17ce1a8c7bff1e8674cb12c992375d8d0800c9190177ecf0ad93e0097224095", - "sha256:b50191ead8c70adfa12495fba19ce6d75f2e0275c14c5a7beb653d6799b512bd" + "sha256:a9f185022cfa69e9ca5f7eabfd5a58b689894cb78a11e3c8c89398a8ccbb8e7f", + "sha256:df1403cd3aebeb2b1dcd3515ca062eecb5bd3ea7611f18cba81130c68707e879" ], - "version": "==0.8" + "version": "==0.17" }, "junit-xml": { "hashes": [ @@ -279,18 +286,18 @@ }, "pluggy": { "hashes": [ - "sha256:19ecf9ce9db2fce065a7a0586e07cfb4ac8614fe96edf628a264b1c70116cf8f", - "sha256:84d306a647cc805219916e62aab89caa97a33a1dd8c342e87a37f91073cd4746" + "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc", + "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c" ], - "version": "==0.9.0" + "version": "==0.12.0" }, "pre-commit": { "hashes": [ - "sha256:d3d69c63ae7b7584c4b51446b0b583d454548f9df92575b2fe93a68ec800c4d3", - "sha256:fc512f129b9526e35e80d656a16a31c198f584c4fce3a5c739045b5140584917" + "sha256:6ca409d1f22d444af427fb023a33ca8b69625d508a50e1b7eaabd59247c93043", + "sha256:94dd519597f5bff06a4b0df194a79c524b78f4b1534c1ce63241a9d4fb23b926" ], "index": "pypi", - "version": "==1.14.4" + "version": "==1.16.1" }, "py": { "hashes": [ @@ -323,19 +330,19 @@ }, "pytest": { "hashes": [ - "sha256:592eaa2c33fae68c7d75aacf042efc9f77b27c08a6224a4f59beab8d9a420523", - "sha256:ad3ad5c450284819ecde191a654c09b0ec72257a2c711b9633d677c71c9850c4" + "sha256:1a8aa4fa958f8f451ac5441f3ac130d9fc86ea38780dd2715e6d5c5882700b24", + "sha256:b8bf138592384bd4e87338cb0f256bf5f615398a649d4bd83915f0e4047a5ca6" ], "index": "pypi", - "version": "==4.3.1" + "version": "==4.5.0" }, "pytest-cov": { "hashes": [ - "sha256:0ab664b25c6aa9716cbf203b17ddb301932383046082c081b9848a0edf5add33", - "sha256:230ef817450ab0699c6cc3c9c8f7a829c34674456f2ed8df1fe1d39780f7c87f" + "sha256:2b097cde81a302e1047331b48cadacf23577e431b61e9c6f49a1170bbe3d3da6", + "sha256:e00ea4fdde970725482f1f35630d12f074e121a23801aabf2ae154ec6bdd343a" ], "index": "pypi", - "version": "==2.6.1" + "version": "==2.7.1" }, "pytest-dependency": { "hashes": [ @@ -383,17 +390,24 @@ }, "virtualenv": { "hashes": [ - "sha256:6aebaf4dd2568a0094225ebbca987859e369e3e5c22dc7d52e5406d504890417", - "sha256:984d7e607b0a5d1329425dd8845bd971b957424b5ba664729fab51ab8c11bc39" + "sha256:99acaf1e35c7ccf9763db9ba2accbca2f4254d61d1912c5ee364f9cc4a8942a0", + "sha256:fe51cdbf04e5d8152af06c075404745a7419de27495a83f0d72518ad50be3ce8" + ], + "version": "==16.6.0" + }, + "wcwidth": { + "hashes": [ + "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", + "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" ], - "version": "==16.4.3" + "version": "==0.1.7" }, "zipp": { "hashes": [ - "sha256:55ca87266c38af6658b84db8cfb7343cdb0bf275f93c7afaea0d8e7a209c7478", - "sha256:682b3e1c62b7026afe24eadf6be579fb45fec54c07ea218bded8092af07a68c4" + "sha256:8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d", + "sha256:ca943a7e809cc12257001ccfb99e3563da9af99d52f261725e96dfe0f9275bc3" ], - "version": "==0.3.3" + "version": "==0.5.1" } } } diff --git a/snekbox/__init__.py b/snekbox/__init__.py index fc6070e..f14fc89 100644 --- a/snekbox/__init__.py +++ b/snekbox/__init__.py @@ -1,8 +1,8 @@ import logging import sys -logformat = logging.Formatter(fmt='[%(asctime)s] [%(process)s] [%(levelname)s] %(message)s', - datefmt='%Y-%m-%d %H:%M:%S %z') +logformat = logging.Formatter(fmt="[%(asctime)s] [%(process)s] [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S %z") log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) console = logging.StreamHandler(sys.stdout) diff --git a/snekbox/nsjail.py b/snekbox/nsjail.py index 5c7d0f0..c807b9e 100644 --- a/snekbox/nsjail.py +++ b/snekbox/nsjail.py @@ -7,25 +7,25 @@ class NsJail: """Core Snekbox functionality, providing safe execution of Python code.""" def __init__(self, - nsjail_binary='nsjail', - python_binary=os.path.dirname(sys.executable) + os.sep + 'python3.7'): + nsjail_binary="nsjail", + python_binary=os.path.dirname(sys.executable) + os.sep + "python3.7"): self.nsjail_binary = nsjail_binary self.python_binary = python_binary self._nsjail_workaround() env = { - 'PATH': ( - '/snekbox/.venv/bin:/usr/local/bin:/usr/local/' - 'sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + "PATH": ( + "/snekbox/.venv/bin:/usr/local/bin:/usr/local/" + "sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ), - 'LANG': 'en_US.UTF-8', - 'PYTHON_VERSION': '3.7.3', - 'PYTHON_PIP_VERSION': '19.0.3', - 'PYTHONDONTWRITEBYTECODE': '1', + "LANG": "en_US.UTF-8", + "PYTHON_VERSION": "3.7.3", + "PYTHON_PIP_VERSION": "19.0.3", + "PYTHONDONTWRITEBYTECODE": "1", } def _nsjail_workaround(self): - dirs = ['/sys/fs/cgroup/pids/NSJAIL', '/sys/fs/cgroup/memory/NSJAIL'] + dirs = ["/sys/fs/cgroup/pids/NSJAIL", "/sys/fs/cgroup/memory/NSJAIL"] for d in dirs: if not os.path.exists(d): os.makedirs(d) @@ -41,20 +41,20 @@ class NsJail: Returns the output of executing the command (stdout) if successful, or a error message if the execution failed. """ - args = [self.nsjail_binary, '-Mo', - '--rlimit_as', '700', - '--chroot', '/', - '-E', 'LANG=en_US.UTF-8', - '-R/usr', '-R/lib', '-R/lib64', - '--user', 'nobody', - '--group', 'nogroup', - '--time_limit', '2', - '--disable_proc', - '--iface_no_lo', - '--cgroup_pids_max=1', - '--cgroup_mem_max=52428800', - '--quiet', '--', - self.python_binary, '-ISq', '-c', cmd] + args = [self.nsjail_binary, "-Mo", + "--rlimit_as", "700", + "--chroot", "/", + "-E", "LANG=en_US.UTF-8", + "-R/usr", "-R/lib", "-R/lib64", + "--user", "nobody", + "--group", "nogroup", + "--time_limit", "2", + "--disable_proc", + "--iface_no_lo", + "--cgroup_pids_max=1", + "--cgroup_mem_max=52428800", + "--quiet", "--", + self.python_binary, "-ISq", "-c", cmd] try: proc = subprocess.Popen(args, stdin=subprocess.PIPE, @@ -63,7 +63,7 @@ class NsJail: env=self.env, universal_newlines=True) except ValueError: - return 'ValueError: embedded null byte' + return "ValueError: embedded null byte" stdout, stderr = proc.communicate() if proc.returncode == 0: @@ -72,23 +72,23 @@ class NsJail: elif proc.returncode == 1: try: filtered = [] - for line in stderr.split('\n'): - if not line.startswith('['): + for line in stderr.split("\n"): + if not line.startswith("["): filtered.append(line) - output = '\n'.join(filtered) + output = "\n".join(filtered) except IndexError: - output = '' + output = "" elif proc.returncode == 109: - return 'timed out or memory limit exceeded' + return "timed out or memory limit exceeded" elif proc.returncode == 255: - return 'permission denied (root required)' + return "permission denied (root required)" elif proc.returncode: - return f'unknown error, code: {proc.returncode}' + return f"unknown error, code: {proc.returncode}" else: - return 'unknown error, no error code' + return "unknown error, no error code" return output diff --git a/snekbox/site/snekapp.py b/snekbox/site/snekapp.py index ef96148..3954238 100644 --- a/snekbox/site/snekapp.py +++ b/snekbox/site/snekapp.py @@ -12,22 +12,22 @@ app.use_reloader = False log = app.logger -@app.route('/') +@app.route("/") def index(): """Return a page with a form for inputting code to be executed.""" - return render_template('index.html') + return render_template("index.html") -@app.route('/result', methods=["POST", "GET"]) +@app.route("/result", methods=["POST", "GET"]) def result(): """Execute code and return a page displaying the results.""" if request.method == "POST": code = request.form["Code"] output = nsjail.python3(code) - return render_template('result.html', code=code, result=output) + return render_template("result.html", code=code, result=output) -@app.route('/input', methods=["POST"]) +@app.route("/input", methods=["POST"]) def code_input(): """Execute code and return the results.""" body = request.get_json() diff --git a/tests/test_snekbox.py b/tests/test_snekbox.py index c08178f..46319d6 100644 --- a/tests/test_snekbox.py +++ b/tests/test_snekbox.py @@ -7,44 +7,44 @@ nsjail = NsJail() class SnekTests(unittest.TestCase): def test_nsjail(self): - result = nsjail.python3('print("test")') - self.assertEquals(result.strip(), 'test') + result = nsjail.python3("print('test')") + self.assertEquals(result.strip(), "test") # def test_memory_error(self): - # code = ('x = "*"\n' - # 'while True:\n' - # ' x = x * 99\n') + # code = ("x = "*"\n" + # "while True:\n" + # " x = x * 99\n") # result = nsjail.python3(code) - # self.assertEquals(result.strip(), 'timed out or memory limit exceeded') + # self.assertEquals(result.strip(), "timed out or memory limit exceeded") def test_timeout(self): code = ( - 'x = "*"\n' - 'while True:\n' - ' try:\n' - ' x = x * 99\n' - ' except:\n' - ' continue\n' + "x = '*'\n" + "while True:\n" + " try:\n" + " x = x * 99\n" + " except:\n" + " continue\n" ) result = nsjail.python3(code) - self.assertEquals(result.strip(), 'timed out or memory limit exceeded') + self.assertEquals(result.strip(), "timed out or memory limit exceeded") def test_kill(self): - code = ('import subprocess\n' - 'print(subprocess.check_output("kill -9 6", shell=True).decode())') + code = ("import subprocess\n" + "print(subprocess.check_output('kill -9 6', shell=True).decode())") result = nsjail.python3(code) - if 'ModuleNotFoundError' in result.strip(): - self.assertIn('ModuleNotFoundError', result.strip()) + if "ModuleNotFoundError" in result.strip(): + self.assertIn("ModuleNotFoundError", result.strip()) else: - self.assertIn('(PIDs left: 0)', result.strip()) + self.assertIn("(PIDs left: 0)", result.strip()) def test_forkbomb(self): - code = ('import os\n' - 'while 1:\n' - ' os.fork()') + code = ("import os\n" + "while 1:\n" + " os.fork()") result = nsjail.python3(code) - self.assertIn('Resource temporarily unavailable', result.strip()) + self.assertIn("Resource temporarily unavailable", result.strip()) def test_juan_golf(self): # in honour of Juan code = ("func = lambda: None\n" @@ -53,4 +53,4 @@ class SnekTests(unittest.TestCase): "exec(bytecode)") result = nsjail.python3(code) - self.assertEquals('unknown error, code: 111', result.strip()) + self.assertEquals("unknown error, code: 111", result.strip()) -- cgit v1.2.3 From 4deb784129f1e2c1cdadbe1c27c2be4b404a69dd Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 30 May 2019 02:01:33 -0700 Subject: Fix pre-commit configuration Excluded files and directories in the flake8 config file are overwritten by the --files argument, which is considered intended behaviour. However, this is problematic due to pre-commit's reliance on the --files argument. Therefore, pre-commit's own exclude config option must be leveraged. * Properly exclude tests directory * Add flake8 plugins as dependencies so that they are actually used --- .pre-commit-config.yaml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1d75342..5d2d40a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,4 +2,17 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.0.0 hooks: - - id: flake8 \ No newline at end of file + - id: flake8 + args: [--config=.flake8] + exclude: ^tests/ + additional_dependencies: [ + flake8-docstrings, + flake8-bugbear, + flake8-import-order, + flake8-tidy-imports, + flake8-todo, + flake8-string-format, + flake8-formatter-junit-xml, + flake8-quotes + ] + -- cgit v1.2.3 From e75c764f693c3688a59af0d679e0d3e94f003503 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 30 May 2019 03:19:34 -0700 Subject: Lint tests Tests ignore all D1xx warnings because tests shouldn't require docstrings. --- .pre-commit-config.yaml | 16 +++++++++++++++- tests/.flake8 | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tests/.flake8 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d2d40a..4f97db9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,6 +3,7 @@ repos: rev: v2.0.0 hooks: - id: flake8 + name: Flake8 (snekbox) args: [--config=.flake8] exclude: ^tests/ additional_dependencies: [ @@ -15,4 +16,17 @@ repos: flake8-formatter-junit-xml, flake8-quotes ] - + - id: flake8 + name: Flake8 (tests) + args: [--config=tests/.flake8] + exclude: ^(?!tests/) + additional_dependencies: [ + flake8-docstrings, + flake8-bugbear, + flake8-import-order, + flake8-tidy-imports, + flake8-todo, + flake8-string-format, + flake8-formatter-junit-xml, + flake8-quotes + ] diff --git a/tests/.flake8 b/tests/.flake8 new file mode 100644 index 0000000..c1c5031 --- /dev/null +++ b/tests/.flake8 @@ -0,0 +1,15 @@ +[flake8] +max-line-length=100 +application_import_names=snekbox,tests +ignore= + P102,B311,W503,E226,S311, + # Missing Docstrings + D1, + # Docstring Whitespace + D203,D212,D214,D215, + # Docstring Quotes + D301,D302, + # Docstring Content + D400,D401,D402,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414 +import-order-style=pycharm +inline-quotes = " -- cgit v1.2.3 From c2416eb672afc80476fe2da1b364af5d018a16b0 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 30 May 2019 03:29:55 -0700 Subject: Run flake8 via pre-commit for the pipenv lint script Pre-commit is a nice wrapper for this particularily because it allows both the snekbox and tests to be linted with a single command while clearly distinguishing their results. The Python script is a cross-platform implementation of the following: git ls-files -- '*.py' | xargs pre-commit run flake8 --files --- Pipfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index 69bb0df..ea6299b 100644 --- a/Pipfile +++ b/Pipfile @@ -26,7 +26,13 @@ flake8-quotes = "*" python_version = "3.7" [scripts] -lint = "flake8" +lint = """ +python -c +\"import subprocess, sys +files = subprocess.check_output(('git', 'ls-files', '--', '*.py')).decode('UTF-8').strip() +args = ['pre-commit', 'run', 'flake8', '--files'] + files.split('\\n') +subprocess.run(args, stdout=sys.stdout, stderr=sys.stderr)\" +""" precommit = "pre-commit install" test = "pytest tests --cov . --cov-report term-missing -v" report = "pytest tests --cov . --cov-report=html" -- cgit v1.2.3 From e9d6c4c4cf6fe7cd6fe4ff85fe1fdb3470d9929d Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 30 May 2019 05:11:26 -0700 Subject: Refactor NsJail --- snekbox/nsjail.py | 141 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 74 insertions(+), 67 deletions(-) diff --git a/snekbox/nsjail.py b/snekbox/nsjail.py index c807b9e..2484ba2 100644 --- a/snekbox/nsjail.py +++ b/snekbox/nsjail.py @@ -1,93 +1,100 @@ -import os import subprocess import sys +from pathlib import Path + +# Explicitly define constants for NsJail's default values. +CGROUP_PIDS_PARENT = Path("/sys/fs/cgroup/pids/NSJAIL") +CGROUP_MEMORY_PARENT = Path("/sys/fs/cgroup/memory/NSJAIL") + +ENV = { + "PATH": ( + "/snekbox/.venv/bin:/usr/local/bin:/usr/local/" + "sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ), + "LANG": "en_US.UTF-8", + "PYTHON_VERSION": "3.7.3", + "PYTHON_PIP_VERSION": "19.0.3", + "PYTHONDONTWRITEBYTECODE": "1", +} class NsJail: - """Core Snekbox functionality, providing safe execution of Python code.""" + """ + Core Snekbox functionality, providing safe execution of Python code. + + NsJail configuration: + + - Root directory is mounted as read-only + - Time limit of 2 seconds + - Maximum of 1 PID + - Maximum memory of 52428800 bytes + - Loopback interface is down + - procfs is disabled - def __init__(self, - nsjail_binary="nsjail", - python_binary=os.path.dirname(sys.executable) + os.sep + "python3.7"): + Python configuration: + + - Isolated mode + - Neither the script's directory nor the user's site packages are in sys.path + - All PYTHON* environment variables are ignored + - Import of the site module is disabled + """ + + def __init__(self, nsjail_binary="nsjail", python_binary=sys.executable): self.nsjail_binary = nsjail_binary self.python_binary = python_binary - self._nsjail_workaround() - - env = { - "PATH": ( - "/snekbox/.venv/bin:/usr/local/bin:/usr/local/" - "sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - ), - "LANG": "en_US.UTF-8", - "PYTHON_VERSION": "3.7.3", - "PYTHON_PIP_VERSION": "19.0.3", - "PYTHONDONTWRITEBYTECODE": "1", - } - - def _nsjail_workaround(self): - dirs = ["/sys/fs/cgroup/pids/NSJAIL", "/sys/fs/cgroup/memory/NSJAIL"] - for d in dirs: - if not os.path.exists(d): - os.makedirs(d) - - def python3(self, cmd): - """ - Execute Python 3 code in a isolated environment. - The value of ``cmd`` is passed using '-c' to a Python - interpreter that is started in a ``nsjail``, isolating it - from the rest of the system. + self._create_parent_cgroups() + + @staticmethod + def _create_parent_cgroups(pids: Path = CGROUP_PIDS_PARENT, mem: Path = CGROUP_MEMORY_PARENT): + """ + Create the PIDs and memory cgroups which NsJail will use as its parent cgroups. - Returns the output of executing the command (stdout) if - successful, or a error message if the execution failed. + NsJail doesn't do this automatically because it requires privileges NsJail usually doesn't + have. """ - args = [self.nsjail_binary, "-Mo", - "--rlimit_as", "700", - "--chroot", "/", - "-E", "LANG=en_US.UTF-8", - "-R/usr", "-R/lib", "-R/lib64", - "--user", "nobody", - "--group", "nogroup", - "--time_limit", "2", - "--disable_proc", - "--iface_no_lo", - "--cgroup_pids_max=1", - "--cgroup_mem_max=52428800", - "--quiet", "--", - self.python_binary, "-ISq", "-c", cmd] + pids.mkdir(parents=True, exist_ok=True) + mem.mkdir(parents=True, exist_ok=True) + + def python3(self, code: str) -> str: + """Execute Python 3 code in an isolated environment and return stdout or an error.""" + args = ( + self.nsjail_binary, "-Mo", + "--rlimit_as", "700", + "--chroot", "/", + "-E", "LANG=en_US.UTF-8", + "-R/usr", "-R/lib", "-R/lib64", + "--user", "nobody", + "--group", "nogroup", + "--time_limit", "2", + "--disable_proc", + "--iface_no_lo", + "--cgroup_mem_max=52428800", + "--cgroup_mem_mount", str(CGROUP_MEMORY_PARENT.parent), + "--cgroup_mem_parent", CGROUP_MEMORY_PARENT.name, + "--cgroup_pids_max=1", + "--cgroup_pids_mount", str(CGROUP_PIDS_PARENT.parent), + "--cgroup_pids_parent", CGROUP_PIDS_PARENT.name, + "--quiet", "--", + self.python_binary, "-ISq", "-c", code + ) + try: - proc = subprocess.Popen(args, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=self.env, - universal_newlines=True) + proc = subprocess.run(args, capture_output=True, env=ENV, text=True) except ValueError: return "ValueError: embedded null byte" - stdout, stderr = proc.communicate() if proc.returncode == 0: - output = stdout - + output = proc.stdout elif proc.returncode == 1: - try: - filtered = [] - for line in stderr.split("\n"): - if not line.startswith("["): - filtered.append(line) - output = "\n".join(filtered) - except IndexError: - output = "" - + filtered = (line for line in proc.stderr.split("\n") if not line.startswith("[")) + output = "\n".join(filtered) elif proc.returncode == 109: return "timed out or memory limit exceeded" - elif proc.returncode == 255: return "permission denied (root required)" - elif proc.returncode: return f"unknown error, code: {proc.returncode}" - else: return "unknown error, no error code" -- cgit v1.2.3 From 6eb7afc321d7e6d2aeebcb90e4bb18cd5e43a6e2 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 30 May 2019 18:26:38 -0700 Subject: Revert "Lint tests" This reverts commit e75c764f693c3688a59af0d679e0d3e94f003503. --- .pre-commit-config.yaml | 16 +--------------- tests/.flake8 | 15 --------------- 2 files changed, 1 insertion(+), 30 deletions(-) delete mode 100644 tests/.flake8 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4f97db9..5d2d40a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,6 @@ repos: rev: v2.0.0 hooks: - id: flake8 - name: Flake8 (snekbox) args: [--config=.flake8] exclude: ^tests/ additional_dependencies: [ @@ -16,17 +15,4 @@ repos: flake8-formatter-junit-xml, flake8-quotes ] - - id: flake8 - name: Flake8 (tests) - args: [--config=tests/.flake8] - exclude: ^(?!tests/) - additional_dependencies: [ - flake8-docstrings, - flake8-bugbear, - flake8-import-order, - flake8-tidy-imports, - flake8-todo, - flake8-string-format, - flake8-formatter-junit-xml, - flake8-quotes - ] + diff --git a/tests/.flake8 b/tests/.flake8 deleted file mode 100644 index c1c5031..0000000 --- a/tests/.flake8 +++ /dev/null @@ -1,15 +0,0 @@ -[flake8] -max-line-length=100 -application_import_names=snekbox,tests -ignore= - P102,B311,W503,E226,S311, - # Missing Docstrings - D1, - # Docstring Whitespace - D203,D212,D214,D215, - # Docstring Quotes - D301,D302, - # Docstring Content - D400,D401,D402,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414 -import-order-style=pycharm -inline-quotes = " -- cgit v1.2.3 From ce8aedc9a150dfa53d12bd9b9145fea9c05d8799 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 30 May 2019 18:27:58 -0700 Subject: Revert "Run flake8 via pre-commit for the pipenv lint script" This reverts commit c2416eb672afc80476fe2da1b364af5d018a16b0. --- Pipfile | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Pipfile b/Pipfile index ea6299b..69bb0df 100644 --- a/Pipfile +++ b/Pipfile @@ -26,13 +26,7 @@ flake8-quotes = "*" python_version = "3.7" [scripts] -lint = """ -python -c -\"import subprocess, sys -files = subprocess.check_output(('git', 'ls-files', '--', '*.py')).decode('UTF-8').strip() -args = ['pre-commit', 'run', 'flake8', '--files'] + files.split('\\n') -subprocess.run(args, stdout=sys.stdout, stderr=sys.stderr)\" -""" +lint = "flake8" precommit = "pre-commit install" test = "pytest tests --cov . --cov-report term-missing -v" report = "pytest tests --cov . --cov-report=html" -- cgit v1.2.3 From f9d716e1b036fdc8cd71ccf7e7acaa2348e3ffb7 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Thu, 30 May 2019 18:30:04 -0700 Subject: Lint tests * Ignore D1xx for tests because they shouldn't require docstrings * Update pre-commit-hooks repo to v2.2.3 --- .flake8 | 6 +++--- .pre-commit-config.yaml | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.flake8 b/.flake8 index 347bdb0..1a61894 100644 --- a/.flake8 +++ b/.flake8 @@ -13,7 +13,7 @@ ignore= D400,D401,D402,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414 exclude= __pycache__,.cache, - venv,.venv, - tests + venv,.venv +per-file-ignores=tests/*:D1 import-order-style=pycharm -inline-quotes = " +inline-quotes=" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d2d40a..d2737fe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,8 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.0.0 + rev: v2.2.3 hooks: - id: flake8 - args: [--config=.flake8] - exclude: ^tests/ additional_dependencies: [ flake8-docstrings, flake8-bugbear, -- cgit v1.2.3 From a8fec66504a94c6b830681eef1c3276104b7ff25 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 4 Jun 2019 12:33:38 -0700 Subject: Configure gunicorn logging --- Pipfile | 2 +- snekbox/__init__.py | 32 +++++++++++++++++++++++--------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/Pipfile b/Pipfile index 69bb0df..a50cba5 100644 --- a/Pipfile +++ b/Pipfile @@ -30,7 +30,7 @@ lint = "flake8" precommit = "pre-commit install" test = "pytest tests --cov . --cov-report term-missing -v" report = "pytest tests --cov . --cov-report=html" -snekbox = "gunicorn -w 2 -b 0.0.0.0:8060 snekbox.site.snekapp:app" +snekbox = "gunicorn -w 2 -b 0.0.0.0:8060 --logger-class snekbox.GunicornLogger --access-logfile - snekbox.site.snekapp:app" buildbox = "docker build -t pythondiscord/snekbox:latest -f docker/Dockerfile ." pushbox = "docker push pythondiscord/snekbox:latest" buildboxbase = "docker build -t pythondiscord/snekbox-base:latest -f docker/base.Dockerfile ." diff --git a/snekbox/__init__.py b/snekbox/__init__.py index f14fc89..af8429b 100644 --- a/snekbox/__init__.py +++ b/snekbox/__init__.py @@ -1,10 +1,24 @@ import logging -import sys - -logformat = logging.Formatter(fmt="[%(asctime)s] [%(process)s] [%(levelname)s] %(message)s", - datefmt="%Y-%m-%d %H:%M:%S %z") -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) -console = logging.StreamHandler(sys.stdout) -console.setFormatter(logformat) -log.addHandler(console) +import os + +from gunicorn import glogging + +DEBUG = os.environ.get("DEBUG", False) + + +class GunicornLogger(glogging.Logger): + """Logger for Gunicorn with custom formatting and support for the DEBUG environment variable.""" + + error_fmt = "%(asctime)s | %(process)5s | %(name)30s | %(levelname)8s | %(message)s" + datefmt = "%Y-%m-%d %H:%M:%S" + + def setup(self, cfg): + """Set up loggers and set error logger's level to DEBUG if the DEBUG env var is set.""" + super().setup(cfg) + + if DEBUG: + self.loglevel = logging.DEBUG + else: + self.loglevel = self.LOG_LEVELS.get(cfg.loglevel.lower(), logging.INFO) + + self.error_log.setLevel(self.loglevel) -- cgit v1.2.3 From 7916804de8176fa34e4dfc56c0543157e72985f1 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Tue, 4 Jun 2019 22:29:08 -0700 Subject: Add logging for NsJail NsJail's is configured to log to a temporary file rather than stderr. The contents of the file are parsed using regex after the process exits. When not debugging, some blacklisted messages and most info-level messages are skipped. * Add a snekbox logger * Log the Python code being executed if debugging * Use nested single quotes in a test to fix a linter error --- snekbox/__init__.py | 10 +++++ snekbox/nsjail.py | 106 ++++++++++++++++++++++++++++++++++++------------- tests/api/test_eval.py | 2 +- 3 files changed, 90 insertions(+), 28 deletions(-) diff --git a/snekbox/__init__.py b/snekbox/__init__.py index af8429b..a48abd5 100644 --- a/snekbox/__init__.py +++ b/snekbox/__init__.py @@ -1,5 +1,6 @@ import logging import os +import sys from gunicorn import glogging @@ -22,3 +23,12 @@ class GunicornLogger(glogging.Logger): self.loglevel = self.LOG_LEVELS.get(cfg.loglevel.lower(), logging.INFO) self.error_log.setLevel(self.loglevel) + + +log = logging.getLogger("snekbox") +log.setLevel(logging.DEBUG if DEBUG else logging.INFO) +log.propagate = True +formatter = logging.Formatter(GunicornLogger.error_fmt, GunicornLogger.datefmt) +handler = logging.StreamHandler(sys.stdout) +handler.setFormatter(formatter) +log.addHandler(handler) diff --git a/snekbox/nsjail.py b/snekbox/nsjail.py index 2484ba2..0ebfe0c 100644 --- a/snekbox/nsjail.py +++ b/snekbox/nsjail.py @@ -1,6 +1,20 @@ +import logging +import re import subprocess import sys +import textwrap from pathlib import Path +from tempfile import NamedTemporaryFile + +from snekbox import DEBUG + +log = logging.getLogger(__name__) + +# [level][timestamp][PID]? function_signature:line_no? message +LOG_PATTERN = re.compile( + r"\[(?P(I)|[WEF])\]\[.+?\](?(2)|(?P\[\d+\] .+?:\d+ )) ?(?P.+)" +) +LOG_BLACKLIST = ("Process will be ",) # Explicitly define constants for NsJail's default values. CGROUP_PIDS_PARENT = Path("/sys/fs/cgroup/pids/NSJAIL") @@ -56,39 +70,77 @@ class NsJail: pids.mkdir(parents=True, exist_ok=True) mem.mkdir(parents=True, exist_ok=True) + @staticmethod + def _parse_log(log_file): + """Parse and log NsJail's log messages.""" + for line in log_file.read().decode("UTF-8").splitlines(): + match = LOG_PATTERN.fullmatch(line) + if match is None: + log.warning(f"Failed to parse log line '{line}'") + continue + + msg = match["msg"] + if not DEBUG and any(msg.startswith(s) for s in LOG_BLACKLIST): + # Skip blacklisted messages if not debugging. + continue + + if DEBUG and match["func"]: + # Prepend PID, function signature, and line number if debugging. + msg = f"{match['func']}{msg}" + + if match["level"] == "D": + log.debug(msg) + elif match["level"] == "I": + if DEBUG or msg.startswith("pid="): + # Skip messages unrelated to process exit if not debugging. + log.info(msg) + elif match["level"] == "W": + log.warning(msg) + else: + # Treat fatal as error. + log.error(msg) + def python3(self, code: str) -> str: """Execute Python 3 code in an isolated environment and return stdout or an error.""" - args = ( - self.nsjail_binary, "-Mo", - "--rlimit_as", "700", - "--chroot", "/", - "-E", "LANG=en_US.UTF-8", - "-R/usr", "-R/lib", "-R/lib64", - "--user", "nobody", - "--group", "nogroup", - "--time_limit", "2", - "--disable_proc", - "--iface_no_lo", - "--cgroup_mem_max=52428800", - "--cgroup_mem_mount", str(CGROUP_MEMORY_PARENT.parent), - "--cgroup_mem_parent", CGROUP_MEMORY_PARENT.name, - "--cgroup_pids_max=1", - "--cgroup_pids_mount", str(CGROUP_PIDS_PARENT.parent), - "--cgroup_pids_parent", CGROUP_PIDS_PARENT.name, - "--quiet", "--", - self.python_binary, "-ISq", "-c", code - ) - - try: - proc = subprocess.run(args, capture_output=True, env=ENV, text=True) - except ValueError: - return "ValueError: embedded null byte" + with NamedTemporaryFile() as nsj_log: + args = ( + self.nsjail_binary, "-Mo", + "--rlimit_as", "700", + "--chroot", "/", + "-E", "LANG=en_US.UTF-8", + "-R/usr", "-R/lib", "-R/lib64", + "--user", "nobody", + "--group", "nogroup", + "--time_limit", "2", + "--disable_proc", + "--iface_no_lo", + "--log", nsj_log.name, + "--cgroup_mem_max=52428800", + "--cgroup_mem_mount", str(CGROUP_MEMORY_PARENT.parent), + "--cgroup_mem_parent", CGROUP_MEMORY_PARENT.name, + "--cgroup_pids_max=1", + "--cgroup_pids_mount", str(CGROUP_PIDS_PARENT.parent), + "--cgroup_pids_parent", CGROUP_PIDS_PARENT.name, + "--", + self.python_binary, "-ISq", "-c", code + ) + + try: + msg = "Executing code..." + if DEBUG: + msg = f"{msg[:-3]}:\n{textwrap.indent(code, ' ')}" + log.info(msg) + + proc = subprocess.run(args, capture_output=True, env=ENV, text=True) + except ValueError: + return "ValueError: embedded null byte" + + self._parse_log(nsj_log) if proc.returncode == 0: output = proc.stdout elif proc.returncode == 1: - filtered = (line for line in proc.stderr.split("\n") if not line.startswith("[")) - output = "\n".join(filtered) + output = proc.stderr elif proc.returncode == 109: return "timed out or memory limit exceeded" elif proc.returncode == 255: diff --git a/tests/api/test_eval.py b/tests/api/test_eval.py index a5b83fd..bcd0ec4 100644 --- a/tests/api/test_eval.py +++ b/tests/api/test_eval.py @@ -26,7 +26,7 @@ class TestEvalResource(SnekAPITestCase): self.assertEqual(expected, result.json) def test_post_invalid_content_type_415(self): - body = "{\"input\": \"foo\"}" + body = "{'input': 'foo'}" headers = {"Content-Type": "application/xml"} result = self.simulate_post(self.PATH, body=body, headers=headers) -- cgit v1.2.3 From 8ce01c95e4b1a31ac93c1fd7b9651a360b6ff3ed Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Wed, 5 Jun 2019 09:23:32 -0700 Subject: Revert "Add basic logger for responses" Gunicorn's access logger makes the middleware redundant. This reverts commit c1c854e0ad4deb5c4fa8eb2a35b43a042f15255c. --- snekbox/api/middleware/__init__.py | 3 --- snekbox/api/middleware/logger.py | 11 ----------- snekbox/api/snekapi.py | 3 +-- 3 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 snekbox/api/middleware/__init__.py delete mode 100644 snekbox/api/middleware/logger.py diff --git a/snekbox/api/middleware/__init__.py b/snekbox/api/middleware/__init__.py deleted file mode 100644 index ba97ca6..0000000 --- a/snekbox/api/middleware/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .logger import LoggingMiddleware - -__all__ = ("LoggingMiddleware",) diff --git a/snekbox/api/middleware/logger.py b/snekbox/api/middleware/logger.py deleted file mode 100644 index 393fc64..0000000 --- a/snekbox/api/middleware/logger.py +++ /dev/null @@ -1,11 +0,0 @@ -import logging - -log = logging.getLogger("snekbox.api") - - -class LoggingMiddleware: - """Log basic information about responses.""" - - def process_response(self, req, resp, resource, req_succeeded): - """Log the method, route, and status of a response.""" - log.info(f"{req.method} {req.relative_uri} {resp.status}") diff --git a/snekbox/api/snekapi.py b/snekbox/api/snekapi.py index 849e7d6..cb0356a 100644 --- a/snekbox/api/snekapi.py +++ b/snekbox/api/snekapi.py @@ -1,6 +1,5 @@ import falcon -from .middleware import LoggingMiddleware from .resources import EvalResource @@ -22,6 +21,6 @@ class SnekAPI(falcon.API): """ def __init__(self, *args, **kwargs): - super().__init__(middleware=[LoggingMiddleware()], *args, **kwargs) + super().__init__(*args, **kwargs) self.add_route("/eval", EvalResource()) -- cgit v1.2.3 From 66d3836dd27f3b0e9f1cb780c7c37c8c0f081c70 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Wed, 5 Jun 2019 00:46:04 -0700 Subject: Respond to eval with stdout, stderr, and the return code The previous implementation limited the client's flexibility in presenting the results of the process. A process can write to both stdout and stderr and do so even when the return code is not 0 or 1. * Return a CompletedProcess from NsJail * Don't check the return code; this should be done client-side now --- snekbox/api/resources/eval.py | 23 +++++++++++++++++------ snekbox/nsjail.py | 33 ++++++++++----------------------- tests/api/__init__.py | 8 +++++++- tests/api/test_eval.py | 5 +++-- 4 files changed, 37 insertions(+), 32 deletions(-) diff --git a/snekbox/api/resources/eval.py b/snekbox/api/resources/eval.py index b2f4260..4779557 100644 --- a/snekbox/api/resources/eval.py +++ b/snekbox/api/resources/eval.py @@ -36,7 +36,16 @@ class EvalResource: @validate(REQ_SCHEMA) def on_post(self, req, resp): """ - Evaluate Python code and return the result. + Evaluate Python code and return stdout, stderr, and the return code. + + The return codes mostly resemble those of a Unix shell. Some noteworthy cases: + + - None + The NsJail process failed to launch + - 137 (SIGKILL) + Typically because NsJail killed the Python process due to time or memory constraints + - 255 + NsJail encountered a fatal error Request body: @@ -47,8 +56,9 @@ class EvalResource: Response format: >>> { - ... "input": "print(1 + 1)", - ... "output": "2\\n" + ... "stdout": "2\\n", + ... "stderr": "", + ... "returncode": 0 ... } Status codes: @@ -63,12 +73,13 @@ class EvalResource: code = req.media["input"] try: - output = self.nsjail.python3(code) + result = self.nsjail.python3(code) except Exception: log.exception("An exception occurred while trying to process the request") raise falcon.HTTPInternalServerError resp.media = { - "input": code, - "output": output + "stdout": result.stdout, + "stderr": result.stderr, + "returncode": result.returncode } diff --git a/snekbox/nsjail.py b/snekbox/nsjail.py index 0ebfe0c..ff12ec4 100644 --- a/snekbox/nsjail.py +++ b/snekbox/nsjail.py @@ -100,8 +100,8 @@ class NsJail: # Treat fatal as error. log.error(msg) - def python3(self, code: str) -> str: - """Execute Python 3 code in an isolated environment and return stdout or an error.""" + def python3(self, code: str) -> subprocess.CompletedProcess: + """Execute Python 3 code in an isolated environment and return the completed process.""" with NamedTemporaryFile() as nsj_log: args = ( self.nsjail_binary, "-Mo", @@ -125,29 +125,16 @@ class NsJail: self.python_binary, "-ISq", "-c", code ) - try: - msg = "Executing code..." - if DEBUG: - msg = f"{msg[:-3]}:\n{textwrap.indent(code, ' ')}" - log.info(msg) + msg = "Executing code..." + if DEBUG: + msg = f"{msg[:-3]}:\n{textwrap.indent(code, ' ')}" + log.info(msg) - proc = subprocess.run(args, capture_output=True, env=ENV, text=True) + try: + result = subprocess.run(args, capture_output=True, env=ENV, text=True) except ValueError: - return "ValueError: embedded null byte" + return subprocess.CompletedProcess(args, None, "", "ValueError: embedded null byte") self._parse_log(nsj_log) - if proc.returncode == 0: - output = proc.stdout - elif proc.returncode == 1: - output = proc.stderr - elif proc.returncode == 109: - return "timed out or memory limit exceeded" - elif proc.returncode == 255: - return "permission denied (root required)" - elif proc.returncode: - return f"unknown error, code: {proc.returncode}" - else: - return "unknown error, no error code" - - return output + return result diff --git a/tests/api/__init__.py b/tests/api/__init__.py index fd4679a..dcee5b5 100644 --- a/tests/api/__init__.py +++ b/tests/api/__init__.py @@ -1,3 +1,4 @@ +from subprocess import CompletedProcess from unittest import mock from falcon import testing @@ -11,7 +12,12 @@ class SnekAPITestCase(testing.TestCase): self.patcher = mock.patch("snekbox.api.resources.eval.NsJail", autospec=True) self.mock_nsjail = self.patcher.start() - self.mock_nsjail.return_value.python3.return_value = "test output" + self.mock_nsjail.return_value.python3.return_value = CompletedProcess( + args=[], + returncode=0, + stdout="output", + stderr="error" + ) self.addCleanup(self.patcher.stop) self.app = SnekAPI() diff --git a/tests/api/test_eval.py b/tests/api/test_eval.py index bcd0ec4..03f0e39 100644 --- a/tests/api/test_eval.py +++ b/tests/api/test_eval.py @@ -9,8 +9,9 @@ class TestEvalResource(SnekAPITestCase): result = self.simulate_post(self.PATH, json=body) self.assertEqual(result.status_code, 200) - self.assertEqual(body["input"], result.json["input"]) - self.assertEqual("test output", result.json["output"]) + self.assertEqual("output", result.json["stdout"]) + self.assertEqual("error", result.json["stderr"]) + self.assertEqual(0, result.json["returncode"]) def test_post_invalid_schema_400(self): body = {"stuff": "foo"} -- cgit v1.2.3