From bc130a4d44f38824b6173c0babff4eefe18ac1db Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 15 Jun 2019 18:32:32 -0700 Subject: Merge stdout and stderr Removes the need for redirecting stderr using contextlib in the input. Furthermore, it captures errors which don't directly come from the input, such as SyntaxErrors. --- snekbox/api/resources/eval.py | 2 -- snekbox/nsjail.py | 10 ++++++++-- tests/api/test_eval.py | 1 - 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/snekbox/api/resources/eval.py b/snekbox/api/resources/eval.py index 4779557..c4bd666 100644 --- a/snekbox/api/resources/eval.py +++ b/snekbox/api/resources/eval.py @@ -57,7 +57,6 @@ class EvalResource: >>> { ... "stdout": "2\\n", - ... "stderr": "", ... "returncode": 0 ... } @@ -80,6 +79,5 @@ class EvalResource: resp.media = { "stdout": result.stdout, - "stderr": result.stderr, "returncode": result.returncode } diff --git a/snekbox/nsjail.py b/snekbox/nsjail.py index ff12ec4..1675b3e 100644 --- a/snekbox/nsjail.py +++ b/snekbox/nsjail.py @@ -131,9 +131,15 @@ class NsJail: log.info(msg) try: - result = subprocess.run(args, capture_output=True, env=ENV, text=True) + result = subprocess.run( + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=ENV, + text=True + ) except ValueError: - return subprocess.CompletedProcess(args, None, "", "ValueError: embedded null byte") + return subprocess.CompletedProcess(args, None, "ValueError: embedded null byte", "") self._parse_log(nsj_log) diff --git a/tests/api/test_eval.py b/tests/api/test_eval.py index 03f0e39..3350763 100644 --- a/tests/api/test_eval.py +++ b/tests/api/test_eval.py @@ -10,7 +10,6 @@ class TestEvalResource(SnekAPITestCase): self.assertEqual(result.status_code, 200) self.assertEqual("output", result.json["stdout"]) - self.assertEqual("error", result.json["stderr"]) self.assertEqual(0, result.json["returncode"]) def test_post_invalid_schema_400(self): -- cgit v1.2.3 From 69989114c59ee223a65341136291370337fead7c Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 15 Jun 2019 21:23:59 -0700 Subject: Use system site instead of relying on virtual environment activation * Remove -S option from Python to re-enable importing of site module * Add environment variable NSJAIL_PATH * Remove environment variables that were passed to NsJail subprocess * Add type annotations to NsJail.__init__() --- snekbox/nsjail.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/snekbox/nsjail.py b/snekbox/nsjail.py index 1675b3e..b1dc34d 100644 --- a/snekbox/nsjail.py +++ b/snekbox/nsjail.py @@ -1,4 +1,5 @@ import logging +import os import re import subprocess import sys @@ -20,16 +21,7 @@ LOG_BLACKLIST = ("Process will be ",) CGROUP_PIDS_PARENT = Path("/sys/fs/cgroup/pids/NSJAIL") CGROUP_MEMORY_PARENT = Path("/sys/fs/cgroup/memory/NSJAIL") -ENV = { - "PATH": ( - "/snekbox/.venv/bin:/usr/local/bin:/usr/local/" - "sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - ), - "LANG": "en_US.UTF-8", - "PYTHON_VERSION": "3.7.3", - "PYTHON_PIP_VERSION": "19.0.3", - "PYTHONDONTWRITEBYTECODE": "1", -} +NSJAIL_PATH = os.getenv("NSJAIL_PATH", "/usr/sbin/nsjail") class NsJail: @@ -50,10 +42,9 @@ class NsJail: - Isolated mode - Neither the script's directory nor the user's site packages are in sys.path - All PYTHON* environment variables are ignored - - Import of the site module is disabled """ - def __init__(self, nsjail_binary="nsjail", python_binary=sys.executable): + def __init__(self, nsjail_binary: str = NSJAIL_PATH, python_binary: str = sys.executable): self.nsjail_binary = nsjail_binary self.python_binary = python_binary @@ -122,7 +113,7 @@ class NsJail: "--cgroup_pids_mount", str(CGROUP_PIDS_PARENT.parent), "--cgroup_pids_parent", CGROUP_PIDS_PARENT.name, "--", - self.python_binary, "-ISq", "-c", code + self.python_binary, "-Iq", "-c", code ) msg = "Executing code..." @@ -135,7 +126,6 @@ class NsJail: args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - env=ENV, text=True ) except ValueError: -- cgit v1.2.3 From 9dc011a01215cb908d490efb3f9329df19a0dc48 Mon Sep 17 00:00:00 2001 From: MarkKoz Date: Sat, 15 Jun 2019 22:34:08 -0700 Subject: Use a custom Gunicorn access log format The handler now shares formats with the rest of the handlers. The message is formatted to show the request method, URL + query, response code, response size in bytes, and the request time in seconds. * Use the default ISO 8601 date format for all handlers --- Pipfile | 10 +++++++++- snekbox/__init__.py | 12 +++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Pipfile b/Pipfile index 6b6aa24..986116d 100644 --- a/Pipfile +++ b/Pipfile @@ -31,7 +31,15 @@ lint = "flake8" precommit = "pre-commit install" test = "pytest tests --cov . --cov-report term-missing -v" report = "pytest tests --cov . --cov-report=html" -snekbox = "gunicorn -w 2 -b 0.0.0.0:8060 --logger-class snekbox.GunicornLogger --access-logfile - snekbox.api.app" +snekbox = """ + gunicorn + -w 2 + -b 0.0.0.0:8060 + --logger-class snekbox.GunicornLogger + --access-logformat '%(m)s %(U)s%(q)s %(s)s %(b)s %(L)ss' + --access-logfile - + snekbox.api.app +""" buildbox = "docker build -t pythondiscord/snekbox:latest -f docker/Dockerfile ." pushbox = "docker push pythondiscord/snekbox:latest" buildboxbase = "docker build -t pythondiscord/snekbox-base:latest -f docker/base.Dockerfile ." diff --git a/snekbox/__init__.py b/snekbox/__init__.py index a48abd5..40b76db 100644 --- a/snekbox/__init__.py +++ b/snekbox/__init__.py @@ -11,10 +11,16 @@ class GunicornLogger(glogging.Logger): """Logger for Gunicorn with custom formatting and support for the DEBUG environment variable.""" error_fmt = "%(asctime)s | %(process)5s | %(name)30s | %(levelname)8s | %(message)s" - datefmt = "%Y-%m-%d %H:%M:%S" + access_fmt = error_fmt + datefmt = None # Use the default ISO 8601 format def setup(self, cfg): - """Set up loggers and set error logger's level to DEBUG if the DEBUG env var is set.""" + """ + Set up loggers and set error logger's level to DEBUG if the DEBUG env var is set. + + Note: Access and syslog handlers would need to be recreated to use a custom date format + because they are created with an unspecified datefmt argument by default. + """ super().setup(cfg) if DEBUG: @@ -28,7 +34,7 @@ class GunicornLogger(glogging.Logger): log = logging.getLogger("snekbox") log.setLevel(logging.DEBUG if DEBUG else logging.INFO) log.propagate = True -formatter = logging.Formatter(GunicornLogger.error_fmt, GunicornLogger.datefmt) +formatter = logging.Formatter(GunicornLogger.error_fmt) handler = logging.StreamHandler(sys.stdout) handler.setFormatter(formatter) log.addHandler(handler) -- cgit v1.2.3