Respond to eval with stdout, stderr, and the return code

The previous implementation limited the client's flexibility in
presenting the results of the process. A process can write to both
stdout and stderr and do so even when the return code is not 0 or 1.

* Return a CompletedProcess from NsJail
* Don't check the return code; this should be done client-side now

4 files changed, 37 insertions, 32 deletions

diff --git a/snekbox/api/resources/eval.py b/snekbox/api/resources/eval.py
index b2f4260..4779557 100644
--- a/snekbox/api/resources/eval.py
+++ b/snekbox/api/resources/eval.py
@@ -36,7 +36,16 @@ class EvalResource:
     @validate(REQ_SCHEMA)
     def on_post(self, req, resp):
         """
-        Evaluate Python code and return the result.
+        Evaluate Python code and return stdout, stderr, and the return code.
+
+        The return codes mostly resemble those of a Unix shell. Some noteworthy cases:
+
+        - None
+            The NsJail process failed to launch
+        - 137 (SIGKILL)
+            Typically because NsJail killed the Python process due to time or memory constraints
+        - 255
+            NsJail encountered a fatal error
 
         Request body:
 
@@ -47,8 +56,9 @@ class EvalResource:
         Response format:
 
         >>> {
-        ...     "input": "print(1 + 1)",
-        ...     "output": "2\\n"
+        ...     "stdout": "2\\n",
+        ...     "stderr": "",
+        ...     "returncode": 0
         ... }
 
         Status codes:
@@ -63,12 +73,13 @@ class EvalResource:
         code = req.media["input"]
 
         try:
-            output = self.nsjail.python3(code)
+            result = self.nsjail.python3(code)
         except Exception:
             log.exception("An exception occurred while trying to process the request")
             raise falcon.HTTPInternalServerError
 
         resp.media = {
-            "input": code,
-            "output": output
+            "stdout": result.stdout,
+            "stderr": result.stderr,
+            "returncode": result.returncode
         }
diff --git a/snekbox/nsjail.py b/snekbox/nsjail.py
index 0ebfe0c..ff12ec4 100644
--- a/snekbox/nsjail.py
+++ b/snekbox/nsjail.py
@@ -100,8 +100,8 @@ class NsJail:
                 # Treat fatal as error.
                 log.error(msg)
 
-    def python3(self, code: str) -> str:
-        """Execute Python 3 code in an isolated environment and return stdout or an error."""
+    def python3(self, code: str) -> subprocess.CompletedProcess:
+        """Execute Python 3 code in an isolated environment and return the completed process."""
         with NamedTemporaryFile() as nsj_log:
             args = (
                 self.nsjail_binary, "-Mo",
@@ -125,29 +125,16 @@ class NsJail:
                 self.python_binary, "-ISq", "-c", code
             )
 
-            try:
-                msg = "Executing code..."
-                if DEBUG:
-                    msg = f"{msg[:-3]}:\n{textwrap.indent(code, '    ')}"
-                log.info(msg)
+            msg = "Executing code..."
+            if DEBUG:
+                msg = f"{msg[:-3]}:\n{textwrap.indent(code, '    ')}"
+            log.info(msg)
 
-                proc = subprocess.run(args, capture_output=True, env=ENV, text=True)
+            try:
+                result = subprocess.run(args, capture_output=True, env=ENV, text=True)
             except ValueError:
-                return "ValueError: embedded null byte"
+                return subprocess.CompletedProcess(args, None, "", "ValueError: embedded null byte")
 
             self._parse_log(nsj_log)
 
-        if proc.returncode == 0:
-            output = proc.stdout
-        elif proc.returncode == 1:
-            output = proc.stderr
-        elif proc.returncode == 109:
-            return "timed out or memory limit exceeded"
-        elif proc.returncode == 255:
-            return "permission denied (root required)"
-        elif proc.returncode:
-            return f"unknown error, code: {proc.returncode}"
-        else:
-            return "unknown error, no error code"
-
-        return output
+        return result
diff --git a/tests/api/__init__.py b/tests/api/__init__.py
index fd4679a..dcee5b5 100644
--- a/tests/api/__init__.py
+++ b/tests/api/__init__.py
@@ -1,3 +1,4 @@
+from subprocess import CompletedProcess
 from unittest import mock
 
 from falcon import testing
@@ -11,7 +12,12 @@ class SnekAPITestCase(testing.TestCase):
 
         self.patcher = mock.patch("snekbox.api.resources.eval.NsJail", autospec=True)
         self.mock_nsjail = self.patcher.start()
-        self.mock_nsjail.return_value.python3.return_value = "test output"
+        self.mock_nsjail.return_value.python3.return_value = CompletedProcess(
+            args=[],
+            returncode=0,
+            stdout="output",
+            stderr="error"
+        )
         self.addCleanup(self.patcher.stop)
 
         self.app = SnekAPI()
diff --git a/tests/api/test_eval.py b/tests/api/test_eval.py
index bcd0ec4..03f0e39 100644
--- a/tests/api/test_eval.py
+++ b/tests/api/test_eval.py
@@ -9,8 +9,9 @@ class TestEvalResource(SnekAPITestCase):
         result = self.simulate_post(self.PATH, json=body)
 
         self.assertEqual(result.status_code, 200)
-        self.assertEqual(body["input"], result.json["input"])
-        self.assertEqual("test output", result.json["output"])
+        self.assertEqual("output", result.json["stdout"])
+        self.assertEqual("error", result.json["stderr"])
+        self.assertEqual(0, result.json["returncode"])
 
     def test_post_invalid_schema_400(self):
         body = {"stuff": "foo"}