Merge #108 - allow custom arguments to be passed to eval

5 files changed, 66 insertions, 7 deletions

diff --git a/.github/workflows/lint-test-build-push.yaml b/.github/workflows/lint-test-build-push.yaml
index 92ca53e..7b862ec 100644
--- a/.github/workflows/lint-test-build-push.yaml
+++ b/.github/workflows/lint-test-build-push.yaml
@@ -95,10 +95,17 @@ jobs:
       # pre-commit's venv doesn't work with user installs.
       # Skip the flake8 hook because the following step will run it.
       - name: Run pre-commit hooks
+        id: run-pre-commit-hooks
         run: >-
           docker exec snekbox_dev /bin/bash -c
           'PIP_USER=0 SKIP=flake8 pre-commit run --all-files'
 
+      - name: Show pre-commit logs
+        if: always() && steps.run-pre-commit-hooks.outcome != 'success'
+        run: >-
+          docker exec snekbox_dev /bin/bash -c
+          'cat /root/.cache/pre-commit/pre-commit.log'
+
       # This runs `flake8` in the container and asks `flake8` to output
       # linting errors in the format of the command for registering workflow
       # error messages/annotations. This means that Github Actions will pick
diff --git a/snekbox/api/resources/eval.py b/snekbox/api/resources/eval.py
index d96b811..9560d0b 100644
--- a/snekbox/api/resources/eval.py
+++ b/snekbox/api/resources/eval.py
@@ -23,6 +23,12 @@ class EvalResource:
         "properties": {
             "input": {
                 "type": "string"
+            },
+            "args": {
+                "type": "array",
+                "items": {
+                    "type": "string"
+                }
             }
         },
         "required": [
@@ -38,6 +44,10 @@ class EvalResource:
         """
         Evaluate Python code and return stdout, stderr, and the return code.
 
+        A list of arguments for the Python subprocess can be specified as `args`.
+        Otherwise, the default argument "-c" is used to execute the input code.
+        The input code is always passed as the last argument to Python.
+
         The return codes mostly resemble those of a Unix shell. Some noteworthy cases:
 
         - None
@@ -50,13 +60,14 @@ class EvalResource:
         Request body:
 
         >>> {
-        ...     "input": "print(1 + 1)"
+        ...     "input": "[i for i in range(1000)]",
+        ...     "args": ["-m", "timeit"] # This is optional
         ... }
 
         Response format:
 
         >>> {
-        ...     "stdout": "2\\n",
+        ...     "stdout": "10000 loops, best of 5: 23.8 usec per loop\n",
         ...     "returncode": 0
         ... }
 
@@ -70,9 +81,10 @@ class EvalResource:
             Unsupported content type; only application/JSON is supported
         """
         code = req.media["input"]
+        args = req.media.get("args", ("-c",))
 
         try:
-            result = self.nsjail.python3(code)
+            result = self.nsjail.python3(code, py_args=args)
         except Exception:
             log.exception("An exception occurred while trying to process the request")
             raise falcon.HTTPInternalServerError
diff --git a/snekbox/nsjail.py b/snekbox/nsjail.py
index 9367cb2..ce2b28f 100644
--- a/snekbox/nsjail.py
+++ b/snekbox/nsjail.py
@@ -171,12 +171,21 @@ class NsJail:
 
         return "".join(output)
 
-    def python3(self, code: str, *args) -> CompletedProcess:
+    def python3(
+        self,
+        code: str,
+        *,
+        nsjail_args: Iterable[str] = (),
+        py_args: Iterable[str] = ("-c",)
+    ) -> CompletedProcess:
         """
         Execute Python 3 code in an isolated environment and return the completed process.
 
-        Additional arguments passed will be used to override the values in the NsJail config.
+        The `nsjail_args` passed will be used to override the values in the NsJail config.
         These arguments are only options for NsJail; they do not affect Python's arguments.
+
+        `py_args` are arguments to pass to the Python subprocess before the code,
+        which is the last argument. By default, it's "-c", which executes the code given.
         """
         cgroup = self._create_dynamic_cgroups()
 
@@ -188,9 +197,9 @@ class NsJail:
                 # Set our dynamically created parent cgroups
                 "--cgroup_mem_parent", cgroup,
                 "--cgroup_pids_parent", cgroup,
-                *args,
+                *nsjail_args,
                 "--",
-                self.config.exec_bin.path, *self.config.exec_bin.arg, "-c", code
+                self.config.exec_bin.path, *self.config.exec_bin.arg, *py_args, code
             )
 
             msg = "Executing code..."
diff --git a/tests/api/test_eval.py b/tests/api/test_eval.py
index 3350763..bdeee3e 100644
--- a/tests/api/test_eval.py
+++ b/tests/api/test_eval.py
@@ -25,6 +25,24 @@ class TestEvalResource(SnekAPITestCase):
 
         self.assertEqual(expected, result.json)
 
+    def test_post_invalid_data_400(self):
+        bodies = (
+            {"input": 400}, {"input": "", "args": [400]}
+        )
+
+        for body in bodies:
+            with self.subTest():
+                result = self.simulate_post(self.PATH, json=body)
+
+                self.assertEqual(result.status_code, 400)
+
+                expected = {
+                    "title": "Request data failed validation",
+                    "description": "400 is not of type 'string'"
+                }
+
+                self.assertEqual(expected, result.json)
+
     def test_post_invalid_content_type_415(self):
         body = "{'input': 'foo'}"
         headers = {"Content-Type": "application/xml"}
diff --git a/tests/test_nsjail.py b/tests/test_nsjail.py
index 46193b2..8955b4a 100644
--- a/tests/test_nsjail.py
+++ b/tests/test_nsjail.py
@@ -217,3 +217,16 @@ class NsJailTests(unittest.TestCase):
 
         output = self.nsjail._consume_stdout(nsjail_subprocess)
         self.assertEqual(output, chunk * expected_chunks)
+
+    def test_nsjail_args(self):
+        args = ("foo", "bar")
+        result = self.nsjail.python3("", nsjail_args=args)
+
+        self.assertEqual(result.args[9:11], args)
+
+    def test_py_args(self):
+        args = ("-m", "timeit")
+        result = self.nsjail.python3("", py_args=args)
+
+        self.assertEqual(result.returncode, 0)
+        self.assertEqual(result.args[-3:-1], args)