From d931810f51f27e55455be7e8a87c5a27a04d431d Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Wed, 24 Feb 2021 12:07:13 +0100 Subject: Add unittest template --- resources/unittest_template.py | 81 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 resources/unittest_template.py (limited to 'resources/unittest_template.py') diff --git a/resources/unittest_template.py b/resources/unittest_template.py new file mode 100644 index 0000000..7e21b97 --- /dev/null +++ b/resources/unittest_template.py @@ -0,0 +1,81 @@ +import ast +import io +import os +import sys +import traceback +import unittest +from itertools import chain +from types import ModuleType, SimpleNamespace +from typing import NoReturn +from unittest import mock + +### USER CODE + + +class RunnerTestCase(unittest.TestCase): +### UNIT CODE + + +# Fake file object not writing anything +DEVNULL = SimpleNamespace(write=lambda *_: None, flush=lambda *_: None) + +RESULT = io.StringIO() +ORIGINAL_STDOUT = sys.stdout + +sys.stdout = DEVNULL +sys.stderr = DEVNULL + + +def _exit_sandbox(code: int) -> NoReturn: + """ + Codes: + - 0: Executed with success + - 5: Syntax error while parsing user code + - 99: Internal error + """ + result_content = RESULT.getvalue() + + print( + f"{result_content}", + file=ORIGINAL_STDOUT + ) + sys.exit(code) + + +def _load_user_module() -> ModuleType: + try: + ast.parse(USER_CODE, "") + except SyntaxError: + RESULT.write("".join(traceback.format_exception(*sys.exc_info(), limit=0))) + _exit_sandbox(5) + + _module = ModuleType("module") + exec(USER_CODE, _module.__dict__) + + return _module + + +def _main() -> None: + suite = unittest.defaultTestLoader.loadTestsFromTestCase(RunnerTestCase) + result = suite.run(unittest.TestResult()) + + RESULT.write(str(int(result.wasSuccessful()))) + + if not result.wasSuccessful(): + RESULT.write( + ";".join(chain( + (error[0]._testMethodName.lstrip("test_") for error in result.errors), + (failure[0]._testMethodName.lstrip("test_") for failure in result.failures) + )) + ) + + _exit_sandbox(0) + + +try: + module = _load_user_module() + _main() +except Exception: + print("Uncaught exception:\n", file=RESULT) + traceback.print_exc(file=RESULT) + _exit_sandbox(99) -- cgit v1.2.3 From 0e6e8e239a2778519e51a04b9ac05f049cb763df Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Wed, 24 Feb 2021 13:54:08 +0100 Subject: Don't lint the unittest template --- resources/unittest_template.py | 1 + 1 file changed, 1 insertion(+) (limited to 'resources/unittest_template.py') diff --git a/resources/unittest_template.py b/resources/unittest_template.py index 7e21b97..c792944 100644 --- a/resources/unittest_template.py +++ b/resources/unittest_template.py @@ -1,3 +1,4 @@ +# flake8: noqa import ast import io import os -- cgit v1.2.3 From 0a9026dcdd23eaf7c48256eb7da5af774892673b Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Wed, 24 Feb 2021 15:16:13 +0100 Subject: Document unittest code --- backend/routes/forms/submit.py | 2 ++ backend/routes/forms/unittesting.py | 20 ++++++++++++++------ resources/unittest_template.py | 13 +++++++------ 3 files changed, 23 insertions(+), 12 deletions(-) (limited to 'resources/unittest_template.py') diff --git a/backend/routes/forms/submit.py b/backend/routes/forms/submit.py index 7618a33..d6b549e 100644 --- a/backend/routes/forms/submit.py +++ b/backend/routes/forms/submit.py @@ -131,12 +131,14 @@ class SubmitForm(Route): except ValidationError as e: return JSONResponse(e.errors(), status_code=422) + # Run unittests if needed has_unittests = any("unittests" in question.data for question in form.questions) if has_unittests: unittest_results = await execute_unittest(response_obj, form) was_successful = all(test.passed for test in unittest_results) if not was_successful: + # Return 500 if we encountered an internal error (code 99). status_code = 500 if any( test.return_code == 99 for test in unittest_results ) else 403 diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index 0cb7d8d..e038f3a 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -30,6 +30,7 @@ def filter_unittests(form: Form) -> Form: def _make_unit_code(units: dict[str, str]) -> str: + """Compose a dict mapping unit names to their code into an actual class body.""" result = "" for unit_name, unit_code in units.items(): @@ -39,14 +40,16 @@ def _make_unit_code(units: dict[str, str]) -> str: def _make_user_code(code: str) -> str: - # Make sure that we we escape triple quotes and backslashes in the user code - code = code.replace('"""', '\\"""').replace("\\", "\\\\") - return f'USER_CODE = """{code}"""' + """Compose the user code into an actual string variable.""" + # Make sure that we we escape triple quotes in the user code + code = code.replace('"""', '\\"""') + return f'USER_CODE = r"""{code}"""' async def _post_eval(code: str) -> Optional[dict[str, str]]: - data = {"input": code} + """Post the eval to snekbox and return the response.""" async with httpx.AsyncClient() as client: + data = {"input": code} response = await client.post(SNEKBOX_URL, json=data) if not response.status_code == 200: @@ -56,12 +59,14 @@ async def _post_eval(code: str) -> Optional[dict[str, str]]: async def execute_unittest(form_response: FormResponse, form: Form) -> list[UnittestResult]: + """Execute all the unittests in this form and return the results.""" unittest_results = [] for question in form.questions: if question.type == "code" and "unittests" in question.data: passed = False + # Tests starting with an hashtag should have censored names. hidden_test_counter = count(1) hidden_tests = { test.lstrip("#"): next(hidden_test_counter) @@ -69,19 +74,20 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit if test.startswith("#") } + # Compose runner code unit_code = _make_unit_code(question.data["unittests"]) user_code = _make_user_code(form_response.response[question.id]) code = TEST_TEMPLATE.replace("### USER CODE", user_code) code = code.replace("### UNIT CODE", unit_code) - # Make sure that the code is well formatted (we don't check for the user code) + # Make sure that the code is well formatted (we don't check for the user code). try: ast.parse(code) except SyntaxError: return_code = 99 result = "Invalid generated unit code." - + # The runner is correctly formatted, we can run it. else: response = await _post_eval(code) @@ -91,6 +97,7 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit else: return_code = int(response["returncode"]) + # Another code has been returned by CPython because of another failure. if return_code not in (0, 5, 99): return_code = 99 result = "Internal error." @@ -98,6 +105,7 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit stdout = response["stdout"] passed = bool(int(stdout[0])) + # If the test failed, we have to populate the result string. if not passed: failed_tests = stdout[1:].strip().split(";") diff --git a/resources/unittest_template.py b/resources/unittest_template.py index c792944..4c9b0bb 100644 --- a/resources/unittest_template.py +++ b/resources/unittest_template.py @@ -1,4 +1,5 @@ # flake8: noqa +"""This template is used inside snekbox to evaluate and test user code.""" import ast import io import os @@ -23,27 +24,26 @@ DEVNULL = SimpleNamespace(write=lambda *_: None, flush=lambda *_: None) RESULT = io.StringIO() ORIGINAL_STDOUT = sys.stdout +# stdout/err is patched in order to control what is outputted by the runner sys.stdout = DEVNULL sys.stderr = DEVNULL def _exit_sandbox(code: int) -> NoReturn: """ + Exit the sandbox by printing the result to the actual stdout and exit with the provided code. + Codes: - 0: Executed with success - 5: Syntax error while parsing user code - 99: Internal error """ - result_content = RESULT.getvalue() - - print( - f"{result_content}", - file=ORIGINAL_STDOUT - ) + print(RESULT.getvalue(), file=ORIGINAL_STDOUT, end="") sys.exit(code) def _load_user_module() -> ModuleType: + """Load the user code into a new module and return it.""" try: ast.parse(USER_CODE, "") except SyntaxError: @@ -74,6 +74,7 @@ def _main() -> None: try: + # Load the user code as a global module variable module = _load_user_module() _main() except Exception: -- cgit v1.2.3 From 80433234dba5c8504e79fc829d05f1ef68a82f9f Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Wed, 24 Feb 2021 15:18:55 +0100 Subject: Move most of the unittest template inside of the handler --- resources/unittest_template.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'resources/unittest_template.py') diff --git a/resources/unittest_template.py b/resources/unittest_template.py index 4c9b0bb..7919b5c 100644 --- a/resources/unittest_template.py +++ b/resources/unittest_template.py @@ -18,17 +18,6 @@ class RunnerTestCase(unittest.TestCase): ### UNIT CODE -# Fake file object not writing anything -DEVNULL = SimpleNamespace(write=lambda *_: None, flush=lambda *_: None) - -RESULT = io.StringIO() -ORIGINAL_STDOUT = sys.stdout - -# stdout/err is patched in order to control what is outputted by the runner -sys.stdout = DEVNULL -sys.stderr = DEVNULL - - def _exit_sandbox(code: int) -> NoReturn: """ Exit the sandbox by printing the result to the actual stdout and exit with the provided code. @@ -74,10 +63,20 @@ def _main() -> None: try: + # Fake file object not writing anything + DEVNULL = SimpleNamespace(write=lambda *_: None, flush=lambda *_: None) + + RESULT = io.StringIO() + ORIGINAL_STDOUT = sys.stdout + + # stdout/err is patched in order to control what is outputted by the runner + sys.stdout = DEVNULL + sys.stderr = DEVNULL + # Load the user code as a global module variable module = _load_user_module() _main() except Exception: print("Uncaught exception:\n", file=RESULT) traceback.print_exc(file=RESULT) - _exit_sandbox(99) + _exit_sandbox(99) \ No newline at end of file -- cgit v1.2.3 From 5c17be752dedb46370af3b1c370baa59b17a1d4b Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Wed, 24 Feb 2021 15:45:08 +0100 Subject: Don't communicate the traceback back to the backend --- resources/unittest_template.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'resources/unittest_template.py') diff --git a/resources/unittest_template.py b/resources/unittest_template.py index 7919b5c..755f7cc 100644 --- a/resources/unittest_template.py +++ b/resources/unittest_template.py @@ -77,6 +77,5 @@ try: module = _load_user_module() _main() except Exception: - print("Uncaught exception:\n", file=RESULT) - traceback.print_exc(file=RESULT) + print("Uncaught exception inside runner.", file=RESULT) _exit_sandbox(99) \ No newline at end of file -- cgit v1.2.3 From 8939c8e127d49f9f534679d5ff9bdef907730e13 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Thu, 25 Feb 2021 14:15:15 +0100 Subject: Add return code 6 for exceptions when loading module --- backend/routes/forms/unittesting.py | 2 +- resources/unittest_template.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) (limited to 'resources/unittest_template.py') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index f7f6072..c00fc4c 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -98,7 +98,7 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit return_code = int(response["returncode"]) # Another code has been returned by CPython because of another failure. - if return_code not in (0, 5, 99): + if return_code not in (0, 5, 6, 99): return_code = 99 result = "Internal error." else: diff --git a/resources/unittest_template.py b/resources/unittest_template.py index 755f7cc..02d3894 100644 --- a/resources/unittest_template.py +++ b/resources/unittest_template.py @@ -25,6 +25,7 @@ def _exit_sandbox(code: int) -> NoReturn: Codes: - 0: Executed with success - 5: Syntax error while parsing user code + - 6: Uncaught exception while loading user code - 99: Internal error """ print(RESULT.getvalue(), file=ORIGINAL_STDOUT, end="") @@ -74,8 +75,12 @@ try: sys.stderr = DEVNULL # Load the user code as a global module variable - module = _load_user_module() + try: + module = _load_user_module() + except Exception: + RESULT.write("Uncaught exception while loading user code.") + _exit_sandbox(6) _main() except Exception: - print("Uncaught exception inside runner.", file=RESULT) - _exit_sandbox(99) \ No newline at end of file + RESULT.write("Uncaught exception inside runner.") + _exit_sandbox(99) -- cgit v1.2.3 From 7d34cb8563d8c01e5f5d1b038e0fbd507063e853 Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Thu, 25 Feb 2021 14:28:51 +0100 Subject: Add return code 7 for processes killed by NsJail --- backend/routes/forms/unittesting.py | 47 ++++++++++++++++++++----------------- resources/unittest_template.py | 2 ++ 2 files changed, 27 insertions(+), 22 deletions(-) (limited to 'resources/unittest_template.py') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index c00fc4c..57bf5db 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -50,7 +50,7 @@ async def _post_eval(code: str) -> Optional[dict[str, str]]: """Post the eval to snekbox and return the response.""" async with httpx.AsyncClient() as client: data = {"input": code} - response = await client.post(SNEKBOX_URL, json=data) + response = await client.post(SNEKBOX_URL, json=data, timeout=10) if not response.status_code == 200: return @@ -97,30 +97,33 @@ async def execute_unittest(form_response: FormResponse, form: Form) -> list[Unit else: return_code = int(response["returncode"]) + # Parse the stdout if the tests ran successfully + if return_code == 0: + stdout = response["stdout"] + passed = bool(int(stdout[0])) + + # If the test failed, we have to populate the result string. + if not passed: + failed_tests = stdout[1:].strip().split(";") + + # Redact failed hidden tests + for i, failed_test in enumerate(failed_tests[:]): + if failed_test in hidden_tests: + failed_tests[i] = f"hidden_test_{hidden_tests[failed_test]}" + + result = ";".join(failed_tests) + else: + result = "" + elif return_code in (5, 6, 99): + result = response["stdout"] + # Killed by NsJail + elif return_code == 137: + return_code = 7 + result = "Timed out or ran out of memory." # Another code has been returned by CPython because of another failure. - if return_code not in (0, 5, 6, 99): + else: return_code = 99 result = "Internal error." - else: - # Parse the stdout if the tests ran successfully - if return_code == 0: - stdout = response["stdout"] - passed = bool(int(stdout[0])) - - # If the test failed, we have to populate the result string. - if not passed: - failed_tests = stdout[1:].strip().split(";") - - # Redact failed hidden tests - for i, failed_test in enumerate(failed_tests[:]): - if failed_test in hidden_tests: - failed_tests[i] = f"hidden_test_{hidden_tests[failed_test]}" - - result = ";".join(failed_tests) - else: - result = "" - else: - result = response["stdout"] unittest_results.append(UnittestResult( question_id=question.id, diff --git a/resources/unittest_template.py b/resources/unittest_template.py index 02d3894..38e3be8 100644 --- a/resources/unittest_template.py +++ b/resources/unittest_template.py @@ -27,6 +27,8 @@ def _exit_sandbox(code: int) -> NoReturn: - 5: Syntax error while parsing user code - 6: Uncaught exception while loading user code - 99: Internal error + + 137 can also be generated by NsJail when killing the process. """ print(RESULT.getvalue(), file=ORIGINAL_STDOUT, end="") sys.exit(code) -- cgit v1.2.3 From 9d2c3794a4c95f6c63a7de64172bc35a68403a4c Mon Sep 17 00:00:00 2001 From: Matteo Bertucci Date: Fri, 26 Feb 2021 14:21:00 +0100 Subject: Use base64 encoded code snippets --- backend/routes/forms/unittesting.py | 8 ++++---- resources/unittest_template.py | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'resources/unittest_template.py') diff --git a/backend/routes/forms/unittesting.py b/backend/routes/forms/unittesting.py index 175701f..b12cff2 100644 --- a/backend/routes/forms/unittesting.py +++ b/backend/routes/forms/unittesting.py @@ -1,3 +1,4 @@ +import base64 from collections import namedtuple from itertools import count from textwrap import indent @@ -45,10 +46,9 @@ def _make_unit_code(units: dict[str, str]) -> str: def _make_user_code(code: str) -> str: - """Compose the user code into an actual string variable.""" - # Make sure that we we escape triple quotes in the user code - code = code.replace('"""', '\\"""') - return f'USER_CODE = r"""{code}"""' + """Compose the user code into an actual base64-encoded string variable.""" + code = base64.b64encode(code.encode("utf8")).decode("utf8") + return f'USER_CODE = b"{code}"' async def _post_eval(code: str) -> dict[str, str]: diff --git a/resources/unittest_template.py b/resources/unittest_template.py index 38e3be8..2410278 100644 --- a/resources/unittest_template.py +++ b/resources/unittest_template.py @@ -1,6 +1,7 @@ # flake8: noqa """This template is used inside snekbox to evaluate and test user code.""" import ast +import base64 import io import os import sys @@ -36,14 +37,15 @@ def _exit_sandbox(code: int) -> NoReturn: def _load_user_module() -> ModuleType: """Load the user code into a new module and return it.""" + code = base64.b64decode(USER_CODE).decode("utf8") try: - ast.parse(USER_CODE, "") + ast.parse(code, "") except SyntaxError: RESULT.write("".join(traceback.format_exception(*sys.exc_info(), limit=0))) _exit_sandbox(5) _module = ModuleType("module") - exec(USER_CODE, _module.__dict__) + exec(code, _module.__dict__) return _module -- cgit v1.2.3